Compare commits

...

1 Commits

Author SHA1 Message Date
Markus Hartung
17eac4978a fix(combo): attach combo diagnostics to round-robin and runtime-unit retry-limit 503s (#11462)
The round-robin combo strategy's terminal "Maximum combo retry limit
reached" 503 (open-sse/services/combo.ts, handleRoundRobinCombo) and the
nested runtime-unit loop used by pipeline/fusion combo steps
(open-sse/services/combo/runtimeUnits.ts) returned a bare errorResponse()
with zero diagnostics, while the priority-strategy path in the same file
already attaches a full errorResponseWithComboDiagnostics trace
(poolSize, attemptOrder, excluded providers/reasons, terminalReason,
recovery hint) for the identical terminal condition.

Wire the same diagnostics helper into both paths so callers get the same
actionable trace regardless of combo strategy.
2026-08-26 13:09:38 -03:00
5 changed files with 206 additions and 3 deletions

View File

@@ -0,0 +1 @@
- **fix(combo):** attach the same combo-diagnostics trace (`poolSize`/`attemptOrder`/`excluded`/`terminalReason`, plus `x-omniroute-combo-*` headers) to the round-robin strategy's and the nested pipeline/fusion runtime-unit loop's "Maximum combo retry limit reached" 503 that the priority-strategy path already attaches for the identical terminal condition — previously those two paths returned a bare, contextless 503 ([#11462](https://github.com/diegosouzapw/OmniRoute/issues/11462)).

View File

@@ -3286,7 +3286,24 @@ async function handleRoundRobinCombo({
"COMBO-RR",
`Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.`
);
return errorResponse(503, "Maximum combo retry limit reached");
return errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
{
poolSize: modelCount,
attempted: globalAttempts,
excluded: [
...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
],
attemptOrder: rrOutcomes.map((o) => ({
provider: o.model.split("/")[0] || "unknown",
model: o.model,
})),
terminalReason: "max_attempts_exceeded",
recovery: buildRecoveryHint("max_attempts_exceeded"),
}
);
}
if (retry > 0) {
log.info(

View File

@@ -5,7 +5,8 @@
* @changes
* - [2026-07-24] [Composer] - Skip execute-mode units at concurrency cap before dispatch
*/
import { errorResponse } from "../../utils/error.ts";
import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts";
import type { ComboDiagnostics } from "../../utils/error.ts";
import { recordComboRequest } from "../comboMetrics.ts";
import { resolveDelayMs } from "./comboPredicates.ts";
import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts";
@@ -216,6 +217,17 @@ export async function executeRuntimeUnitCombo(args: {
};
const finalFailure = (response: Response): Response =>
withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null);
// #11462: attempts already made this loop, tracked for the attempt-budget-exceeded
// diagnostics trace below (mirrors the poolSize/attemptOrder shape combo.ts already
// attaches for the priority/round-robin strategies).
const attemptedUnits: Array<{ provider: string; model: string }> = [];
const buildAttemptBudgetDiag = (): ComboDiagnostics => ({
poolSize: orderedUnits.length,
attempted: args.nesting.attemptBudget.count,
excluded: [],
attemptOrder: attemptedUnits,
terminalReason: "max_attempts_exceeded",
});
for (const unit of orderedUnits) {
const protectedPriorityUnit =
@@ -247,13 +259,21 @@ export async function executeRuntimeUnitCombo(args: {
}
args.nesting.attemptBudget.count += 1;
if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
lastResponse = errorResponse(503, "Maximum combo retry limit reached");
lastResponse = errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
buildAttemptBudgetDiag()
);
await observeFailure(lastResponse, unit);
return { response: finalFailure(lastResponse), unit };
}
if (retry > 0) {
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
}
attemptedUnits.push({
provider: unit.kind === "model" ? unit.provider : "combo-ref",
model: unitDisplayName(unit),
});
args.log.info(
"COMBO",
`Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}`

View File

@@ -0,0 +1,77 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-rr-diag-11462-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const core = await import("../../src/lib/db/core.ts");
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
const { resetAll: resetAllSemaphores } = await import(
"../../open-sse/services/rateLimitSemaphore.ts"
);
function createLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse() {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test.beforeEach(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
});
test.after(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
});
test(
"#11462: round-robin combo's 'Maximum combo retry limit reached' 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/excluded/terminalReason)",
async () => {
const result = await handleComboChat({
body: { messages: [{ role: "user", content: "hi" }] },
combo: {
name: "rr-probe-11462",
strategy: "round-robin",
models: ["openai/rr-a", "anthropic/rr-b"],
config: {
maxRetries: 0,
maxGlobalAttempts: 1,
concurrencyPerModel: 1,
queueTimeoutMs: 1000,
},
},
handleSingleModel: async () => failResponse(),
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null,
allCombos: null,
});
assert.equal(result.status, 503);
const body = await result.json();
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "round-robin 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics.attemptOrder));
assert.ok(typeof body.diagnostics.terminalReason === "string");
}
);

View File

@@ -0,0 +1,88 @@
/**
* #11462: the nested runtime-unit loop (open-sse/services/combo/runtimeUnits.ts,
* used by the pipeline/fusion combo strategies via dispatchPrelude.ts and
* fusionPanel.ts) had the same bare-`errorResponse()` gap as the round-robin
* strategy's "Maximum combo retry limit reached" 503 — no diagnostics trace.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { executeRuntimeUnitCombo } from "../../open-sse/services/combo/runtimeUnits.ts";
import type { ResolvedComboUnit, ComboNestingContext } from "../../open-sse/services/combo/types.ts";
function noopLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse(): Response {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test(
"#11462: nested runtime-unit combo's attempt-budget-exceeded 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/terminalReason)",
async () => {
const units: ResolvedComboUnit[] = [
{
kind: "model",
stepId: "step-a",
executionKey: "a",
modelStr: "openai/ru-a",
provider: "openai",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
{
kind: "model",
stepId: "step-b",
executionKey: "b",
modelStr: "anthropic/ru-b",
provider: "anthropic",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
];
const nesting: ComboNestingContext = {
depth: 0,
maxDepth: 5,
visitedComboNames: [],
rootComboName: "ru-probe-11462",
// Budget of 1 trips on the very first attempt, deterministically hitting the
// terminal branch under test without needing every unit to actually fail.
attemptBudget: { count: 0, limit: 1 },
};
const result = await executeRuntimeUnitCombo({
body: { messages: [{ role: "user", content: "hi" }] },
combo: { name: "ru-probe-11462", strategy: "pipeline" },
strategy: "pipeline",
units,
handleSingleModel: async () => failResponse(),
log: noopLog() as never,
config: { maxRetries: 0 },
allCombos: [],
nesting,
baseOptions: {} as never,
runCombo: async () => failResponse(),
});
assert.equal(result.response.status, 503);
const body = (await result.response.json()) as {
error: { message: string };
diagnostics?: { poolSize: number; attemptOrder: unknown[]; terminalReason: string };
};
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "runtime-unit 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics?.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics?.attemptOrder));
assert.equal(body.diagnostics?.terminalReason, "max_attempts_exceeded");
}
);