fix(combo): attach combo diagnostics to round-robin and runtime-unit retry-limit 503s (#11462)

The round-robin combo strategy's terminal "Maximum combo retry limit
reached" 503 (open-sse/services/combo.ts, handleRoundRobinCombo) and the
nested runtime-unit loop used by pipeline/fusion combo steps
(open-sse/services/combo/runtimeUnits.ts) returned a bare errorResponse()
with zero diagnostics, while the priority-strategy path in the same file
already attaches a full errorResponseWithComboDiagnostics trace
(poolSize, attemptOrder, excluded providers/reasons, terminalReason,
recovery hint) for the identical terminal condition.

Wire the same diagnostics helper into both paths so callers get the same
actionable trace regardless of combo strategy.
This commit is contained in:
Markus Hartung
2026-08-26 13:09:38 -03:00
parent 91aeca0440
commit 17eac4978a
5 changed files with 206 additions and 3 deletions

View File

@@ -0,0 +1 @@
- **fix(combo):** attach the same combo-diagnostics trace (`poolSize`/`attemptOrder`/`excluded`/`terminalReason`, plus `x-omniroute-combo-*` headers) to the round-robin strategy's and the nested pipeline/fusion runtime-unit loop's "Maximum combo retry limit reached" 503 that the priority-strategy path already attaches for the identical terminal condition — previously those two paths returned a bare, contextless 503 ([#11462](https://github.com/diegosouzapw/OmniRoute/issues/11462)).

View File

@@ -3286,7 +3286,24 @@ async function handleRoundRobinCombo({
"COMBO-RR",
`Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.`
);
return errorResponse(503, "Maximum combo retry limit reached");
return errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
{
poolSize: modelCount,
attempted: globalAttempts,
excluded: [
...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
],
attemptOrder: rrOutcomes.map((o) => ({
provider: o.model.split("/")[0] || "unknown",
model: o.model,
})),
terminalReason: "max_attempts_exceeded",
recovery: buildRecoveryHint("max_attempts_exceeded"),
}
);
}
if (retry > 0) {
log.info(

View File

@@ -5,7 +5,8 @@
* @changes
* - [2026-07-24] [Composer] - Skip execute-mode units at concurrency cap before dispatch
*/
import { errorResponse } from "../../utils/error.ts";
import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts";
import type { ComboDiagnostics } from "../../utils/error.ts";
import { recordComboRequest } from "../comboMetrics.ts";
import { resolveDelayMs } from "./comboPredicates.ts";
import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts";
@@ -216,6 +217,17 @@ export async function executeRuntimeUnitCombo(args: {
};
const finalFailure = (response: Response): Response =>
withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null);
// #11462: attempts already made this loop, tracked for the attempt-budget-exceeded
// diagnostics trace below (mirrors the poolSize/attemptOrder shape combo.ts already
// attaches for the priority/round-robin strategies).
const attemptedUnits: Array<{ provider: string; model: string }> = [];
const buildAttemptBudgetDiag = (): ComboDiagnostics => ({
poolSize: orderedUnits.length,
attempted: args.nesting.attemptBudget.count,
excluded: [],
attemptOrder: attemptedUnits,
terminalReason: "max_attempts_exceeded",
});
for (const unit of orderedUnits) {
const protectedPriorityUnit =
@@ -247,13 +259,21 @@ export async function executeRuntimeUnitCombo(args: {
}
args.nesting.attemptBudget.count += 1;
if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
lastResponse = errorResponse(503, "Maximum combo retry limit reached");
lastResponse = errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
buildAttemptBudgetDiag()
);
await observeFailure(lastResponse, unit);
return { response: finalFailure(lastResponse), unit };
}
if (retry > 0) {
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
}
attemptedUnits.push({
provider: unit.kind === "model" ? unit.provider : "combo-ref",
model: unitDisplayName(unit),
});
args.log.info(
"COMBO",
`Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}`

View File

@@ -0,0 +1,77 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-rr-diag-11462-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const core = await import("../../src/lib/db/core.ts");
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
const { resetAll: resetAllSemaphores } = await import(
"../../open-sse/services/rateLimitSemaphore.ts"
);
function createLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse() {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test.beforeEach(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
});
test.after(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
});
test(
"#11462: round-robin combo's 'Maximum combo retry limit reached' 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/excluded/terminalReason)",
async () => {
const result = await handleComboChat({
body: { messages: [{ role: "user", content: "hi" }] },
combo: {
name: "rr-probe-11462",
strategy: "round-robin",
models: ["openai/rr-a", "anthropic/rr-b"],
config: {
maxRetries: 0,
maxGlobalAttempts: 1,
concurrencyPerModel: 1,
queueTimeoutMs: 1000,
},
},
handleSingleModel: async () => failResponse(),
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null,
allCombos: null,
});
assert.equal(result.status, 503);
const body = await result.json();
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "round-robin 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics.attemptOrder));
assert.ok(typeof body.diagnostics.terminalReason === "string");
}
);

View File

@@ -0,0 +1,88 @@
/**
* #11462: the nested runtime-unit loop (open-sse/services/combo/runtimeUnits.ts,
* used by the pipeline/fusion combo strategies via dispatchPrelude.ts and
* fusionPanel.ts) had the same bare-`errorResponse()` gap as the round-robin
* strategy's "Maximum combo retry limit reached" 503 — no diagnostics trace.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { executeRuntimeUnitCombo } from "../../open-sse/services/combo/runtimeUnits.ts";
import type { ResolvedComboUnit, ComboNestingContext } from "../../open-sse/services/combo/types.ts";
function noopLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse(): Response {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test(
"#11462: nested runtime-unit combo's attempt-budget-exceeded 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/terminalReason)",
async () => {
const units: ResolvedComboUnit[] = [
{
kind: "model",
stepId: "step-a",
executionKey: "a",
modelStr: "openai/ru-a",
provider: "openai",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
{
kind: "model",
stepId: "step-b",
executionKey: "b",
modelStr: "anthropic/ru-b",
provider: "anthropic",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
];
const nesting: ComboNestingContext = {
depth: 0,
maxDepth: 5,
visitedComboNames: [],
rootComboName: "ru-probe-11462",
// Budget of 1 trips on the very first attempt, deterministically hitting the
// terminal branch under test without needing every unit to actually fail.
attemptBudget: { count: 0, limit: 1 },
};
const result = await executeRuntimeUnitCombo({
body: { messages: [{ role: "user", content: "hi" }] },
combo: { name: "ru-probe-11462", strategy: "pipeline" },
strategy: "pipeline",
units,
handleSingleModel: async () => failResponse(),
log: noopLog() as never,
config: { maxRetries: 0 },
allCombos: [],
nesting,
baseOptions: {} as never,
runCombo: async () => failResponse(),
});
assert.equal(result.response.status, 503);
const body = (await result.response.json()) as {
error: { message: string };
diagnostics?: { poolSize: number; attemptOrder: unknown[]; terminalReason: string };
};
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "runtime-unit 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics?.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics?.attemptOrder));
assert.equal(body.diagnostics?.terminalReason, "max_attempts_exceeded");
}
);