diff --git a/changelog.d/fixes/11462-roundrobin-combo-diagnostics.md b/changelog.d/fixes/11462-roundrobin-combo-diagnostics.md new file mode 100644 index 0000000000..1d1eec514a --- /dev/null +++ b/changelog.d/fixes/11462-roundrobin-combo-diagnostics.md @@ -0,0 +1 @@ +- **fix(combo):** attach the same combo-diagnostics trace (`poolSize`/`attemptOrder`/`excluded`/`terminalReason`, plus `x-omniroute-combo-*` headers) to the round-robin strategy's and the nested pipeline/fusion runtime-unit loop's "Maximum combo retry limit reached" 503 that the priority-strategy path already attaches for the identical terminal condition — previously those two paths returned a bare, contextless 503 ([#11462](https://github.com/diegosouzapw/OmniRoute/issues/11462)). diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 0ee416e2f7..749b30376a 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -3286,7 +3286,24 @@ async function handleRoundRobinCombo({ "COMBO-RR", `Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.` ); - return errorResponse(503, "Maximum combo retry limit reached"); + return errorResponseWithComboDiagnostics( + 503, + "Maximum combo retry limit reached", + { + poolSize: modelCount, + attempted: globalAttempts, + excluded: [ + ...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })), + ...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))), + ], + attemptOrder: rrOutcomes.map((o) => ({ + provider: o.model.split("/")[0] || "unknown", + model: o.model, + })), + terminalReason: "max_attempts_exceeded", + recovery: buildRecoveryHint("max_attempts_exceeded"), + } + ); } if (retry > 0) { log.info( diff --git a/open-sse/services/combo/runtimeUnits.ts b/open-sse/services/combo/runtimeUnits.ts index 453ce38116..f6321a68d2 100644 --- a/open-sse/services/combo/runtimeUnits.ts +++ b/open-sse/services/combo/runtimeUnits.ts @@ -5,7 +5,8 @@ * @changes * - [2026-07-24] [Composer] - Skip execute-mode units at concurrency cap before dispatch */ -import { errorResponse } from "../../utils/error.ts"; +import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts"; +import type { ComboDiagnostics } from "../../utils/error.ts"; import { recordComboRequest } from "../comboMetrics.ts"; import { resolveDelayMs } from "./comboPredicates.ts"; import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts"; @@ -216,6 +217,17 @@ export async function executeRuntimeUnitCombo(args: { }; const finalFailure = (response: Response): Response => withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null); + // #11462: attempts already made this loop, tracked for the attempt-budget-exceeded + // diagnostics trace below (mirrors the poolSize/attemptOrder shape combo.ts already + // attaches for the priority/round-robin strategies). + const attemptedUnits: Array<{ provider: string; model: string }> = []; + const buildAttemptBudgetDiag = (): ComboDiagnostics => ({ + poolSize: orderedUnits.length, + attempted: args.nesting.attemptBudget.count, + excluded: [], + attemptOrder: attemptedUnits, + terminalReason: "max_attempts_exceeded", + }); for (const unit of orderedUnits) { const protectedPriorityUnit = @@ -247,13 +259,21 @@ export async function executeRuntimeUnitCombo(args: { } args.nesting.attemptBudget.count += 1; if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) { - lastResponse = errorResponse(503, "Maximum combo retry limit reached"); + lastResponse = errorResponseWithComboDiagnostics( + 503, + "Maximum combo retry limit reached", + buildAttemptBudgetDiag() + ); await observeFailure(lastResponse, unit); return { response: finalFailure(lastResponse), unit }; } if (retry > 0) { await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); } + attemptedUnits.push({ + provider: unit.kind === "model" ? unit.provider : "combo-ref", + model: unitDisplayName(unit), + }); args.log.info( "COMBO", `Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}` diff --git a/tests/unit/combo-rr-diagnostics-11462.test.ts b/tests/unit/combo-rr-diagnostics-11462.test.ts new file mode 100644 index 0000000000..ade088e3ce --- /dev/null +++ b/tests/unit/combo-rr-diagnostics-11462.test.ts @@ -0,0 +1,77 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-rr-diag-11462-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const core = await import("../../src/lib/db/core.ts"); +const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts"); +const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); +const { resetAll: resetAllSemaphores } = await import( + "../../open-sse/services/rateLimitSemaphore.ts" +); + +function createLog() { + return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; +} +function failResponse() { + return new Response(JSON.stringify({ error: { message: "upstream 500" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }); +} + +test.beforeEach(() => { + resetAllComboMetrics(); + resetAllCircuitBreakers(); + resetAllSemaphores(); +}); +test.after(() => { + resetAllComboMetrics(); + resetAllCircuitBreakers(); + resetAllSemaphores(); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +test( + "#11462: round-robin combo's 'Maximum combo retry limit reached' 503 must carry " + + "the combo diagnostics trace (poolSize/attemptOrder/excluded/terminalReason)", + async () => { + const result = await handleComboChat({ + body: { messages: [{ role: "user", content: "hi" }] }, + combo: { + name: "rr-probe-11462", + strategy: "round-robin", + models: ["openai/rr-a", "anthropic/rr-b"], + config: { + maxRetries: 0, + maxGlobalAttempts: 1, + concurrencyPerModel: 1, + queueTimeoutMs: 1000, + }, + }, + handleSingleModel: async () => failResponse(), + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null, + allCombos: null, + }); + + assert.equal(result.status, 503); + const body = await result.json(); + assert.equal(body.error.message, "Maximum combo retry limit reached"); + assert.ok(body.diagnostics, "round-robin 503 should carry a diagnostics field"); + assert.ok(typeof body.diagnostics.poolSize === "number"); + assert.ok(Array.isArray(body.diagnostics.attemptOrder)); + assert.ok(typeof body.diagnostics.terminalReason === "string"); + } +); diff --git a/tests/unit/combo-runtimeunits-diagnostics-11462.test.ts b/tests/unit/combo-runtimeunits-diagnostics-11462.test.ts new file mode 100644 index 0000000000..6612a25ca8 --- /dev/null +++ b/tests/unit/combo-runtimeunits-diagnostics-11462.test.ts @@ -0,0 +1,88 @@ +/** + * #11462: the nested runtime-unit loop (open-sse/services/combo/runtimeUnits.ts, + * used by the pipeline/fusion combo strategies via dispatchPrelude.ts and + * fusionPanel.ts) had the same bare-`errorResponse()` gap as the round-robin + * strategy's "Maximum combo retry limit reached" 503 — no diagnostics trace. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { executeRuntimeUnitCombo } from "../../open-sse/services/combo/runtimeUnits.ts"; +import type { ResolvedComboUnit, ComboNestingContext } from "../../open-sse/services/combo/types.ts"; + +function noopLog() { + return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }; +} + +function failResponse(): Response { + return new Response(JSON.stringify({ error: { message: "upstream 500" } }), { + status: 500, + headers: { "content-type": "application/json" }, + }); +} + +test( + "#11462: nested runtime-unit combo's attempt-budget-exceeded 503 must carry " + + "the combo diagnostics trace (poolSize/attemptOrder/terminalReason)", + async () => { + const units: ResolvedComboUnit[] = [ + { + kind: "model", + stepId: "step-a", + executionKey: "a", + modelStr: "openai/ru-a", + provider: "openai", + providerId: null, + connectionId: null, + weight: 1, + label: null, + }, + { + kind: "model", + stepId: "step-b", + executionKey: "b", + modelStr: "anthropic/ru-b", + provider: "anthropic", + providerId: null, + connectionId: null, + weight: 1, + label: null, + }, + ]; + + const nesting: ComboNestingContext = { + depth: 0, + maxDepth: 5, + visitedComboNames: [], + rootComboName: "ru-probe-11462", + // Budget of 1 trips on the very first attempt, deterministically hitting the + // terminal branch under test without needing every unit to actually fail. + attemptBudget: { count: 0, limit: 1 }, + }; + + const result = await executeRuntimeUnitCombo({ + body: { messages: [{ role: "user", content: "hi" }] }, + combo: { name: "ru-probe-11462", strategy: "pipeline" }, + strategy: "pipeline", + units, + handleSingleModel: async () => failResponse(), + log: noopLog() as never, + config: { maxRetries: 0 }, + allCombos: [], + nesting, + baseOptions: {} as never, + runCombo: async () => failResponse(), + }); + + assert.equal(result.response.status, 503); + const body = (await result.response.json()) as { + error: { message: string }; + diagnostics?: { poolSize: number; attemptOrder: unknown[]; terminalReason: string }; + }; + assert.equal(body.error.message, "Maximum combo retry limit reached"); + assert.ok(body.diagnostics, "runtime-unit 503 should carry a diagnostics field"); + assert.ok(typeof body.diagnostics?.poolSize === "number"); + assert.ok(Array.isArray(body.diagnostics?.attemptOrder)); + assert.equal(body.diagnostics?.terminalReason, "max_attempts_exceeded"); + } +);