mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-27 01:22:10 +03:00
Compare commits
1 Commits
security/v
...
fix/11462-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17eac4978a |
1
changelog.d/fixes/11462-roundrobin-combo-diagnostics.md
Normal file
1
changelog.d/fixes/11462-roundrobin-combo-diagnostics.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(combo):** attach the same combo-diagnostics trace (`poolSize`/`attemptOrder`/`excluded`/`terminalReason`, plus `x-omniroute-combo-*` headers) to the round-robin strategy's and the nested pipeline/fusion runtime-unit loop's "Maximum combo retry limit reached" 503 that the priority-strategy path already attaches for the identical terminal condition — previously those two paths returned a bare, contextless 503 ([#11462](https://github.com/diegosouzapw/OmniRoute/issues/11462)).
|
||||
@@ -3286,7 +3286,24 @@ async function handleRoundRobinCombo({
|
||||
"COMBO-RR",
|
||||
`Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.`
|
||||
);
|
||||
return errorResponse(503, "Maximum combo retry limit reached");
|
||||
return errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Maximum combo retry limit reached",
|
||||
{
|
||||
poolSize: modelCount,
|
||||
attempted: globalAttempts,
|
||||
excluded: [
|
||||
...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
|
||||
...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
|
||||
],
|
||||
attemptOrder: rrOutcomes.map((o) => ({
|
||||
provider: o.model.split("/")[0] || "unknown",
|
||||
model: o.model,
|
||||
})),
|
||||
terminalReason: "max_attempts_exceeded",
|
||||
recovery: buildRecoveryHint("max_attempts_exceeded"),
|
||||
}
|
||||
);
|
||||
}
|
||||
if (retry > 0) {
|
||||
log.info(
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
* @changes
|
||||
* - [2026-07-24] [Composer] - Skip execute-mode units at concurrency cap before dispatch
|
||||
*/
|
||||
import { errorResponse } from "../../utils/error.ts";
|
||||
import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts";
|
||||
import type { ComboDiagnostics } from "../../utils/error.ts";
|
||||
import { recordComboRequest } from "../comboMetrics.ts";
|
||||
import { resolveDelayMs } from "./comboPredicates.ts";
|
||||
import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts";
|
||||
@@ -216,6 +217,17 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
};
|
||||
const finalFailure = (response: Response): Response =>
|
||||
withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null);
|
||||
// #11462: attempts already made this loop, tracked for the attempt-budget-exceeded
|
||||
// diagnostics trace below (mirrors the poolSize/attemptOrder shape combo.ts already
|
||||
// attaches for the priority/round-robin strategies).
|
||||
const attemptedUnits: Array<{ provider: string; model: string }> = [];
|
||||
const buildAttemptBudgetDiag = (): ComboDiagnostics => ({
|
||||
poolSize: orderedUnits.length,
|
||||
attempted: args.nesting.attemptBudget.count,
|
||||
excluded: [],
|
||||
attemptOrder: attemptedUnits,
|
||||
terminalReason: "max_attempts_exceeded",
|
||||
});
|
||||
|
||||
for (const unit of orderedUnits) {
|
||||
const protectedPriorityUnit =
|
||||
@@ -247,13 +259,21 @@ export async function executeRuntimeUnitCombo(args: {
|
||||
}
|
||||
args.nesting.attemptBudget.count += 1;
|
||||
if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
|
||||
lastResponse = errorResponse(503, "Maximum combo retry limit reached");
|
||||
lastResponse = errorResponseWithComboDiagnostics(
|
||||
503,
|
||||
"Maximum combo retry limit reached",
|
||||
buildAttemptBudgetDiag()
|
||||
);
|
||||
await observeFailure(lastResponse, unit);
|
||||
return { response: finalFailure(lastResponse), unit };
|
||||
}
|
||||
if (retry > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
|
||||
}
|
||||
attemptedUnits.push({
|
||||
provider: unit.kind === "model" ? unit.provider : "combo-ref",
|
||||
model: unitDisplayName(unit),
|
||||
});
|
||||
args.log.info(
|
||||
"COMBO",
|
||||
`Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}`
|
||||
|
||||
77
tests/unit/combo-rr-diagnostics-11462.test.ts
Normal file
77
tests/unit/combo-rr-diagnostics-11462.test.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-rr-diag-11462-"));
|
||||
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
|
||||
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
|
||||
const { resetAll: resetAllSemaphores } = await import(
|
||||
"../../open-sse/services/rateLimitSemaphore.ts"
|
||||
);
|
||||
|
||||
function createLog() {
|
||||
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
|
||||
}
|
||||
function failResponse() {
|
||||
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
|
||||
status: 500,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
test.beforeEach(() => {
|
||||
resetAllComboMetrics();
|
||||
resetAllCircuitBreakers();
|
||||
resetAllSemaphores();
|
||||
});
|
||||
test.after(() => {
|
||||
resetAllComboMetrics();
|
||||
resetAllCircuitBreakers();
|
||||
resetAllSemaphores();
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
|
||||
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
|
||||
});
|
||||
|
||||
test(
|
||||
"#11462: round-robin combo's 'Maximum combo retry limit reached' 503 must carry " +
|
||||
"the combo diagnostics trace (poolSize/attemptOrder/excluded/terminalReason)",
|
||||
async () => {
|
||||
const result = await handleComboChat({
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
combo: {
|
||||
name: "rr-probe-11462",
|
||||
strategy: "round-robin",
|
||||
models: ["openai/rr-a", "anthropic/rr-b"],
|
||||
config: {
|
||||
maxRetries: 0,
|
||||
maxGlobalAttempts: 1,
|
||||
concurrencyPerModel: 1,
|
||||
queueTimeoutMs: 1000,
|
||||
},
|
||||
},
|
||||
handleSingleModel: async () => failResponse(),
|
||||
isModelAvailable: async () => true,
|
||||
log: createLog(),
|
||||
settings: null,
|
||||
relayOptions: null,
|
||||
allCombos: null,
|
||||
});
|
||||
|
||||
assert.equal(result.status, 503);
|
||||
const body = await result.json();
|
||||
assert.equal(body.error.message, "Maximum combo retry limit reached");
|
||||
assert.ok(body.diagnostics, "round-robin 503 should carry a diagnostics field");
|
||||
assert.ok(typeof body.diagnostics.poolSize === "number");
|
||||
assert.ok(Array.isArray(body.diagnostics.attemptOrder));
|
||||
assert.ok(typeof body.diagnostics.terminalReason === "string");
|
||||
}
|
||||
);
|
||||
88
tests/unit/combo-runtimeunits-diagnostics-11462.test.ts
Normal file
88
tests/unit/combo-runtimeunits-diagnostics-11462.test.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* #11462: the nested runtime-unit loop (open-sse/services/combo/runtimeUnits.ts,
|
||||
* used by the pipeline/fusion combo strategies via dispatchPrelude.ts and
|
||||
* fusionPanel.ts) had the same bare-`errorResponse()` gap as the round-robin
|
||||
* strategy's "Maximum combo retry limit reached" 503 — no diagnostics trace.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { executeRuntimeUnitCombo } from "../../open-sse/services/combo/runtimeUnits.ts";
|
||||
import type { ResolvedComboUnit, ComboNestingContext } from "../../open-sse/services/combo/types.ts";
|
||||
|
||||
function noopLog() {
|
||||
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
|
||||
}
|
||||
|
||||
function failResponse(): Response {
|
||||
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
|
||||
status: 500,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
test(
|
||||
"#11462: nested runtime-unit combo's attempt-budget-exceeded 503 must carry " +
|
||||
"the combo diagnostics trace (poolSize/attemptOrder/terminalReason)",
|
||||
async () => {
|
||||
const units: ResolvedComboUnit[] = [
|
||||
{
|
||||
kind: "model",
|
||||
stepId: "step-a",
|
||||
executionKey: "a",
|
||||
modelStr: "openai/ru-a",
|
||||
provider: "openai",
|
||||
providerId: null,
|
||||
connectionId: null,
|
||||
weight: 1,
|
||||
label: null,
|
||||
},
|
||||
{
|
||||
kind: "model",
|
||||
stepId: "step-b",
|
||||
executionKey: "b",
|
||||
modelStr: "anthropic/ru-b",
|
||||
provider: "anthropic",
|
||||
providerId: null,
|
||||
connectionId: null,
|
||||
weight: 1,
|
||||
label: null,
|
||||
},
|
||||
];
|
||||
|
||||
const nesting: ComboNestingContext = {
|
||||
depth: 0,
|
||||
maxDepth: 5,
|
||||
visitedComboNames: [],
|
||||
rootComboName: "ru-probe-11462",
|
||||
// Budget of 1 trips on the very first attempt, deterministically hitting the
|
||||
// terminal branch under test without needing every unit to actually fail.
|
||||
attemptBudget: { count: 0, limit: 1 },
|
||||
};
|
||||
|
||||
const result = await executeRuntimeUnitCombo({
|
||||
body: { messages: [{ role: "user", content: "hi" }] },
|
||||
combo: { name: "ru-probe-11462", strategy: "pipeline" },
|
||||
strategy: "pipeline",
|
||||
units,
|
||||
handleSingleModel: async () => failResponse(),
|
||||
log: noopLog() as never,
|
||||
config: { maxRetries: 0 },
|
||||
allCombos: [],
|
||||
nesting,
|
||||
baseOptions: {} as never,
|
||||
runCombo: async () => failResponse(),
|
||||
});
|
||||
|
||||
assert.equal(result.response.status, 503);
|
||||
const body = (await result.response.json()) as {
|
||||
error: { message: string };
|
||||
diagnostics?: { poolSize: number; attemptOrder: unknown[]; terminalReason: string };
|
||||
};
|
||||
assert.equal(body.error.message, "Maximum combo retry limit reached");
|
||||
assert.ok(body.diagnostics, "runtime-unit 503 should carry a diagnostics field");
|
||||
assert.ok(typeof body.diagnostics?.poolSize === "number");
|
||||
assert.ok(Array.isArray(body.diagnostics?.attemptOrder));
|
||||
assert.equal(body.diagnostics?.terminalReason, "max_attempts_exceeded");
|
||||
}
|
||||
);
|
||||
Reference in New Issue
Block a user