Compare commits

..

1 Commits

Author SHA1 Message Date
Markus Hartung
17eac4978a fix(combo): attach combo diagnostics to round-robin and runtime-unit retry-limit 503s (#11462)
The round-robin combo strategy's terminal "Maximum combo retry limit
reached" 503 (open-sse/services/combo.ts, handleRoundRobinCombo) and the
nested runtime-unit loop used by pipeline/fusion combo steps
(open-sse/services/combo/runtimeUnits.ts) returned a bare errorResponse()
with zero diagnostics, while the priority-strategy path in the same file
already attaches a full errorResponseWithComboDiagnostics trace
(poolSize, attemptOrder, excluded providers/reasons, terminalReason,
recovery hint) for the identical terminal condition.

Wire the same diagnostics helper into both paths so callers get the same
actionable trace regardless of combo strategy.
2026-08-26 13:09:38 -03:00
10 changed files with 209 additions and 127 deletions

View File

@@ -1 +0,0 @@
- **fix(combos):** the combo builder's precision-select, global-model-search, and manual-entry flows now serialize a model step's `model` string using the provider's already-computed routing-alias prefix (e.g. `oc/`) instead of rebuilding it from the raw canonical `providerId`, fixing the no-auth "OpenCode Free" provider (`opencode`) being routed to the unrelated paid "OpenCode Zen" provider (`opencode-zen`) because `opencode` doubles as a manual routing-prefix override ([#11433](https://github.com/diegosouzapw/OmniRoute/issues/11433)).

View File

@@ -0,0 +1 @@
- **fix(combo):** attach the same combo-diagnostics trace (`poolSize`/`attemptOrder`/`excluded`/`terminalReason`, plus `x-omniroute-combo-*` headers) to the round-robin strategy's and the nested pipeline/fusion runtime-unit loop's "Maximum combo retry limit reached" 503 that the priority-strategy path already attaches for the identical terminal condition — previously those two paths returned a bare, contextless 503 ([#11462](https://github.com/diegosouzapw/OmniRoute/issues/11462)).

View File

@@ -3286,7 +3286,24 @@ async function handleRoundRobinCombo({
"COMBO-RR",
`Maximum combo attempts (${maxGlobalAttempts}) exceeded. Terminating loop to prevent runaway requests.`
);
return errorResponse(503, "Maximum combo retry limit reached");
return errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
{
poolSize: modelCount,
attempted: globalAttempts,
excluded: [
...[...exhaustedProviders].map((p) => ({ provider: p, reason: "exhausted" })),
...[...exhaustedConnections].map((c) => formatExhaustedConnectionKey(String(c))),
],
attemptOrder: rrOutcomes.map((o) => ({
provider: o.model.split("/")[0] || "unknown",
model: o.model,
})),
terminalReason: "max_attempts_exceeded",
recovery: buildRecoveryHint("max_attempts_exceeded"),
}
);
}
if (retry > 0) {
log.info(

View File

@@ -5,7 +5,8 @@
* @changes
* - [2026-07-24] [Composer] - Skip execute-mode units at concurrency cap before dispatch
*/
import { errorResponse } from "../../utils/error.ts";
import { errorResponse, errorResponseWithComboDiagnostics } from "../../utils/error.ts";
import type { ComboDiagnostics } from "../../utils/error.ts";
import { recordComboRequest } from "../comboMetrics.ts";
import { resolveDelayMs } from "./comboPredicates.ts";
import { isRuntimeUnitAtConcurrencyCap } from "./runtimeUnitCapacity.ts";
@@ -216,6 +217,17 @@ export async function executeRuntimeUnitCombo(args: {
};
const finalFailure = (response: Response): Response =>
withQuotaExhaustionClassification(response, observedFailure ? allObservedFailuresQuota : null);
// #11462: attempts already made this loop, tracked for the attempt-budget-exceeded
// diagnostics trace below (mirrors the poolSize/attemptOrder shape combo.ts already
// attaches for the priority/round-robin strategies).
const attemptedUnits: Array<{ provider: string; model: string }> = [];
const buildAttemptBudgetDiag = (): ComboDiagnostics => ({
poolSize: orderedUnits.length,
attempted: args.nesting.attemptBudget.count,
excluded: [],
attemptOrder: attemptedUnits,
terminalReason: "max_attempts_exceeded",
});
for (const unit of orderedUnits) {
const protectedPriorityUnit =
@@ -247,13 +259,21 @@ export async function executeRuntimeUnitCombo(args: {
}
args.nesting.attemptBudget.count += 1;
if (args.nesting.attemptBudget.count > args.nesting.attemptBudget.limit) {
lastResponse = errorResponse(503, "Maximum combo retry limit reached");
lastResponse = errorResponseWithComboDiagnostics(
503,
"Maximum combo retry limit reached",
buildAttemptBudgetDiag()
);
await observeFailure(lastResponse, unit);
return { response: finalFailure(lastResponse), unit };
}
if (retry > 0) {
await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
}
attemptedUnits.push({
provider: unit.kind === "model" ? unit.provider : "combo-ref",
model: unitDisplayName(unit),
});
args.log.info(
"COMBO",
`Trying ${unit.kind} ${unitDisplayName(unit)}${retry > 0 ? ` (retry ${retry})` : ""}`

View File

@@ -2177,9 +2177,6 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
builderConnectionId !== COMBO_BUILDER_AUTO_CONNECTION ? builderConnectionId : null,
connectionLabel: selectedBuilderConnection?.label || null,
allowedConnectionIds: builderEffectiveAllowedConnectionIds,
// #11433: use the already-corrected routing prefix (e.g. "oc" for
// OpenCode Free) instead of letting it default to the raw providerId.
modelPrefix: parseQualifiedModel(selectedBuilderModel.qualifiedModel)?.providerId,
})
: null;
const builderHasDuplicate =
@@ -2504,9 +2501,6 @@ function ComboFormModal({ isOpen, combo, onClose, onSave, activeProviders, combo
builderConnectionId !== COMBO_BUILDER_AUTO_CONNECTION ? builderConnectionId : null,
connectionLabel: selectedBuilderConnection?.label || null,
allowedConnectionIds: builderEffectiveAllowedConnectionIds,
// #11433: use the already-corrected routing prefix (e.g. "oc" for
// OpenCode Free) instead of letting it default to the raw providerId.
modelPrefix: parseQualifiedModel(selectedBuilderModel.qualifiedModel)?.providerId,
});
if (hasExactModelStepDuplicate(models, nextStep)) {

View File

@@ -83,7 +83,6 @@ export function buildPrecisionComboModelStep({
connectionLabel,
allowedConnectionIds = null,
weight = 0,
modelPrefix,
}: {
providerId: string;
modelId: string;
@@ -92,22 +91,9 @@ export function buildPrecisionComboModelStep({
/** #3266: account allowlist scoping round-robin to a subset of connections. */
allowedConnectionIds?: string[] | null;
weight?: number;
/**
* #11433: the routing-prefix segment to serialize into `model` (e.g. "oc"
* for the no-auth OpenCode Free provider), when it differs from the
* canonical `providerId`. Some canonical provider ids collide with an
* unrelated manual `ALIAS_TO_PROVIDER_ID` routing override (`opencode` →
* `opencode-zen`), so reconstructing `model` from the raw `providerId`
* alone can round-trip to the wrong provider on request routing. Falls
* back to `providerId` when omitted/blank. `step.providerId` always stays
* the canonical id regardless, so routing/duplicate-detection identity is
* unaffected.
*/
modelPrefix?: string | null;
}): ComboModelStep {
const normalizedProviderId = toTrimmedString(providerId) || "provider";
const normalizedModelId = toTrimmedString(modelId) || "model";
const normalizedModelPrefix = toTrimmedString(modelPrefix) || normalizedProviderId;
const normalizedConnectionId = toTrimmedString(connectionId);
const normalizedConnectionLabel = toTrimmedString(connectionLabel);
// A pinned single connection wins over an allowlist, so only carry the allowlist
@@ -124,7 +110,7 @@ export function buildPrecisionComboModelStep({
return {
kind: "model",
providerId: normalizedProviderId,
model: `${normalizedModelPrefix}/${normalizedModelId}`,
model: `${normalizedProviderId}/${normalizedModelId}`,
...(normalizedConnectionId ? { connectionId: normalizedConnectionId } : {}),
...(normalizedConnectionLabel ? { label: normalizedConnectionLabel } : {}),
...(normalizedAllowed.length > 0 ? { allowedConnectionIds: normalizedAllowed } : {}),
@@ -174,15 +160,10 @@ export function buildManualComboModelStep({
const providerId = resolveComboBuilderProviderId(parsed.providerId, providers);
if (!providerId) return null;
// #11433: preserve the user-typed prefix (e.g. "oc") as the routing prefix
// instead of letting buildPrecisionComboModelStep rebuild `model` from the
// resolved canonical providerId, which can collide with an unrelated
// manual alias override (e.g. "opencode" -> "opencode-zen").
return buildPrecisionComboModelStep({
providerId,
modelId: parsed.modelId,
weight,
modelPrefix: parsed.providerId,
});
}
@@ -244,7 +225,7 @@ type ComboBuilderGlobalProvider = {
displayName?: unknown;
connectionCount?: unknown;
connections?: unknown[];
models?: Array<{ id?: unknown; name?: unknown; qualifiedModel?: unknown }>;
models?: Array<{ id?: unknown; name?: unknown }>;
};
/**
@@ -271,18 +252,12 @@ export function buildGlobalModelList(
const modelId = toTrimmedString(model?.id);
if (!modelId) return;
const modelName = toTrimmedString(model?.name) || modelId;
// #11433: derive the routing prefix from the model's already-corrected
// `qualifiedModel` (e.g. "oc/<model>" for the OpenCode Free provider)
// instead of defaulting to the raw providerId, which can collide with
// an unrelated manual alias override.
const modelPrefix = parseQualifiedModel(model?.qualifiedModel)?.providerId || providerId;
const step = buildPrecisionComboModelStep({
providerId,
modelId,
connectionId: null,
connectionLabel: null,
allowedConnectionIds: [],
modelPrefix,
});
list.push({
providerId,

View File

@@ -42,12 +42,6 @@ test("buildPrecisionComboModelStep preserves provider/model/account triple", ()
});
test("buildManualComboModelStep resolves provider aliases and uses dynamic account", () => {
// #11433: `providerId` resolves to the canonical id ("codex") for
// duplicate-detection/routing identity, but the serialized `model` string
// now preserves the user-typed prefix ("cx/") verbatim instead of
// collapsing back to the canonical id — some canonical ids (e.g.
// "opencode") collide with an unrelated manual routing-alias override, so
// rebuilding `model` from the canonical id alone can silently misroute.
assert.deepEqual(
builderDraft.buildManualComboModelStep({
value: "cx/gpt-5.5",
@@ -56,7 +50,7 @@ test("buildManualComboModelStep resolves provider aliases and uses dynamic accou
{
kind: "model",
providerId: "codex",
model: "cx/gpt-5.5",
model: "codex/gpt-5.5",
weight: 0,
}
);

View File

@@ -0,0 +1,77 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-rr-diag-11462-"));
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
process.env.DATA_DIR = TEST_DATA_DIR;
const { handleComboChat } = await import("../../open-sse/services/combo.ts");
const core = await import("../../src/lib/db/core.ts");
const { resetAllComboMetrics } = await import("../../open-sse/services/comboMetrics.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
const { resetAll: resetAllSemaphores } = await import(
"../../open-sse/services/rateLimitSemaphore.ts"
);
function createLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse() {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test.beforeEach(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
});
test.after(() => {
resetAllComboMetrics();
resetAllCircuitBreakers();
resetAllSemaphores();
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
});
test(
"#11462: round-robin combo's 'Maximum combo retry limit reached' 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/excluded/terminalReason)",
async () => {
const result = await handleComboChat({
body: { messages: [{ role: "user", content: "hi" }] },
combo: {
name: "rr-probe-11462",
strategy: "round-robin",
models: ["openai/rr-a", "anthropic/rr-b"],
config: {
maxRetries: 0,
maxGlobalAttempts: 1,
concurrencyPerModel: 1,
queueTimeoutMs: 1000,
},
},
handleSingleModel: async () => failResponse(),
isModelAvailable: async () => true,
log: createLog(),
settings: null,
relayOptions: null,
allCombos: null,
});
assert.equal(result.status, 503);
const body = await result.json();
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "round-robin 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics.attemptOrder));
assert.ok(typeof body.diagnostics.terminalReason === "string");
}
);

View File

@@ -0,0 +1,88 @@
/**
* #11462: the nested runtime-unit loop (open-sse/services/combo/runtimeUnits.ts,
* used by the pipeline/fusion combo strategies via dispatchPrelude.ts and
* fusionPanel.ts) had the same bare-`errorResponse()` gap as the round-robin
* strategy's "Maximum combo retry limit reached" 503 — no diagnostics trace.
*/
import test from "node:test";
import assert from "node:assert/strict";
import { executeRuntimeUnitCombo } from "../../open-sse/services/combo/runtimeUnits.ts";
import type { ResolvedComboUnit, ComboNestingContext } from "../../open-sse/services/combo/types.ts";
function noopLog() {
return { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} };
}
function failResponse(): Response {
return new Response(JSON.stringify({ error: { message: "upstream 500" } }), {
status: 500,
headers: { "content-type": "application/json" },
});
}
test(
"#11462: nested runtime-unit combo's attempt-budget-exceeded 503 must carry " +
"the combo diagnostics trace (poolSize/attemptOrder/terminalReason)",
async () => {
const units: ResolvedComboUnit[] = [
{
kind: "model",
stepId: "step-a",
executionKey: "a",
modelStr: "openai/ru-a",
provider: "openai",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
{
kind: "model",
stepId: "step-b",
executionKey: "b",
modelStr: "anthropic/ru-b",
provider: "anthropic",
providerId: null,
connectionId: null,
weight: 1,
label: null,
},
];
const nesting: ComboNestingContext = {
depth: 0,
maxDepth: 5,
visitedComboNames: [],
rootComboName: "ru-probe-11462",
// Budget of 1 trips on the very first attempt, deterministically hitting the
// terminal branch under test without needing every unit to actually fail.
attemptBudget: { count: 0, limit: 1 },
};
const result = await executeRuntimeUnitCombo({
body: { messages: [{ role: "user", content: "hi" }] },
combo: { name: "ru-probe-11462", strategy: "pipeline" },
strategy: "pipeline",
units,
handleSingleModel: async () => failResponse(),
log: noopLog() as never,
config: { maxRetries: 0 },
allCombos: [],
nesting,
baseOptions: {} as never,
runCombo: async () => failResponse(),
});
assert.equal(result.response.status, 503);
const body = (await result.response.json()) as {
error: { message: string };
diagnostics?: { poolSize: number; attemptOrder: unknown[]; terminalReason: string };
};
assert.equal(body.error.message, "Maximum combo retry limit reached");
assert.ok(body.diagnostics, "runtime-unit 503 should carry a diagnostics field");
assert.ok(typeof body.diagnostics?.poolSize === "number");
assert.ok(Array.isArray(body.diagnostics?.attemptOrder));
assert.equal(body.diagnostics?.terminalReason, "max_attempts_exceeded");
}
);

View File

@@ -1,83 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import {
buildPrecisionComboModelStep,
buildGlobalModelList,
buildManualComboModelStep,
} from "../../src/lib/combos/builderDraft.ts";
import { resolveProviderAlias, parseModel } from "../../open-sse/services/model.ts";
// Issue #11433: the combo builder's precision-select path builds a step's
// `model` string as `${providerId}/${modelId}` using the CANONICAL provider id.
// For the no-auth "opencode" (OpenCode Free) provider this produces
// `model: "opencode/<modelId>"`, but `opencode` is ALSO a manual routing-prefix
// override (`ALIAS_TO_PROVIDER_ID["opencode"] = "opencode-zen"`) intended only
// for user-typed `opencode/` prefixes referring to the OpenCode Zen (api-key)
// tier. Parsing the step's own `model` string therefore resolves to a
// DIFFERENT provider than the one recorded in `step.providerId`.
test('sanity: resolveProviderAlias("opencode") is the manual override causing the collision', () => {
// Documents the root cause directly: the manual alias override in
// open-sse/services/model.ts unconditionally rewrites "opencode" to
// "opencode-zen", even though "opencode" is also a registered canonical
// provider id (src/shared/constants/providers/noauth.ts).
assert.equal(resolveProviderAlias("opencode"), "opencode-zen");
});
test("issue #11433 fix: buildPrecisionComboModelStep honors an explicit modelPrefix override", () => {
// The combo builder call sites now thread through the already-computed
// routing-alias prefix (e.g. "oc") instead of letting the step default to
// the raw providerId, so the serialized `model` field round-trips to the
// correct provider.
const step = buildPrecisionComboModelStep({
providerId: "opencode",
modelId: "big-pickle",
modelPrefix: "oc",
});
assert.equal(step.providerId, "opencode");
assert.equal(step.model, "oc/big-pickle");
const parsed = parseModel(step.model);
assert.equal(parsed.provider, step.providerId);
});
test("issue #11433 fix: buildGlobalModelList derives modelPrefix from qualifiedModel for the no-auth OpenCode Free provider", () => {
// Mirrors what src/lib/combos/builderOptions.ts::rewriteQualifiedModelPrefix
// produces for the no-auth "opencode" provider entry: `qualifiedModel` is
// already rewritten to the "oc/" alias prefix, but (pre-fix)
// buildGlobalModelList ignored it and rebuilt `model` from the raw
// providerId, producing "opencode/big-pickle" which parses back to the
// wrong provider ("opencode-zen").
const [entry] = buildGlobalModelList([
{
providerId: "opencode",
displayName: "OpenCode Free",
connectionCount: 0,
connections: [],
models: [{ id: "big-pickle", name: "Big Pickle", qualifiedModel: "oc/big-pickle" }],
},
]);
assert.equal(entry.step.providerId, "opencode");
assert.equal(entry.step.model, "oc/big-pickle");
assert.equal(parseModel(entry.step.model).provider, entry.step.providerId);
});
test("issue #11433 fix: buildManualComboModelStep preserves a user-typed oc/<model> prefix", () => {
// buildManualComboModelStep resolves the typed alias ("oc") back to the
// canonical providerId ("opencode") before building the step. Pre-fix, it
// then handed that canonical id straight to buildPrecisionComboModelStep,
// which rebuilt `model` from it and collapsed "oc/<model>" back down to
// "opencode/<model>" — reproducing the same collision for manual entry.
const step = buildManualComboModelStep({
value: "oc/big-pickle",
providers: [{ providerId: "opencode", alias: "oc" }],
});
assert.ok(step);
assert.equal(step?.providerId, "opencode");
assert.equal(step?.model, "oc/big-pickle");
assert.equal(parseModel(step!.model).provider, step!.providerId);
});