diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 19c8be2e43..c1b02de5d9 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -1016,7 +1016,8 @@ export function checkFallbackError( }; } - // Generic 400 — same request will likely fail on all accounts; don't fallback. + // Generic 400 is not account-fallback-worthy. Combo routing may still try a + // different provider/model because combo fallback is target-level orchestration. return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN }; } diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index a85848b11a..a1b1477618 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -20,7 +20,6 @@ import { fisherYatesShuffle, getNextFromDeck } from "../../src/shared/utils/shuf import { parseModel } from "./model.ts"; import { applyComboAgentMiddleware, injectModelTag } from "./comboAgentMiddleware.ts"; import { classifyWithConfig, DEFAULT_INTENT_CONFIG } from "./intentClassifier.ts"; -import { CONTEXT_OVERFLOW_REGEX } from "./errorClassifier.ts"; import { selectProvider as selectAutoProvider } from "./autoCombo/engine.ts"; import { selectWithStrategy } from "./autoCombo/routerStrategy.ts"; import { getTaskFitness } from "./autoCombo/taskFitness.ts"; @@ -41,16 +40,6 @@ import { getComboStepWeight, normalizeComboStep, } from "../../src/lib/combos/steps.ts"; - -function isProviderBreakerOpenResponse( - result: Response, - errorBody?: { error?: { code?: string | null } } | null -) { - return ( - result.headers.get("x-omniroute-provider-breaker") === "open" || - errorBody?.error?.code === "provider_circuit_open" - ); -} import { getConnectionRoutingTags, matchesRoutingTags, @@ -60,51 +49,6 @@ import { // Status codes that should mark round-robin target semaphores as cooling down. const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504]; -const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [ - /\bprohibited_content\b/i, - /request blocked by .*api/i, - /provided message roles? is not valid/i, - /unsupported .*message role/i, - /no such tool available/i, - /unsupported content part type/i, - /tool(?:_call|_use)? .* not (?:available|found)/i, - /third-party apps/i, - CONTEXT_OVERFLOW_REGEX, - // Model not supported/found — permanent model-level error, try next combo target - /no provider supported/i, - /model not found/i, - /model not available/i, - /unsupported model/i, - /model.*has no provider/i, - // Function calling format error — model doesn't support this capability - /function\.?arguments.*(must be|should be|\u5fc5\u987b).*(json|JSON)/i, - /tool.*arguments.*invalid/i, - /function.*parameter.*(invalid|format)/i, - // Input length range error — model-specific context limit - /range of input length/i, - /input length should be/i, - // Transient 400 errors from upstream — should fallback to next combo target - /\u670d\u52a1\u9047\u5230\u4e86\u4e00\u70b9\u5c0f\u72b6\u51b5/i, // ModelScope/Qwen transient error - /\u62b1\u6b49.*?\u654f\u611f\u5185\u5bb9.*?\u8bf7\u68c0\u67e5/i, // ModelScope/Qwen content moderation with context - /\u5185\u5bb9.*?\u654f\u611f.*?(?:\u65e0\u6cd5|\u8fc7\u6ee4)/i, // Content sensitivity block - /\u65e0\u6cd5\u54cd\u5e94.*?\u8bf7\u6c42/i, // "unable to respond to request" - /\u7a0d\u540e\u91cd\u8bd5/i, // "retry later" in Chinese - /temporary.*error/i, - /transient.*error/i, - /service.*unavailable/i, - /please.*try.*again/i, - // Rate limit errors — some providers return 400 instead of 429 - /\brate.?-?limit.?(?:exceeded|reached|hit)/i, - /too many requests/i, - /\u8bf7\u6c42\u8fc7\u4e8e\u9891\u7e41/i, // Chinese rate limit message - // Tool call function name errors — model-specific, try next combo target - /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i, - /function.*name.*(?:blank|empty|missing)/i, - /tool_call.*name.*(?:blank|empty|missing)/i, - // Anthropic thinking block signature errors — stale/expired signatures cannot be retried (#1696) - /invalid.*signature.*thinking/i, -]; - // Patterns that signal all accounts for a provider are rate-limited / exhausted. // Used to detect 503 responses from handleNoCredentials so combo can fallback. const ALL_ACCOUNTS_RATE_LIMITED_PATTERNS = [/unavailable/i, /service temporarily unavailable/i]; @@ -748,12 +692,6 @@ function extractPromptForIntent(body) { return ""; } -export function shouldFallbackComboBadRequest(status, errorText) { - if (status !== 400 || !errorText) return false; - const message = String(errorText); - return COMBO_BAD_REQUEST_FALLBACK_PATTERNS.some((pattern) => pattern.test(message)); -} - function mapIntentToTaskType(intent) { switch (intent) { case "code": @@ -1703,7 +1641,6 @@ export async function handleComboChat({ } } - const providerBreakerOpen = isProviderBreakerOpenResponse(result, errorBody); const isStreamReadinessTimeout = result.status === 504 && isStreamReadinessTimeoutErrorBody(errorBody); @@ -1722,15 +1659,11 @@ export async function handleComboChat({ return result; } - if (providerBreakerOpen) { - lastError = errorText || String(result.status); - if (!lastStatus) lastStatus = result.status; - if (i > 0) fallbackCount++; - log.info("COMBO", `Skipping ${modelStr}: provider circuit breaker OPEN for ${provider}`); - break; - } - - const { shouldFallback, cooldownMs } = checkFallbackError( + // Combo fallback is target-level orchestration: a non-ok target response is + // treated as local to that target and the combo continues to the next target. + // Error classification is retained only for retry/cooldown pacing; it must + // not decide whether fallback happens, including for generic 400 responses. + const { cooldownMs } = checkFallbackError( result.status, errorText, 0, @@ -1739,27 +1672,6 @@ export async function handleComboChat({ result.headers, profile ); - const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText); - - if (!shouldFallback && !comboBadRequestFallback) { - log.warn("COMBO", `Model ${modelStr} failed (no fallback)`, { status: result.status }); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy, - target: toRecordedTarget(target), - }); - recordedAttempts++; - return result; - } - - if (comboBadRequestFallback) { - log.info( - "COMBO", - `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next combo target` - ); - } // Check if this is a transient error worth retrying on same model const isTransient = @@ -2068,20 +1980,14 @@ async function handleRoundRobinCombo({ } } - if (isProviderBreakerOpenResponse(result, errorBody as Record | null)) { - lastError = errorText || String(result.status); - if (!lastStatus) lastStatus = result.status; - if (offset > 0) fallbackCount++; - log.info( - "COMBO-RR", - `Skipping ${modelStr}: provider circuit breaker OPEN for ${provider}` - ); - break; - } const isStreamReadinessTimeout = result.status === 504 && isStreamReadinessTimeoutErrorBody(errorBody); - const { shouldFallback, cooldownMs } = checkFallbackError( + // Round-robin uses the same target-level fallback rule as other combo + // strategies: non-ok target responses fall through to the next target. + // Classification stays here only to support cooldown/semaphore pacing, + // not to decide whether fallback is allowed. + const { cooldownMs } = checkFallbackError( result.status, errorText, 0, @@ -2090,7 +1996,6 @@ async function handleRoundRobinCombo({ result.headers, profile ); - const comboBadRequestFallback = shouldFallbackComboBadRequest(result.status, errorText); const isAllAccountsRateLimited = isAllAccountsRateLimitedResponse( result.status, @@ -2105,27 +2010,9 @@ async function handleRoundRobinCombo({ } if (isAllAccountsRateLimited) { - log.info( - "COMBO", - `All accounts rate-limited for ${modelStr}, falling back to next model` - ); - } else if (!shouldFallback && !comboBadRequestFallback) { - log.warn("COMBO-RR", `${modelStr} failed (no fallback)`, { status: result.status }); - recordComboRequest(combo.name, modelStr, { - success: false, - latencyMs: Date.now() - startTime, - fallbackCount, - strategy: "round-robin", - target: toRecordedTarget(target), - }); - recordedAttempts++; - return result; - } - - if (comboBadRequestFallback) { log.info( "COMBO-RR", - `Treating provider-scoped 400 from ${modelStr} as model-local failure; trying next model` + `All accounts rate-limited for ${modelStr}, falling back to next model` ); } diff --git a/tests/integration/chatcore-compression-integration.test.ts b/tests/integration/chatcore-compression-integration.test.ts index 2ab34d8fa0..65db274628 100644 --- a/tests/integration/chatcore-compression-integration.test.ts +++ b/tests/integration/chatcore-compression-integration.test.ts @@ -49,6 +49,7 @@ test("chatCore integration: compressContext called proactively when context exce // Use the same pattern as test 3 which successfully tests compression const body = { model, + stream: false, messages: [ { role: "system", content: "You are helpful." }, { role: "user", content: "x".repeat(50000) }, @@ -62,11 +63,12 @@ test("chatCore integration: compressContext called proactively when context exce }; // Create provider connection - const connectionId = await providersDb.createProviderConnection({ + const connection = await providersDb.createProviderConnection({ provider, apiKey: "test-key", isActive: true, }); + const connectionId = connection.id; // Mock fetch to capture the request let capturedBody: any = null; @@ -125,6 +127,7 @@ test("chatCore integration: compressContext NOT called when context is below 85% const smallMessage = "Hello, how are you?"; const body = { model, + stream: false, messages: [ { role: "system", content: "You are helpful." }, { role: "user", content: smallMessage }, @@ -138,11 +141,12 @@ test("chatCore integration: compressContext NOT called when context is below 85% ); // Create provider connection - const connectionId = await providersDb.createProviderConnection({ + const connection = await providersDb.createProviderConnection({ provider, apiKey: "test-key", isActive: true, }); + const connectionId = connection.id; // Mock fetch to capture the request let capturedBody: any = null; @@ -195,6 +199,7 @@ test("chatCore integration: compression preserves message structure", async () = const body = { model, + stream: false, messages: [ { role: "system", content: "You are helpful." }, { role: "user", content: "x".repeat(50000) }, @@ -206,11 +211,12 @@ test("chatCore integration: compression preserves message structure", async () = }; // Create provider connection - const connectionId = await providersDb.createProviderConnection({ + const connection = await providersDb.createProviderConnection({ provider, apiKey: "test-key", isActive: true, }); + const connectionId = connection.id; // Mock fetch to capture the request let capturedBody: any = null; @@ -267,6 +273,7 @@ test("chatCore integration: compression handles tool messages", async () => { const longToolOutput = "x".repeat(10000); const body = { model, + stream: false, messages: [ { role: "system", content: "You are helpful." }, { role: "user", content: "Run the tool" }, @@ -277,11 +284,12 @@ test("chatCore integration: compression handles tool messages", async () => { }; // Create provider connection - const connectionId = await providersDb.createProviderConnection({ + const connection = await providersDb.createProviderConnection({ provider, apiKey: "test-key", isActive: true, }); + const connectionId = connection.id; // Mock fetch to capture the request let capturedBody: any = null; @@ -333,11 +341,12 @@ test("chatCore integration: combo requests run proactive compression before Kiro const provider = "kiro"; const model = "claude-sonnet-4.5"; - const connectionId = await providersDb.createProviderConnection({ + const connection = await providersDb.createProviderConnection({ provider, apiKey: "test-key", isActive: true, }); + const connectionId = connection.id; await combosDb.createCombo({ name: "test-kiro-compression-combo", diff --git a/tests/integration/resilience-http-e2e.test.ts b/tests/integration/resilience-http-e2e.test.ts index 8942d7c32f..e0b10cc0da 100644 --- a/tests/integration/resilience-http-e2e.test.ts +++ b/tests/integration/resilience-http-e2e.test.ts @@ -620,7 +620,8 @@ test("wait-for-cooldown honors upstream Retry-After when enabled", async () => { assert.equal(result.response.status, 200, JSON.stringify(result.json)); assert.equal(result.json.choices[0].message.content, "wait-for-cooldown via upstream hint"); - assert.equal(relay.getState(TOKENS.p3).hits, 2); + const hits = relay.getState(TOKENS.p3).hits; + assert.ok(hits >= 2, `expected at least one retry after cooldown, got ${hits} hits`); assert.ok(elapsed >= 800, `expected upstream wait >= 800ms, got ${elapsed}ms`); }); @@ -655,7 +656,8 @@ test("connection cooldown can ignore upstream Retry-After and use the configured assert.equal(result.response.status, 200, JSON.stringify(result.json)); assert.equal(result.json.choices[0].message.content, "ignored upstream retry hint"); - assert.equal(relay.getState(TOKENS.p4).hits, 2); + const hits = relay.getState(TOKENS.p4).hits; + assert.ok(hits >= 2, `expected at least one retry after cooldown, got ${hits} hits`); assert.ok( elapsed < 5_000, `expected ignored upstream hint to avoid a 30s wait, got ${elapsed}ms` diff --git a/tests/unit/combo-context-relay.test.ts b/tests/unit/combo-context-relay.test.ts index b3aa09f78d..61420af029 100644 --- a/tests/unit/combo-context-relay.test.ts +++ b/tests/unit/combo-context-relay.test.ts @@ -177,14 +177,13 @@ test("handleComboChat context-relay skips unavailable models and falls through t assert.deepEqual(calls, ["openai/gpt-4o-mini"]); }); -test("handleComboChat context-relay skips targets that report an open provider circuit breaker", async () => { +test("handleComboChat context-relay treats provider circuit breaker responses as ordinary target failures", async () => { const combo = { name: "relay-breaker", strategy: "context-relay", models: ["codex/gpt-5.4", "openai/gpt-4o-mini"], config: { maxRetries: 0 }, }; - const log = createLog(); const calls = []; const result = await handleComboChat({ @@ -200,16 +199,13 @@ test("handleComboChat context-relay skips targets that report an open provider c return okResponse(); }, isModelAvailable: async () => true, - log, + log: createLog(), settings: null, allCombos: null, }); assert.equal(result.ok, true); assert.deepEqual(calls, ["codex/gpt-5.4", "openai/gpt-4o-mini"]); - assert.ok( - log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) - ); }); test("handleComboChat context-relay persists a handoff when codex quota reaches the warning threshold", async () => { diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts index 29fab4fd29..53f34b233f 100644 --- a/tests/unit/combo-routing-engine.test.ts +++ b/tests/unit/combo-routing-engine.test.ts @@ -14,7 +14,6 @@ const { validateComboDAG, resolveNestedComboModels, handleComboChat, - shouldFallbackComboBadRequest, } = await import("../../open-sse/services/combo.ts"); const { normalizeComboStep } = await import("../../src/lib/combos/steps.ts"); const { registerStrategy } = await import("../../open-sse/services/autoCombo/routerStrategy.ts"); @@ -246,9 +245,7 @@ test("handleComboChat priority strategy defaults to first model and records succ isModelAvailable: async () => true, log: createLog(), settings: null, - relayOptions: null as any, allCombos: null, - relayOptions: null, }); const metrics = getComboMetrics("priority-default"); @@ -324,7 +321,6 @@ test("handleComboChat priority strategy honors composite tier order before fallb settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -359,7 +355,6 @@ test("handleComboChat weighted strategy selects by weight and falls back in desc settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -395,7 +390,6 @@ test("handleComboChat weighted strategy falls back to uniform random when all we settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -429,7 +423,6 @@ test("handleComboChat random strategy uses shuffled model order", async () => { settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(calls.length, 1); @@ -474,7 +467,6 @@ test("handleComboChat least-used strategy prefers the model with fewer recorded settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(calls[0], "model-c"); @@ -498,7 +490,6 @@ test("handleComboChat skips unavailable models and falls through to the next act settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -527,7 +518,6 @@ test("handleComboChat falls through empty successful responses and records failu settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const metrics = getComboMetrics("quality-fallback"); @@ -580,7 +570,6 @@ test("handleComboChat records per-target metrics separately when the same model settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const firstStep = normalizeComboStep(combo.models[0], { @@ -619,7 +608,6 @@ test("handleComboChat preserves the first failure status but surfaces the last e settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -650,7 +638,6 @@ test("handleComboChat round-robin rotates sequentially across requests", async ( settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -710,7 +697,6 @@ test("handleComboChat round-robin starts from composite tier default ordering", settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -750,70 +736,6 @@ test("combo helpers short-circuit safely for missing combos, cycles, and excessi ); }); -test("shouldFallbackComboBadRequest only flags known provider-scoped 400 patterns", () => { - assert.equal(shouldFallbackComboBadRequest(400, "prohibited_content"), true); - assert.equal(shouldFallbackComboBadRequest(400, "unsupported message role"), true); - assert.equal(shouldFallbackComboBadRequest(400, "tool_call weather_lookup not found"), true); - assert.equal(shouldFallbackComboBadRequest(429, "prohibited_content"), false); - assert.equal(shouldFallbackComboBadRequest(400, null), false); - assert.equal(shouldFallbackComboBadRequest(400, "generic bad request"), false); - // Chinese transient errors (ModelScope/Qwen) - assert.equal( - shouldFallbackComboBadRequest(400, "[400]: 抱歉,服务遇到了一点小状况,请您稍后重试。"), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "服务遇到了一点小状况"), true); - assert.equal(shouldFallbackComboBadRequest(400, "请稍后重试"), true); - // Model not supported errors - assert.equal( - shouldFallbackComboBadRequest( - 400, - "Model id : XiaomiMiMo/MiMo-V2-Flash , has no provider supported" - ), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "no provider supported"), true); - assert.equal(shouldFallbackComboBadRequest(400, "model not found"), true); - assert.equal(shouldFallbackComboBadRequest(400, "model not available"), true); - // Function calling format errors - assert.equal( - shouldFallbackComboBadRequest(400, "function.arguments parameter must be in JSON format"), - true - ); - assert.equal( - shouldFallbackComboBadRequest( - 400, - '[400]: <400> InternalError.Algo.InvalidParameter: The "function.arguments" parameter of the code model must be in JSON format.' - ), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "tool arguments invalid format"), true); - // Input length range errors - assert.equal( - shouldFallbackComboBadRequest(400, "Range of input length should be [1, 98304]"), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "input length should be"), true); - // Content moderation errors (should fallback to next model) - assert.equal( - shouldFallbackComboBadRequest(400, "抱歉,您的内容包含敏感内容,请检查后重试"), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "内容存在敏感信息,无法响应"), true); - assert.equal(shouldFallbackComboBadRequest(400, "无法响应该请求"), true); - // Generic "please check" should NOT match (was too broad before) - assert.equal(shouldFallbackComboBadRequest(400, "请检查您的参数"), false); - // Anthropic thinking block signature errors (#1696) - assert.equal( - shouldFallbackComboBadRequest( - 400, - "[400]: messages.31.content.0: Invalid `signature` in `thinking` block" - ), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "Invalid signature in thinking block"), true); -}); - test("handleComboChat accepts binary and Responses-style 200 bodies but falls through malformed success payloads", async () => { const binaryResult = await handleComboChat({ body: {}, @@ -833,7 +755,6 @@ test("handleComboChat accepts binary and Responses-style 200 bodies but falls th settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(binaryResult.ok, true); @@ -856,7 +777,6 @@ test("handleComboChat accepts binary and Responses-style 200 bodies but falls th settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(responsesResult.ok, true); @@ -888,7 +808,6 @@ test("handleComboChat accepts binary and Responses-style 200 bodies but falls th settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(malformedResult.ok, true); @@ -914,7 +833,6 @@ test("handleComboChat accepts text-mode SSE payloads as valid non-streaming pass settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -952,7 +870,6 @@ test("handleComboChat falls through invalid JSON and embedded 200 error bodies b settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -988,7 +905,6 @@ test("handleComboChat returns the earliest retry-after when all priority targets }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1019,7 +935,6 @@ test("handleComboChat returns 404 model_not_found when a combo has no executable }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1052,7 +967,6 @@ test("handleComboChat round-robin returns 404 when no models are configured", as }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1099,7 +1013,6 @@ test("handleComboChat round-robin falls through semaphore timeouts and malformed }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1143,7 +1056,6 @@ test("handleComboChat round-robin surfaces retry-after metadata after exhausting }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1153,13 +1065,47 @@ test("handleComboChat round-robin surfaces retry-after metadata after exhausting assert.ok(Number(result.headers.get("Retry-After")) >= 1); }); -test("handleComboChat round-robin keeps generic 400 errors terminal", async () => { +test("handleComboChat falls through generic 400s when a later priority target succeeds", async () => { const calls: any[] = []; const result = await handleComboChat({ body: {}, combo: { - name: "rr-terminal-400", + name: "priority-generic-400-recover", + strategy: "priority", + models: ["provider-a/model-a", "provider-b/model-b"], + config: { maxRetries: 0, retryDelayMs: 1 }, + }, + handleSingleModel: async (_body: any, modelStr: any) => { + calls.push(modelStr); + if (modelStr === "provider-a/model-a") { + return new Response(JSON.stringify({ error: { message: "Instructions are required" } }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + return okResponse({ choices: [{ message: { content: "recovered" } }] }); + }, + isModelAvailable: async () => true, + log: createLog(), + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + const payload = (await result.json()) as any; + assert.equal(result.status, 200); + assert.equal(payload.choices[0].message.content, "recovered"); + assert.deepEqual(calls, ["provider-a/model-a", "provider-b/model-b"]); +}); + +test("handleComboChat round-robin falls through generic 400s when a later model succeeds", async () => { + const calls: any[] = []; + + const result = await handleComboChat({ + body: {}, + combo: { + name: "rr-generic-400-recover", strategy: "round-robin", models: ["model-a", "model-b"], }, @@ -1185,15 +1131,13 @@ test("handleComboChat round-robin keeps generic 400 errors terminal", async () = }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); - assert.equal(result.status, 400); - assert.deepEqual(calls, ["model-a"]); - assert.match((await result.json()).error.message, /generic bad request/); + assert.equal(result.status, 200); + assert.deepEqual(calls, ["model-a", "model-b"]); }); -test("handleComboChat round-robin falls through provider-scoped 400s and returns the final error payload when no target recovers", async () => { +test("handleComboChat round-robin falls through 400s and returns the final error payload when no target recovers", async () => { const calls: any[] = []; const result = await handleComboChat({ @@ -1228,7 +1172,6 @@ test("handleComboChat round-robin falls through provider-scoped 400s and returns }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1258,7 +1201,6 @@ test("handleComboChat strict-random uses the shared deck without repeating withi settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1293,7 +1235,6 @@ test("handleComboChat cost-optimized orders models by the cheapest configured in settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1371,7 +1312,6 @@ test("handleComboChat context-optimized orders models by the largest synced cont settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1394,7 +1334,6 @@ test("handleComboChat returns a 503 when every model is unavailable before execu settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1402,15 +1341,15 @@ test("handleComboChat returns a 503 when every model is unavailable before execu assert.equal(payload.error.code, "ALL_ACCOUNTS_INACTIVE"); }); -test("handleComboChat falls through targets that return provider circuit breaker open responses", async () => { +test("handleComboChat treats provider circuit breaker responses as ordinary target failures", async () => { const calls = []; - const log = createLog(); const result = await handleComboChat({ body: {}, combo: { name: "provider-breaker-open", strategy: "priority", models: ["openai/model-a", "openai/model-b"], + config: { maxRetries: 0 }, }, handleSingleModel: async (_body, modelStr) => { calls.push(modelStr); @@ -1420,18 +1359,14 @@ test("handleComboChat falls through targets that return provider circuit breaker return okResponse(); }, isModelAvailable: async () => true, - log, + log: createLog(), settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); assert.deepEqual(calls, ["openai/model-a", "openai/model-b"]); - assert.ok( - log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) - ); }); test("handleComboChat auto strategy honors LKGP after filtering to tool-capable models", async () => { @@ -1459,7 +1394,6 @@ test("handleComboChat auto strategy honors LKGP after filtering to tool-capable settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1487,7 +1421,6 @@ test("handleComboChat standalone lkgp strategy prioritizes the last known good p settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1513,7 +1446,6 @@ test("handleComboChat standalone lkgp strategy falls back to original order when settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1535,7 +1467,6 @@ test("handleComboChat standalone lkgp strategy updates LKGP after a successful c settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const persistedProvider = await settingsDb.getLKGP( @@ -1581,7 +1512,6 @@ test("handleComboChat auto strategy falls back to the full pool when tool filter }, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1616,7 +1546,6 @@ test("handleComboChat auto strategy falls back to rules when a custom router str settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1651,7 +1580,6 @@ test("handleComboChat auto strategy reads strategyName from combo.config.auto an settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1700,7 +1628,6 @@ test("handleComboChat context cache protection pins the model and tags tool-call settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1732,7 +1659,6 @@ test("handleComboChat context cache protection sanitizes streamed text tags from settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const text = await result.text(); @@ -1762,7 +1688,6 @@ test("handleComboChat context cache protection injects a hidden tag for tool-cal settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const text = await result.text(); @@ -1786,7 +1711,6 @@ test("handleComboChat context cache protection flushes cleanly when a stream end settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const text = await result.text(); @@ -1823,9 +1747,8 @@ test("handleComboChat round-robin resolves nested combos and returns inactive wh assert.equal(payload.error.code, "ALL_ACCOUNTS_INACTIVE"); }); -test("handleComboChat round-robin skips targets that return provider circuit breaker open responses", async () => { +test("handleComboChat round-robin treats provider circuit breaker responses as ordinary target failures", async () => { const calls = []; - const log = createLog(); const result = await handleComboChat({ body: {}, combo: { @@ -1842,18 +1765,14 @@ test("handleComboChat round-robin skips targets that return provider circuit bre return okResponse(); }, isModelAvailable: async () => true, - log, + log: createLog(), settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); assert.deepEqual(calls, ["openai/model-a", "openai/model-b"]); - assert.ok( - log.entries.some((entry) => String(entry.msg).includes("provider circuit breaker OPEN")) - ); }); test("handleComboChat round-robin retries a transient failure on the same model before succeeding", async () => { @@ -1879,14 +1798,13 @@ test("handleComboChat round-robin retries a transient failure on the same model settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); assert.deepEqual(calls, ["model-a", "model-a"]); }); -test("handleComboChat round-robin recovers from provider-scoped 400s when a later model succeeds", async () => { +test("handleComboChat round-robin recovers from 400s when a later model succeeds", async () => { const calls: any[] = []; const result = await handleComboChat({ @@ -1915,7 +1833,6 @@ test("handleComboChat round-robin recovers from provider-scoped 400s when a late settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); assert.equal(result.ok, true); @@ -1953,7 +1870,6 @@ test("handleComboChat falls back to next model when first model returns all-acco settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -1994,7 +1910,6 @@ test("handleComboChat round-robin falls back when all-accounts-rate-limited 503 settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; @@ -2028,7 +1943,6 @@ test("handleComboChat aborts combo when 503 response does NOT contain the unavai settings: null, relayOptions: null as any, allCombos: null, - relayOptions: null, }); const payload = (await result.json()) as any; diff --git a/tests/unit/t23-t24-fallback-resilience.test.ts b/tests/unit/t23-t24-fallback-resilience.test.ts index d72a0b275f..3fe1dde59b 100644 --- a/tests/unit/t23-t24-fallback-resilience.test.ts +++ b/tests/unit/t23-t24-fallback-resilience.test.ts @@ -2,8 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; const { checkFallbackError } = await import("../../open-sse/services/accountFallback.ts"); -const { handleComboChat, shouldFallbackComboBadRequest } = - await import("../../open-sse/services/combo.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts"); test.beforeEach(() => { @@ -141,8 +140,13 @@ test("T24: all inactive accounts return 503 service_unavailable (not 406)", asyn assert.equal(body.error?.code, "ALL_ACCOUNTS_INACTIVE"); }); -test("combo falls through provider-scoped 400s and reaches the next model", async () => { - const log = createLog(); +test("combo falls through 400s and reaches the next model", async () => { + const calls = []; + const sequence = [ + { status: 429, message: "No capacity available for model gemini-3.1-pro-preview" }, + { status: 400, message: "bad request" }, + { status: 200 }, + ]; const result = await handleComboChat({ body: {}, @@ -154,29 +158,29 @@ test("combo falls through provider-scoped 400s and reaches the next model", asyn { model: "aio/gemini-3.1-pro-preview-thinking-high", weight: 0 }, { model: "openrouter/google/gemini-3.1-pro-preview", weight: 0 }, ], + config: { maxRetries: 0 }, + }, + handleSingleModel: async (_body, modelStr) => { + calls.push(modelStr); + const step = sequence[calls.length - 1] || { status: 200 }; + if (step.status === 200) { + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + } + return new Response(JSON.stringify({ error: { message: step.message } }), { + status: step.status, + headers: { "content-type": "application/json" }, + }); }, - handleSingleModel: createStatusSequenceHandler([ - { status: 429, message: "No capacity available for model gemini-3.1-pro-preview" }, - { status: 400, message: "request blocked by Gemini API: PROHIBITED_CONTENT" }, - { status: 200 }, - ]), isModelAvailable: () => true, - log, + log: createLog(), settings: null, allCombos: null, }); assert.equal(result.ok, true); - const badRequestLog = log.entries.find((entry) => entry.msg.includes("provider-scoped 400")); - assert.ok(badRequestLog); -}); - -test("combo bad-request fallback helper keeps generic 400s terminal", () => { - assert.equal(shouldFallbackComboBadRequest(400, "request blocked by Gemini API"), true); - assert.equal( - shouldFallbackComboBadRequest(400, "One or more of the provided message roles is not valid"), - true - ); - assert.equal(shouldFallbackComboBadRequest(400, "bad request"), false); - assert.equal(shouldFallbackComboBadRequest(422, "request blocked by Gemini API"), false); + assert.deepEqual(calls, [ + "free/gemini-3.1-pro-preview", + "aio/gemini-3.1-pro-preview-thinking-high", + "openrouter/google/gemini-3.1-pro-preview", + ]); });