From 976d670ff3a7712df0c695f13095c43eace5e29b Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 7 Aug 2026 13:45:58 -0300 Subject: [PATCH] fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630) Closes #9630 --- changelog.d/fixes/9630-combo-false-503.md | 1 + .../services/antigravityProjectPersistence.ts | 13 +++ open-sse/services/combo.ts | 82 ++++++++++++------- tests/unit/repro-9630-combo-false-503.test.ts | 79 ++++++++++++++++++ 4 files changed, 146 insertions(+), 29 deletions(-) create mode 100644 changelog.d/fixes/9630-combo-false-503.md create mode 100644 open-sse/services/antigravityProjectPersistence.ts create mode 100644 tests/unit/repro-9630-combo-false-503.test.ts diff --git a/changelog.d/fixes/9630-combo-false-503.md b/changelog.d/fixes/9630-combo-false-503.md new file mode 100644 index 0000000000..5558818649 --- /dev/null +++ b/changelog.d/fixes/9630-combo-false-503.md @@ -0,0 +1 @@ +- fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630) diff --git a/open-sse/services/antigravityProjectPersistence.ts b/open-sse/services/antigravityProjectPersistence.ts new file mode 100644 index 0000000000..f34445fe00 --- /dev/null +++ b/open-sse/services/antigravityProjectPersistence.ts @@ -0,0 +1,13 @@ +/** + * Re-export from `antigravityProjectPersist.ts` plus a connection-preference helper. + */ +import { persistDiscoveredAntigravityProjectId } from "./antigravityProjectPersist.ts"; +export { persistDiscoveredAntigravityProjectId }; + +export function preferAntigravityConnectionsWithStoredProject( + connections: Array> +): Array> { + return connections.filter( + (conn) => conn != null && typeof conn.projectId === "string" && conn.projectId.trim().length > 0 + ); +} diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 37c270e25c..e8fdb613ef 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -2036,23 +2036,35 @@ export async function handleComboChat({ if (setTry < maxSetRetries) continue; // All set retries exhausted — return the final error - if (!lastStatus) { - notifyWebhookEvent("request.failed", { - combo: combo.name, - reason: "ALL_ACCOUNTS_INACTIVE", - latencyMs, - fallbackCount, - }); - // Silent-stop fix: bump the failure counter so the session pin clears on the 3rd - // consecutive all-inactive cascade; buildRecoveryHint emits `switch-combo` with a - // next-step that points the user at /dashboard/providers. - recordComboFailure(effectiveSessionId, combo.name); - return errorResponseWithComboDiagnostics( - 503, - "Service temporarily unavailable: all upstream accounts are inactive", - buildComboDiag("all_accounts_inactive"), - { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" } - ); + if (!lastStatus) { + if (recordedAttempts === 0) { + notifyWebhookEvent("request.failed", { + combo: combo.name, + reason: "ALL_TARGETS_SKIPPED", + latencyMs, + fallbackCount, + }); + return errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", + buildComboDiag("all_targets_skipped"), + { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" } + ); + } + notifyWebhookEvent("request.failed", { + combo: combo.name, + reason: "ALL_ACCOUNTS_INACTIVE", + latencyMs, + fallbackCount, + }); + recordComboFailure(effectiveSessionId, combo.name); + return errorResponseWithComboDiagnostics( + 503, + "Service temporarily unavailable: all upstream accounts are inactive", + buildComboDiag("all_accounts_inactive"), + { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" } + ); + } } const status = lastStatus; @@ -3004,18 +3016,30 @@ async function handleRoundRobinCombo({ }); } - if (!lastStatus) { - return new Response( - JSON.stringify({ - error: { - message: "Service temporarily unavailable: all upstream accounts are inactive", - type: "service_unavailable", - code: "ALL_ACCOUNTS_INACTIVE", - }, - }), - { status: 503, headers: { "Content-Type": "application/json" } } - ); - } + if (!lastStatus) { + if (recordedAttempts === 0) { + return new Response( + JSON.stringify({ + error: { + message: "Service temporarily unavailable: all targets were skipped by pre-dispatch filters", + type: "service_unavailable", + code: "ALL_TARGETS_SKIPPED", + }, + }), + { status: 503, headers: { "Content-Type": "application/json" } } + ); + } + return new Response( + JSON.stringify({ + error: { + message: "Service temporarily unavailable: all upstream accounts are inactive", + type: "service_unavailable", + code: "ALL_ACCOUNTS_INACTIVE", + }, + }), + { status: 503, headers: { "Content-Type": "application/json" } } + ); + } const status = lastStatus; const msg = lastError || "All round-robin combo models unavailable"; diff --git a/tests/unit/repro-9630-combo-false-503.test.ts b/tests/unit/repro-9630-combo-false-503.test.ts new file mode 100644 index 0000000000..53901640c3 --- /dev/null +++ b/tests/unit/repro-9630-combo-false-503.test.ts @@ -0,0 +1,79 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + handleComboChat, +} from "../../open-sse/services/combo.ts"; +import { getCircuitBreaker, STATE } from "../../src/shared/utils/circuitBreaker.js"; + +function okResponse() { + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +test("#9630: combo returns 503 when circuit breaker is OPEN but other healthy targets exist", async () => { + const cb = getCircuitBreaker("openai"); + cb.state = STATE.OPEN; + cb.resetTimeout = 60000; + cb.failureCount = 5; + cb.failureThreshold = 3; + cb.lastFailureTime = Date.now(); + + const result = await handleComboChat({ + body: { messages: [{ role: "user", content: "hello" }] }, + combo: { + name: "repro-9630", + strategy: "priority", + models: ["openai/gpt-4", "anthropic/claude-opus-5"], + }, + handleSingleModel: async (_body: any, modelStr: string) => { + assert.equal(modelStr, "anthropic/claude-opus-5", "should skip openai breaker and try anthropic"); + return okResponse(); + }, + isModelAvailable: async () => true, + log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any, + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.ok(result.ok, "should succeed via anthropic fallback when openai breaker is open"); +}); + +test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when ALL targets are breaker-open", async () => { + const cb = getCircuitBreaker("openai"); + cb.state = STATE.OPEN; + cb.resetTimeout = 60000; + cb.failureCount = 5; + cb.failureThreshold = 3; + cb.lastFailureTime = Date.now(); + + const cb2 = getCircuitBreaker("anthropic"); + cb2.state = STATE.OPEN; + cb2.resetTimeout = 60000; + cb2.failureCount = 5; + cb2.failureThreshold = 3; + cb2.lastFailureTime = Date.now(); + + const result = await handleComboChat({ + body: { messages: [{ role: "user", content: "hello" }] }, + combo: { + name: "repro-9630-all-breaker", + strategy: "priority", + models: ["openai/gpt-4", "anthropic/claude-opus-5"], + }, + handleSingleModel: async () => { throw new Error("should not be called"); }, + isModelAvailable: async () => true, + log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any, + settings: null, + relayOptions: null as any, + allCombos: null, + }); + + assert.equal(result.status, 503); + const body = await result.json(); + // The diagnostic should NOT claim ALL_ACCOUNTS_INACTIVE when no real dispatch was attempted + assert.notEqual(body.error?.code, "ALL_ACCOUNTS_INACTIVE", + "should not claim ALL_ACCOUNTS_INACTIVE when all targets were gated by pre-dispatch checks"); +});