fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630)

Closes #9630

changelog.d/fixes/9630-combo-false-503.md

@@ -0,0 +1 @@
+- fix(combo): distinguish pre-dispatch skips from genuine failures to prevent false 503 ALL_ACCOUNTS_INACTIVE (#9630)

open-sse/services/antigravityProjectPersistence.ts

@@ -0,0 +1,13 @@
+/**
+ * Re-export from `antigravityProjectPersist.ts` plus a connection-preference helper.
+ */
+import { persistDiscoveredAntigravityProjectId } from "./antigravityProjectPersist.ts";
+export { persistDiscoveredAntigravityProjectId };
+
+export function preferAntigravityConnectionsWithStoredProject(
+  connections: Array<Record<string, unknown>>
+): Array<Record<string, unknown>> {
+  return connections.filter(
+    (conn) => conn != null && typeof conn.projectId === "string" && conn.projectId.trim().length > 0
+  );
+}

open-sse/services/combo.ts

@@ -2036,23 +2036,35 @@ export async function handleComboChat({
       if (setTry < maxSetRetries) continue;
 
       // All set retries exhausted — return the final error
-      if (!lastStatus) {
+	      if (!lastStatus) {
-        notifyWebhookEvent("request.failed", {
+	        if (recordedAttempts === 0) {
-          combo: combo.name,
+	          notifyWebhookEvent("request.failed", {
-          reason: "ALL_ACCOUNTS_INACTIVE",
+	            combo: combo.name,
-          latencyMs,
+	            reason: "ALL_TARGETS_SKIPPED",
-          fallbackCount,
+	            latencyMs,
-        });
+	            fallbackCount,
-        // Silent-stop fix: bump the failure counter so the session pin clears on the 3rd
+	          });
-        // consecutive all-inactive cascade; buildRecoveryHint emits `switch-combo` with a
+	          return errorResponseWithComboDiagnostics(
-        // next-step that points the user at /dashboard/providers.
+	            503,
-        recordComboFailure(effectiveSessionId, combo.name);
+	            "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
-        return errorResponseWithComboDiagnostics(
+	            buildComboDiag("all_targets_skipped"),
-          503,
+	            { code: "ALL_TARGETS_SKIPPED", type: "service_unavailable" }
-          "Service temporarily unavailable: all upstream accounts are inactive",
+	          );
-          buildComboDiag("all_accounts_inactive"),
+	        }
-          { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
+	        notifyWebhookEvent("request.failed", {
-        );
+	          combo: combo.name,
+	          reason: "ALL_ACCOUNTS_INACTIVE",
+	          latencyMs,
+	          fallbackCount,
+	        });
+	        recordComboFailure(effectiveSessionId, combo.name);
+	        return errorResponseWithComboDiagnostics(
+	          503,
+	          "Service temporarily unavailable: all upstream accounts are inactive",
+	          buildComboDiag("all_accounts_inactive"),
+	          { code: "ALL_ACCOUNTS_INACTIVE", type: "service_unavailable" }
+	        );
+	      }
       }
 
       const status = lastStatus;
@@ -3004,18 +3016,30 @@ async function handleRoundRobinCombo({
     });
   }
 
-  if (!lastStatus) {
+	  if (!lastStatus) {
-    return new Response(
+	    if (recordedAttempts === 0) {
-      JSON.stringify({
+	      return new Response(
-        error: {
+	        JSON.stringify({
-          message: "Service temporarily unavailable: all upstream accounts are inactive",
+	          error: {
-          type: "service_unavailable",
+	            message: "Service temporarily unavailable: all targets were skipped by pre-dispatch filters",
-          code: "ALL_ACCOUNTS_INACTIVE",
+	            type: "service_unavailable",
-        },
+	            code: "ALL_TARGETS_SKIPPED",
-      }),
+	          },
-      { status: 503, headers: { "Content-Type": "application/json" } }
+	        }),
-    );
+	        { status: 503, headers: { "Content-Type": "application/json" } }
-  }
+	      );
+	    }
+	    return new Response(
+	      JSON.stringify({
+	        error: {
+	          message: "Service temporarily unavailable: all upstream accounts are inactive",
+	          type: "service_unavailable",
+	          code: "ALL_ACCOUNTS_INACTIVE",
+	        },
+	      }),
+	      { status: 503, headers: { "Content-Type": "application/json" } }
+	    );
+	  }
 
   const status = lastStatus;
   const msg = lastError || "All round-robin combo models unavailable";

tests/unit/repro-9630-combo-false-503.test.ts

@@ -0,0 +1,79 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  handleComboChat,
+} from "../../open-sse/services/combo.ts";
+import { getCircuitBreaker, STATE } from "../../src/shared/utils/circuitBreaker.js";
+
+function okResponse() {
+  return new Response(JSON.stringify({ ok: true }), {
+    status: 200,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+test("#9630: combo returns 503 when circuit breaker is OPEN but other healthy targets exist", async () => {
+  const cb = getCircuitBreaker("openai");
+  cb.state = STATE.OPEN;
+  cb.resetTimeout = 60000;
+  cb.failureCount = 5;
+  cb.failureThreshold = 3;
+  cb.lastFailureTime = Date.now();
+
+  const result = await handleComboChat({
+    body: { messages: [{ role: "user", content: "hello" }] },
+    combo: {
+      name: "repro-9630",
+      strategy: "priority",
+      models: ["openai/gpt-4", "anthropic/claude-opus-5"],
+    },
+    handleSingleModel: async (_body: any, modelStr: string) => {
+      assert.equal(modelStr, "anthropic/claude-opus-5", "should skip openai breaker and try anthropic");
+      return okResponse();
+    },
+    isModelAvailable: async () => true,
+    log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.ok(result.ok, "should succeed via anthropic fallback when openai breaker is open");
+});
+
+test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when ALL targets are breaker-open", async () => {
+  const cb = getCircuitBreaker("openai");
+  cb.state = STATE.OPEN;
+  cb.resetTimeout = 60000;
+  cb.failureCount = 5;
+  cb.failureThreshold = 3;
+  cb.lastFailureTime = Date.now();
+
+  const cb2 = getCircuitBreaker("anthropic");
+  cb2.state = STATE.OPEN;
+  cb2.resetTimeout = 60000;
+  cb2.failureCount = 5;
+  cb2.failureThreshold = 3;
+  cb2.lastFailureTime = Date.now();
+
+  const result = await handleComboChat({
+    body: { messages: [{ role: "user", content: "hello" }] },
+    combo: {
+      name: "repro-9630-all-breaker",
+      strategy: "priority",
+      models: ["openai/gpt-4", "anthropic/claude-opus-5"],
+    },
+    handleSingleModel: async () => { throw new Error("should not be called"); },
+    isModelAvailable: async () => true,
+    log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} } as any,
+    settings: null,
+    relayOptions: null as any,
+    allCombos: null,
+  });
+
+  assert.equal(result.status, 503);
+  const body = await result.json();
+  // The diagnostic should NOT claim ALL_ACCOUNTS_INACTIVE when no real dispatch was attempted
+  assert.notEqual(body.error?.code, "ALL_ACCOUNTS_INACTIVE", 
+    "should not claim ALL_ACCOUNTS_INACTIVE when all targets were gated by pre-dispatch checks");
+});