fix(resilience): count resolved 5xx results against the provider breaker (#12360)

CircuitBreaker.execute() treated every resolved promise as a success, but handleChatCore() reports most upstream failures by resolving with { success: false, status: 5xx }. On the chat path that spurious _onSuccess() decayed failureCount right before the call site's _onFailure() for the same attempt, so a provider answering 503s indefinitely stayed CLOSED at failureCount: 1 and kept receiving traffic — the breaker was structurally unable to open. Combo dispatches hit the same cancellation through the shared per-provider breaker.

execute() now takes an optional per-call classifyResult; without it the resolved-means-success contract every throw-based caller relies on is unchanged. executeChatWithBreaker() passes ignore and the chat path accounts for the outcome exactly once where the request context lives, so a combo success is no longer counted twice.

Validated in a combined worktree with all 25 PRs of this batch boarded together: typecheck:core clean, 443/443 node-runner tests plus 14/14 vitest across every test file the batch touches, and check-changelog-integrity, check:cycles (418 files), check:provider-consistency (272 REGISTRY entries, 355 canonical providers), check:docs-counts, check:docs-sync (42 locales) and check-file-size all green.

Thanks @pacocartones.

src/shared/utils/circuitBreaker.ts

@@ -150,6 +150,25 @@ interface CircuitBreakerOptions {
   backoffEscalationCount?: number;
 }
 
+/**
+ * How a RESOLVED `execute()` result is accounted (#12254). Callers such as
+ * `handleChatCore()` report most upstream failures by resolving with
+ * `{ success: false, status: 5xx }` instead of throwing, so a breaker that reads every
+ * resolution as a success never trips on that path.
+ */
+export type CircuitBreakerResultOutcome = "success" | "failure" | "ignore";
+
+export interface CircuitBreakerExecuteOptions<T> {
+  /**
+   * Classify a resolved result. Omitted: every resolution is a success (the
+   * throw-based contract every other caller relies on). Return "ignore" when the
+   * call site accounts for the outcome itself with request context the breaker
+   * does not have — the chat path does (`classifyProviderBreakerResult()` in
+   * chat.ts, `recordProviderFailure()`/`recordProviderSuccess()` in combo.ts).
+   */
+  classifyResult?: (result: T) => CircuitBreakerResultOutcome;
+}
+
 export interface TransitionRecord {
   from: string;
   to: string;
@@ -300,7 +319,7 @@ export class CircuitBreaker {
     );
   }
 
-  async execute<T>(fn: () => Promise<T>): Promise<T> {
+  async execute<T>(fn: () => Promise<T>, options?: CircuitBreakerExecuteOptions<T>): Promise<T> {
     this._refreshOpenState();
 
     if (this.state === STATE.OPEN) {
@@ -325,7 +344,7 @@ export class CircuitBreaker {
 
     try {
       const result = await fn();
-      this._onSuccess();
+      this._recordResolvedResult(result, options?.classifyResult);
       return result;
     } catch (error) {
       if (this.isFailure(error)) {
@@ -387,6 +406,29 @@ export class CircuitBreaker {
 
   // ─── Internal ─────────────────────────────────
 
+  /**
+   * Account a resolved `execute()` result exactly once. A classifier that throws
+   * falls back to the legacy "resolved = success" reading, mirroring `classifyError`.
+   */
+  _recordResolvedResult<T>(
+    result: T,
+    classifyResult?: (result: T) => CircuitBreakerResultOutcome
+  ): void {
+    let outcome: CircuitBreakerResultOutcome = "success";
+    if (classifyResult) {
+      try {
+        outcome = classifyResult(result);
+      } catch {
+        outcome = "success";
+      }
+    }
+    if (outcome === "failure") {
+      this._onFailure();
+    } else if (outcome === "success") {
+      this._onSuccess();
+    }
+  }
+
   _onSuccess() {
     if (this.state === STATE.OPEN) {
       this._transition(STATE.CLOSED, "success-recovery");

src/sse/handlers/chat.ts

@@ -105,10 +105,10 @@ import {
 import { buildModalityBridgeHeader } from "@/lib/guardrails/modalityBridge/bridgeStats";
 import { resolveConversationId } from "@omniroute/open-sse/services/conversationTracker.ts";
 import {
+  classifyProviderBreakerResult,
   isAntigravityMissingProjectError,
   isProviderBreakerFailureStatus,
   resolveStreamReadinessClassificationError,
-  shouldTripProviderBreakerForResult,
 } from "./chatPredicates";
 import { markAntigravityMissingCloudCodeProject } from "@omniroute/open-sse/services/antigravityProjectPersistence.ts";
 import { connectionHasExtraKeys } from "@omniroute/open-sse/services/apiKeyRotator.ts";
@@ -1923,7 +1923,9 @@ async function handleSingleModelChat(
 
       if (result.success) {
         clearModelLock(provider, credentials.connectionId, model);
-        if (!forceLiveComboTest) {
+        // #12254: exactly-once breaker accounting — combo successes are recorded by
+        // combo.ts (recordProviderSuccess); live combo tests never touch the breaker.
+        if (classifyProviderBreakerResult(result, isCombo, forceLiveComboTest) === "success") {
           breaker._onSuccess();
         }
         if (injectedHandoff && runtimeOptions.sessionId && comboName) {
@@ -2370,7 +2372,7 @@ async function handleSingleModelChat(
       // breaker for real traffic (#9817).
       if (
         !(await shouldIsolateProbeFailures()) &&
-        shouldTripProviderBreakerForResult(result, isCombo, forceLiveComboTest)
+        classifyProviderBreakerResult(result, isCombo, forceLiveComboTest) === "failure"
       ) {
         breaker._onFailure();
       }

src/sse/handlers/chatHelpers.ts

@@ -399,6 +399,13 @@ export function checkResourcePressureBeforeProviderWork(): ResourcePressureGuard
   }
 }
 
+// #12254: handleChatCore resolves `{ success: false, status: 5xx }` for most upstream
+// failures, so execute() must not read a resolution as a success (it used to, and that
+// spurious _onSuccess() cancelled the call site's _onFailure() for the same attempt).
+// The chat path accounts for the outcome exactly once where the request context lives:
+// chat.ts via classifyProviderBreakerResult(), combo.ts via recordProviderFailure/Success.
+const chatPathOwnsBreakerAccounting = () => "ignore" as const;
+
 export async function executeChatWithBreaker({
   bypassCircuitBreaker,
   breaker,
@@ -592,13 +599,16 @@ export async function executeChatWithBreaker({
     }
 
     if (tlsFingerprintActive) {
-      const tracked = await breaker.execute(async () =>
-        runWithTlsTracking(tlsTrackingIdentity, chatFn)
+      const tracked = await breaker.execute(
+        async () => runWithTlsTracking(tlsTrackingIdentity, chatFn),
+        { classifyResult: chatPathOwnsBreakerAccounting }
       );
       return { result: tracked.result, tlsFingerprintUsed: tracked.tlsFingerprintUsed };
     }
 
-    const result = await breaker.execute(chatFn);
+    const result = await breaker.execute(chatFn, {
+      classifyResult: chatPathOwnsBreakerAccounting,
+    });
     return { result, tlsFingerprintUsed: false };
   } catch (cbErr: any) {
     if (cbErr instanceof CircuitBreakerOpenError) {

src/sse/handlers/chatPredicates.ts

@@ -43,6 +43,38 @@ export function shouldTripProviderBreakerForResult(
   );
 }
 
+export type ProviderBreakerResultOutcome = "success" | "failure" | "ignore";
+
+/**
+ * #12254: single source of truth for how a resolved dispatch result is accounted
+ * against the per-provider breaker. `handleChatCore()` resolves with
+ * `{ success: false, status: 5xx }` for most upstream failures, so `breaker.execute()`
+ * cannot classify it — the call site does, exactly once:
+ * - combo dispatches and live combo tests are "ignore": the combo target loop owns the
+ *   accounting (`recordProviderFailure()` / `recordProviderSuccess()`), which also knows
+ *   about same-provider-next and `skipProviderBreaker`;
+ * - a successful single-model dispatch is a "success";
+ * - a failed one is a "failure" only when `shouldTripProviderBreakerForResult()` agrees.
+ */
+export function classifyProviderBreakerResult(
+  result: {
+    success?: boolean;
+    status: number;
+    response?: Response;
+    errorCode?: string | null;
+    errorType?: string | null;
+    error?: unknown;
+  },
+  isCombo: boolean,
+  forceLiveComboTest: boolean
+): ProviderBreakerResultOutcome {
+  if (forceLiveComboTest || isCombo) return "ignore";
+  if (result.success) return "success";
+  return shouldTripProviderBreakerForResult(result, isCombo, forceLiveComboTest)
+    ? "failure"
+    : "ignore";
+}
+
 export function isAntigravityMissingProjectError(
   provider: string,
   result: { status?: number; errorCode?: string; errorType?: string }