From d36d077a4d87ce48156278bb4682aa9f29349268 Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Thu, 3 Sep 2026 22:45:47 -0400 Subject: [PATCH] fix(resilience): keep Overloaded STREAM_EARLY_EOF off the provider breaker (#12626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validado em lote numa worktree combinada com os 4 PRs desta leva sobre o tip de `release/v3.8.51`: `typecheck:core` limpo e **119/119** nos 9 arquivos de teste que trazem. Três dos quatro conflitavam apenas no `config/quality/file-size-baseline.json`, todos de forma aditiva (chaves `_rebaseline_` distintas que devem coexistir); resolvidos com validação de JSON a cada passo. Registro que o **#12637 não é duplicata do #12566**, apesar do título quase idêntico: o autor documenta que aquele escopou o cooldown de preflight por família e este cobre o `genericQuotaFetcher`, que é o que o roteamento reset-aware efetivamente chama. Traz também validação ao vivo em VPS (imagem X500, `onmi-gemini3.6` → HTTP 200), satisfazendo a Hard Rule #18. Obrigado, @HouMinXi. --- config/quality/file-size-baseline.json | 5 +- open-sse/services/combo.ts | 41 ++++- open-sse/services/combo/comboCooldownRetry.ts | 34 ++++ open-sse/services/combo/comboPredicates.ts | 17 +- src/shared/utils/circuitBreaker.ts | 24 +++ src/sse/handlers/chatPredicates.ts | 3 + stryker.conf.json | 1 + .../unit/breaker-network-error-guard.test.ts | 2 +- tests/unit/combo-cooldown-retry.test.ts | 86 ++++++++- ...dex-turn-pin-model-scoped-fallback.test.ts | 1 + .../overloaded-not-provider-breaker.test.ts | 169 ++++++++++++++++++ tests/unit/repro-9630-combo-false-503.test.ts | 6 +- 12 files changed, 377 insertions(+), 12 deletions(-) create mode 100644 tests/unit/overloaded-not-provider-breaker.test.ts diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index cc292a2f3b..a7941e072e 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,4 +1,5 @@ { + "_rebaseline_2026_09_03_overloaded_not_provider_breaker": "fix/overloaded-not-provider-breaker own growth: open-sse/services/combo.ts 4036->4075 (check-file-size split-newline, +39). Circuit-open pre-skip now records the breaker retryAfter and, when every target was skipped that way, waits the short reset via resolveCircuitOpenWaitDecision (new leaf in comboCooldownRetry.ts) instead of crystallizing ALL_TARGETS_SKIPPED in ~43ms. skippedForCircuitOpen / earliestCircuitOpenRetryMs reset each setTry so a later iteration cannot inherit a stale retryAfter. Irreducible at the existing ALL_TARGETS_SKIPPED chokepoint (same pattern as #7301/#8213 cooldown-wait). Predicate itself lives in circuitBreaker.ts / comboPredicates.ts / chatPredicates.ts, all under cap. Covered by tests/unit/overloaded-not-provider-breaker.test.ts + combo-cooldown-retry.test.ts.", "_rebaseline_2026_09_03_12649_free_tier_reaudit_gateways": "PR #12649 (fix/free-tier-quota-reaudit) own growth: src/shared/constants/providers/apikey/gateways.ts 1459->1462 (+3 = the nara authHint rewritten for the re-audited 7M/day plan now wraps to two lines, plus the Prettier reflow of two pre-existing >100-col authHint lines (oneminai, freebuff) that lint-staged enforces on any touch of the file; additive text at the existing registry chokepoint, same god-file no-split rationale as prior gateways.ts rebaselines: #11786 seekai, #10987 logfare, #10531 freebuff). Covered by tests/unit/free-tier-reaudit-2026-09.test.ts and tests/unit/free-providers-batch-2026-07.test.ts.", "_rebaseline_2026_09_03_moonshot_native_quota": "PR feat/moonshot-native-quota own growth on release/v3.8.51: src/lib/db/migrationRunner.ts 1201->1206 (+5, case 172 retroactive guard for daily_quota_reset_* columns); src/sse/handlers/chat.ts 2434->2450 (+16, registerMoonshotQuotaFetcher + startup node scan at the existing quota-fetcher registration chokepoint); src/sse/services/auth.ts 3427->3450 (+23, resolveDailyResetForProvider + dailyReset arg on checkFallbackError); open-sse/services/accountFallback.ts 2422->2461 (+39, compatible-node credits_exhausted carve-out + TPD node-clock lock); tests/unit/account-fallback-service.test.ts 2008->2056 (+48, TPD/empty-wallet cases). Wiring at existing chokepoints; Moonshot host predicates, daily reset clock, and the balance fetcher live in new leaves under cap. Covered by tests/unit/moonshot-*.test.ts + account-fallback-service.test.ts (135/135 focused).", "_rebaseline_2026_09_02_11786_seekai_provider": "PR #11786 (feat/11786-seekai-provider, closes #11786) own growth: src/shared/constants/providers/apikey/gateways.ts 1438->1458 (check-file-size split-newline=1459; the seekai APIKEY_PROVIDERS_GATEWAYS catalog entry plus authHint, additive data at the existing registry chokepoint, same god-file no-split rationale as prior gateways.ts rebaselines: #10987 logfare, #10531 freebuff). Covered by tests/unit/seekai-provider.test.ts.", @@ -423,7 +424,7 @@ "open-sse/mcp-server/server.ts": 1572, "open-sse/services/accountFallback.ts": 2467, "open-sse/services/adobeFireflyBrowserLogin.ts": 1401, - "open-sse/services/combo.ts": 4036, + "open-sse/services/combo.ts": 4075, "open-sse/translator/response/openai-responses.ts": 1466, "open-sse/utils/cursorAgentProtobuf.ts": 1547, "open-sse/utils/proxyFetch.ts": 1271, @@ -556,7 +557,7 @@ "open-sse/services/accountFallback.ts": "1978", "open-sse/services/adobeFireflyClient.ts": "2385", "open-sse/services/claudeCodeCompatible.ts": "1202", - "open-sse/services/combo.ts": "3648", + "open-sse/services/combo.ts": "4075", "open-sse/services/compression/strategySelector.ts": "1060", "open-sse/services/rateLimitManager.ts": "1167", "open-sse/translator/response/openai-responses.ts": "1204", diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index be662c600f..635001b646 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -178,6 +178,7 @@ import { } from "./combo/validateQuality.ts"; import { resolveComboCooldownWaitDecision, + resolveCircuitOpenWaitDecision, ResolveComboCooldownDecisionResult, } from "./combo/comboCooldownRetry.ts"; import { @@ -1133,6 +1134,8 @@ async function handleComboChatInner({ let lastError: string | null = null; let earliestRetryAfter: ComboRetryAfter | null = null; let lastStatus: number | null = null; + let skippedForCircuitOpen = false; + let earliestCircuitOpenRetryMs = 0; // #11804: the loop-safety timer is armed per setTry iteration but must be // cleared on EVERY exit path, not just the happy one. Hoisted to function // scope so the `finally` at the end of this function always reaches it — @@ -1151,6 +1154,8 @@ async function handleComboChatInner({ const exhaustedProviders = new Set(); const exhaustedConnections = new Set(); const transientRateLimitedProviders = new Set(); + skippedForCircuitOpen = false; + earliestCircuitOpenRetryMs = 0; if (setTry > 0) { log.info("COMBO", `All targets failed — retrying set (${setTry}/${maxSetRetries})`); await new Promise((resolve) => { @@ -1272,7 +1277,15 @@ async function handleComboChatInner({ }; const cb = getCircuitBreaker(provider); - if (cb.getStatus().state === "OPEN") { + const cbStatus = cb.getStatus(); + if (cbStatus.state === "OPEN") { + skippedForCircuitOpen = true; + if ( + cbStatus.retryAfterMs > 0 && + (earliestCircuitOpenRetryMs === 0 || cbStatus.retryAfterMs < earliestCircuitOpenRetryMs) + ) { + earliestCircuitOpenRetryMs = cbStatus.retryAfterMs; + } log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`); recordComboDecision(traceInvocationId, { step: target.executionKey, @@ -2762,6 +2775,32 @@ async function handleComboChatInner({ // Retry the entire set if more attempts remain if (setTry < maxSetRetries) continue; + if (!lastStatus && recordedAttempts === 0 && comboCooldownWaitEnabled) { + const circuitOpenWait = resolveCircuitOpenWaitDecision({ + skippedForCircuitOpen, + retryAfterMs: earliestCircuitOpenRetryMs, + attempt: comboCooldownAttempt, + budgetLeftMs: comboCooldownBudgetLeftMs, + settings: resilienceSettings.comboCooldownWait, + }); + if (circuitOpenWait.wait) { + log.info( + "COMBO", + `${strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${comboCooldownAttempt + 1}/${resilienceSettings.comboCooldownWait.maxAttempts})` + ); + const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, signal); + if (!completed) { + return errorResponse(499, "Request aborted"); + } + comboCooldownAttempt += 1; + comboCooldownBudgetLeftMs = Math.max( + 0, + comboCooldownBudgetLeftMs - circuitOpenWait.waitMs + ); + return dispatchWithCooldownRetry(); + } + } + // All set retries exhausted — return the final error // #10681: finalize the decision trace (all targets failed or skipped). finalizeComboTrace(traceInvocationId, orderedTargets); diff --git a/open-sse/services/combo/comboCooldownRetry.ts b/open-sse/services/combo/comboCooldownRetry.ts index 4b2b56609a..4b96552b72 100644 --- a/open-sse/services/combo/comboCooldownRetry.ts +++ b/open-sse/services/combo/comboCooldownRetry.ts @@ -56,6 +56,7 @@ export const COMBO_COOLDOWN_RETRYABLE_REASONS: ReadonlySet = new Set([ "transient", "overloaded", "server_error", + "circuit_open", ]); export interface ComboCooldownWaitSettings { @@ -256,3 +257,36 @@ export function resolveComboCooldownWaitDecision( reason: typeof best.reason === "string" ? best.reason : null, }; } + +export interface ResolveCircuitOpenWaitInput { + skippedForCircuitOpen: unknown; + retryAfterMs: unknown; + attempt: number; + budgetLeftMs: number; + settings: ComboCooldownWaitSettings; +} + +/** + * When every combo target was pre-skipped because the whole-provider breaker is + * OPEN, wait out a SHORT reset instead of crystallizing ALL_TARGETS_SKIPPED. + * Same ceilings as model-lockout waits. Live incident 2026-09-03: offical-fable + * (single claude target) returned 43ms 503 while the breaker reset was 60s. + */ +export function resolveCircuitOpenWaitDecision( + input: ResolveCircuitOpenWaitInput +): ResolveComboCooldownDecisionResult { + if (input.settings.enabled !== true || input.skippedForCircuitOpen !== true) { + return { wait: false, waitMs: 0, reason: null }; + } + const retryAfterMs = toFiniteWaitMs(input.retryAfterMs); + if (retryAfterMs <= 0) return { wait: false, waitMs: 0, reason: null }; + const waitMs = retryAfterMs + COMBO_COOLDOWN_WAIT_MARGIN_MS; + const decision = shouldWaitForComboCooldown({ + reason: "circuit_open", + waitMs, + attempt: input.attempt, + budgetLeftMs: input.budgetLeftMs, + settings: input.settings, + }); + return { ...decision, reason: "circuit_open" }; +} diff --git a/open-sse/services/combo/comboPredicates.ts b/open-sse/services/combo/comboPredicates.ts index 972abc5765..6359607cac 100644 --- a/open-sse/services/combo/comboPredicates.ts +++ b/open-sse/services/combo/comboPredicates.ts @@ -11,7 +11,11 @@ import { remainingPercentFromQuotaWindows } from "../antigravityQuotaFamily.ts"; import { errorResponse } from "../../utils/error.ts"; import { parseModel } from "../model.ts"; import { isSelfInflictedUpstreamTimeout } from "../../handlers/chatCore/cooldownClassification.ts"; -import { isLocalStreamLifecycleError, isLocalExecutionError } from "@/shared/utils/circuitBreaker"; +import { + isLocalStreamLifecycleError, + isLocalExecutionError, + isModelCapacityOverloadError, +} from "@/shared/utils/circuitBreaker"; import { CONTEXT_OVERFLOW_PATTERNS, MODEL_ACCESS_DENIED_PATTERNS } from "../accountFallback.ts"; import { isResourceNotFoundResponse } from "../errorClassifier.ts"; import { getTrustedLocalRateLimitResponse } from "../rateLimitManager/errors.ts"; @@ -213,6 +217,12 @@ export function shouldRecordProviderBreakerFailure(args: { }): boolean { return ( (!args.isStreamReadinessFailure || args.isStreamEarlyEof === true) && + // Overloaded 502 (STREAM_EARLY_EOF wrapping "Overloaded") must not trip + // the whole-provider breaker. The status=529 check is defense in depth: + // 529 is not in PROVIDER_BREAKER_FAILURE_STATUSES today, but a later + // addition of 529 to that set must still stay off the breaker. + !isModelCapacityOverloadError(args.error) && + !isModelCapacityOverloadError(args.status) && PROVIDER_BREAKER_FAILURE_STATUSES.has(args.status) && (!args.sameProviderNext || args.isProxyUnreachable === true) && !args.skipProviderBreaker && @@ -441,10 +451,7 @@ export function quotaRemainingPercentFromQuota( const windows = record.windows; if (windows && typeof windows === "object" && !Array.isArray(windows)) { - const fromWindows = remainingPercentFromQuotaWindows( - windows as Record, - scope - ); + const fromWindows = remainingPercentFromQuotaWindows(windows as Record, scope); if (fromWindows !== null) return fromWindows; } diff --git a/src/shared/utils/circuitBreaker.ts b/src/shared/utils/circuitBreaker.ts index 02e4e67811..c84917f0cd 100644 --- a/src/shared/utils/circuitBreaker.ts +++ b/src/shared/utils/circuitBreaker.ts @@ -102,6 +102,30 @@ export function isLocalExecutionError(error: unknown): boolean { return LOCAL_EXECUTION_PATTERNS.some((p) => p.test(message)); } +/** + * Anthropic/Claude model-capacity overload (HTTP 529, body "Overloaded", or a + * STREAM_EARLY_EOF that wraps that body as 502). This is one model being + * capacity-throttled, not a whole-provider outage — the same account still + * serves sibling models. Must not trip the provider circuit breaker. + * + * Accepts an error object/string OR a numeric HTTP status (529). Callers + * pass both `error` and `status` at the two breaker predicates. + * + * Live incident 2026-09-03: STREAM_EARLY_EOF: Overloaded opened `claude` and + * a single-target combo then pre-skipped with ALL_TARGETS_SKIPPED in ~43ms. + */ +export function isModelCapacityOverloadError(error: unknown): boolean { + if (error === 529) return true; + if (typeof error === "number") return false; + if (!error) return false; + const errObj = typeof error === "object" ? (error as Record) : null; + if (errObj && (errObj.status === 529 || errObj.statusCode === 529)) return true; + const message = + typeof error === "string" ? error : typeof errObj?.message === "string" ? errObj.message : ""; + if (!message) return false; + return /\boverloaded(?:_error)?\b/i.test(message); +} + export const STATE = { CLOSED: "CLOSED", DEGRADED: "DEGRADED", diff --git a/src/sse/handlers/chatPredicates.ts b/src/sse/handlers/chatPredicates.ts index f1ccbbaab2..88911a41b5 100644 --- a/src/sse/handlers/chatPredicates.ts +++ b/src/sse/handlers/chatPredicates.ts @@ -1,6 +1,7 @@ import { isLocalStreamLifecycleError, isLocalExecutionError, + isModelCapacityOverloadError, } from "../../shared/utils/circuitBreaker"; import { isRequestScopedUpstreamFailure } from "./comboFailureLogging"; import { getTrustedLocalRateLimitResponse } from "@omniroute/open-sse/services/rateLimitManager/errors"; @@ -39,6 +40,8 @@ export function shouldTripProviderBreakerForResult( result.errorCode !== "proxy_unreachable" && result.errorCode !== "RATE_LIMIT_QUEUE_TIMEOUT" && result.errorCode !== "RATE_LIMIT_QUEUE_WEDGED" && + !isModelCapacityOverloadError(result.error) && + !isModelCapacityOverloadError(result.status) && PROVIDER_BREAKER_FAILURE_STATUSES.has(Number(result.status)) ); } diff --git a/stryker.conf.json b/stryker.conf.json index c2986bcb28..c8ff964e77 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -152,6 +152,7 @@ "tests/unit/circuit-breaker-registry-cap.test.ts", "tests/unit/circuit-breaker-resolved-5xx-12254.test.ts", "tests/unit/circuit-breaker-stream-controller-4602.test.ts", + "tests/unit/overloaded-not-provider-breaker.test.ts", "tests/unit/claude-code-parity.test.ts", "tests/unit/claude-effort-suffix-strip.test.ts", "tests/unit/claude-oauth-provider.test.ts", diff --git a/tests/unit/breaker-network-error-guard.test.ts b/tests/unit/breaker-network-error-guard.test.ts index 8d53edc080..35e1773b0a 100644 --- a/tests/unit/breaker-network-error-guard.test.ts +++ b/tests/unit/breaker-network-error-guard.test.ts @@ -78,7 +78,7 @@ test("forceLiveComboTest=true prevents breaker trip (combo will try next target) // `{ success: false, status: 5xx }` as a success. test("classifyProviderBreakerResult: a resolved 503 on the single-model path is a failure", () => { const outcome = classifyProviderBreakerResult( - { success: false, status: 503, errorCode: null, errorType: null, error: "overloaded" }, + { success: false, status: 503, errorCode: null, errorType: null, error: "service unavailable" }, false, false ); diff --git a/tests/unit/combo-cooldown-retry.test.ts b/tests/unit/combo-cooldown-retry.test.ts index 91d0339b1b..cc2c76699a 100644 --- a/tests/unit/combo-cooldown-retry.test.ts +++ b/tests/unit/combo-cooldown-retry.test.ts @@ -22,8 +22,12 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { shouldWaitForComboCooldown, resolveComboCooldownWaitDecision, COMBO_COOLDOWN_WAIT_MARGIN_MS } = - await import("../../open-sse/services/combo/comboCooldownRetry.ts"); +const { + shouldWaitForComboCooldown, + resolveComboCooldownWaitDecision, + resolveCircuitOpenWaitDecision, + COMBO_COOLDOWN_WAIT_MARGIN_MS, +} = await import("../../open-sse/services/combo/comboCooldownRetry.ts"); function baseSettings(overrides: Partial> = {}) { return { @@ -77,6 +81,11 @@ test("missing/unknown reason (null) → no wait (only an explicit transient reas assert.equal(r.wait, false); }); +test("reason circuit_open is eligible (whole-provider breaker OPEN is a short reset)", () => { + const r = shouldWaitForComboCooldown(baseInput({ reason: "circuit_open" }) as never); + assert.equal(r.wait, true); +}); + test("waitMs above the configured ceiling → no wait", () => { const r = shouldWaitForComboCooldown( baseInput({ waitMs: 5001, settings: baseSettings({ maxWaitMs: 5000 }) }) as never @@ -146,6 +155,7 @@ test("returned waitMs is clamped to a finite number (0 when input invalid)", () // ── resolveComboCooldownWaitDecision (target resolution + hint/fallback) ────── const M = COMBO_COOLDOWN_WAIT_MARGIN_MS; +assert.equal(M, 50, "wait margin is a pinned production constant, not a free parameter"); function decisionInput(overrides: Record = {}) { return { @@ -269,3 +279,75 @@ test("resolve: lock remaining above the ceiling → no wait (not a SHORT cooldow ); assert.equal(r.wait, false); }); + +// Live incident 2026-09-03: a single-target combo pre-skipped every target because +// the whole-provider breaker was OPEN (claude Overloaded STREAM_EARLY_EOF). That +// path crystallized ALL_TARGETS_SKIPPED in ~43ms and never entered the cooldown +// wait, even though the breaker resetTimeout is 60s and comboCooldownWait was on. + +function circuitOpenInput(overrides: Record = {}) { + return { + skippedForCircuitOpen: true, + retryAfterMs: 30_000, + attempt: 0, + budgetLeftMs: 90_000, + settings: baseSettings({ maxWaitMs: 90_000, budgetMs: 300_000, maxAttempts: 5 }), + ...overrides, + }; +} + +test("circuit-open skip with a short breaker reset → wait", () => { + const r = resolveCircuitOpenWaitDecision(circuitOpenInput() as never); + assert.equal(r.wait, true); + assert.equal(r.waitMs, 30_000 + M); + assert.equal(r.reason, "circuit_open"); +}); + +test("circuit-open skip is ignored when no target was skipped for circuit_open", () => { + const r = resolveCircuitOpenWaitDecision( + circuitOpenInput({ skippedForCircuitOpen: false }) as never + ); + assert.equal(r.wait, false); + assert.equal(r.reason, null); +}); + +test("circuit-open skip with zero retryAfter → no wait", () => { + const r = resolveCircuitOpenWaitDecision(circuitOpenInput({ retryAfterMs: 0 }) as never); + assert.equal(r.wait, false); +}); + +test("circuit-open skip above the wait ceiling → no wait", () => { + const r = resolveCircuitOpenWaitDecision( + circuitOpenInput({ + retryAfterMs: 120_000, + settings: baseSettings({ maxWaitMs: 90_000, budgetMs: 300_000, maxAttempts: 5 }), + }) as never + ); + assert.equal(r.wait, false); +}); + +test("circuit-open skip honors attempt/budget the same as model-lockout waits", () => { + assert.equal( + resolveCircuitOpenWaitDecision(circuitOpenInput({ attempt: 5 }) as never).wait, + false + ); + assert.equal( + resolveCircuitOpenWaitDecision(circuitOpenInput({ budgetLeftMs: 1_000 }) as never).wait, + false + ); +}); + +test("circuit-open skip is off when comboCooldownWait.enabled is false", () => { + const r = resolveCircuitOpenWaitDecision( + circuitOpenInput({ + settings: baseSettings({ + enabled: false, + maxWaitMs: 90_000, + budgetMs: 300_000, + maxAttempts: 5, + }), + }) as never + ); + assert.equal(r.wait, false); + assert.equal(r.reason, null); +}); diff --git a/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts index 595161d64e..d37fb55cb8 100644 --- a/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts +++ b/tests/unit/native-codex-turn-pin-model-scoped-fallback.test.ts @@ -29,6 +29,7 @@ const providersDb = await import("../../src/lib/db/providers.ts"); const testSettings = { resilienceSettings: { providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 }, + comboCooldownWait: { enabled: false }, }, }; diff --git a/tests/unit/overloaded-not-provider-breaker.test.ts b/tests/unit/overloaded-not-provider-breaker.test.ts new file mode 100644 index 0000000000..0ba022cb94 --- /dev/null +++ b/tests/unit/overloaded-not-provider-breaker.test.ts @@ -0,0 +1,169 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { isModelCapacityOverloadError } from "../../src/shared/utils/circuitBreaker.ts"; +import { + shouldTripProviderBreakerForResult, + classifyProviderBreakerResult, +} from "../../src/sse/handlers/chatPredicates.ts"; +import { shouldRecordProviderBreakerFailure } from "../../open-sse/services/combo/comboPredicates.ts"; + +/** + * Live incident 2026-09-03 (X500 offical-fable): Anthropic returned + * STREAM_EARLY_EOF wrapping "Overloaded" as HTTP 502. That 502 opened the + * whole-provider `claude` breaker. The single-target combo then pre-skipped + * with ALL_TARGETS_SKIPPED in ~43ms even though the account, pin, and quota + * were healthy. Model capacity (529 / Overloaded) is not a provider outage. + */ + +const OTHER_COMBO_ARGS = { + isStreamReadinessFailure: false, + sameProviderNext: false, + skipProviderBreaker: false, + requestScopedFailure: false, +} as const; + +const LIVE_EOF_OVERLOADED = "Stream ended before producing a non-ping SSE event: Overloaded"; +const PLAIN_EOF = "Stream ended before producing a non-ping SSE event"; + +test("isModelCapacityOverloadError: live STREAM_EARLY_EOF Overloaded text", () => { + assert.equal(isModelCapacityOverloadError(LIVE_EOF_OVERLOADED), true); +}); + +test("isModelCapacityOverloadError: bare Overloaded / HTTP 529", () => { + assert.equal(isModelCapacityOverloadError("Overloaded"), true); + assert.equal(isModelCapacityOverloadError("[529]: Overloaded"), true); + assert.equal(isModelCapacityOverloadError(529), true); + assert.equal(isModelCapacityOverloadError({ message: "overloaded_error" }), true); +}); + +test("isModelCapacityOverloadError: a plain early EOF is NOT capacity", () => { + assert.equal(isModelCapacityOverloadError(PLAIN_EOF), false); + assert.equal(isModelCapacityOverloadError("502 Bad Gateway"), false); + assert.equal(isModelCapacityOverloadError(null), false); + assert.equal(isModelCapacityOverloadError(undefined), false); +}); + +test("combo: STREAM_EARLY_EOF Overloaded 502 does NOT record a whole-provider breaker failure", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + ...OTHER_COMBO_ARGS, + isStreamReadinessFailure: true, + isStreamEarlyEof: true, + status: 502, + error: LIVE_EOF_OVERLOADED, + }), + false + ); +}); + +test("combo: a plain STREAM_EARLY_EOF 502 still records a breaker failure", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + ...OTHER_COMBO_ARGS, + isStreamReadinessFailure: true, + isStreamEarlyEof: true, + status: 502, + error: PLAIN_EOF, + }), + true + ); +}); + +test("combo: HTTP 529 Overloaded does not record even if someone later adds 529 to the status set", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + ...OTHER_COMBO_ARGS, + status: 529, + error: "[529]: Overloaded", + }), + false + ); +}); + +test("combo: HTTP 529 status alone does not record a breaker failure", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + ...OTHER_COMBO_ARGS, + status: 529, + error: "upstream error", + }), + false + ); +}); + +test("combo: a genuine 502 without Overloaded still records", () => { + assert.equal( + shouldRecordProviderBreakerFailure({ + ...OTHER_COMBO_ARGS, + status: 502, + error: "upstream error", + }), + true + ); +}); + +test("single-model: STREAM_EARLY_EOF Overloaded 502 does NOT trip the provider breaker", () => { + assert.equal( + shouldTripProviderBreakerForResult( + { + status: 502, + errorCode: "STREAM_EARLY_EOF", + errorType: "stream_early_eof", + error: LIVE_EOF_OVERLOADED, + }, + false, + false + ), + false + ); +}); + +test("single-model: a genuine 502 without Overloaded still trips", () => { + assert.equal( + shouldTripProviderBreakerForResult( + { status: 502, errorCode: null, errorType: null, error: "upstream error" }, + false, + false + ), + true + ); +}); + +test("single-model: HTTP 529 does not trip", () => { + assert.equal( + shouldTripProviderBreakerForResult( + { status: 529, errorCode: null, errorType: null, error: "Overloaded" }, + false, + false + ), + false + ); +}); + +test("classifyProviderBreakerResult: Overloaded 502 on the single-model path is ignore", () => { + assert.equal( + classifyProviderBreakerResult( + { + success: false, + status: 502, + errorCode: "STREAM_EARLY_EOF", + errorType: "stream_early_eof", + error: LIVE_EOF_OVERLOADED, + }, + false, + false + ), + "ignore" + ); +}); + +test("classifyProviderBreakerResult: Overloaded 529 on the single-model path is ignore", () => { + assert.equal( + classifyProviderBreakerResult( + { success: false, status: 529, errorCode: null, errorType: null, error: "Overloaded" }, + false, + false + ), + "ignore" + ); +}); diff --git a/tests/unit/repro-9630-combo-false-503.test.ts b/tests/unit/repro-9630-combo-false-503.test.ts index 16eae79b05..b3fbbc5bdc 100644 --- a/tests/unit/repro-9630-combo-false-503.test.ts +++ b/tests/unit/repro-9630-combo-false-503.test.ts @@ -70,7 +70,11 @@ test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when }, isModelAvailable: async () => true, log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} }, - settings: null, + settings: { + resilienceSettings: { + comboCooldownWait: { enabled: false }, + }, + }, relayOptions: null, allCombos: null, });