mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-21 14:22:14 +03:00
fix(resilience): keep Overloaded STREAM_EARLY_EOF off the provider breaker (#12626)
Validado em lote numa worktree combinada com os 4 PRs desta leva sobre o tip de `release/v3.8.51`: `typecheck:core` limpo e **119/119** nos 9 arquivos de teste que trazem. Três dos quatro conflitavam apenas no `config/quality/file-size-baseline.json`, todos de forma aditiva (chaves `_rebaseline_` distintas que devem coexistir); resolvidos com validação de JSON a cada passo. Registro que o **#12637 não é duplicata do #12566**, apesar do título quase idêntico: o autor documenta que aquele escopou o cooldown de preflight por família e este cobre o `genericQuotaFetcher`, que é o que o roteamento reset-aware efetivamente chama. Traz também validação ao vivo em VPS (imagem X500, `onmi-gemini3.6` → HTTP 200), satisfazendo a Hard Rule #18. Obrigado, @HouMinXi.
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"_rebaseline_2026_09_03_overloaded_not_provider_breaker": "fix/overloaded-not-provider-breaker own growth: open-sse/services/combo.ts 4036->4075 (check-file-size split-newline, +39). Circuit-open pre-skip now records the breaker retryAfter and, when every target was skipped that way, waits the short reset via resolveCircuitOpenWaitDecision (new leaf in comboCooldownRetry.ts) instead of crystallizing ALL_TARGETS_SKIPPED in ~43ms. skippedForCircuitOpen / earliestCircuitOpenRetryMs reset each setTry so a later iteration cannot inherit a stale retryAfter. Irreducible at the existing ALL_TARGETS_SKIPPED chokepoint (same pattern as #7301/#8213 cooldown-wait). Predicate itself lives in circuitBreaker.ts / comboPredicates.ts / chatPredicates.ts, all under cap. Covered by tests/unit/overloaded-not-provider-breaker.test.ts + combo-cooldown-retry.test.ts.",
|
||||
"_rebaseline_2026_09_03_12649_free_tier_reaudit_gateways": "PR #12649 (fix/free-tier-quota-reaudit) own growth: src/shared/constants/providers/apikey/gateways.ts 1459->1462 (+3 = the nara authHint rewritten for the re-audited 7M/day plan now wraps to two lines, plus the Prettier reflow of two pre-existing >100-col authHint lines (oneminai, freebuff) that lint-staged enforces on any touch of the file; additive text at the existing registry chokepoint, same god-file no-split rationale as prior gateways.ts rebaselines: #11786 seekai, #10987 logfare, #10531 freebuff). Covered by tests/unit/free-tier-reaudit-2026-09.test.ts and tests/unit/free-providers-batch-2026-07.test.ts.",
|
||||
"_rebaseline_2026_09_03_moonshot_native_quota": "PR feat/moonshot-native-quota own growth on release/v3.8.51: src/lib/db/migrationRunner.ts 1201->1206 (+5, case 172 retroactive guard for daily_quota_reset_* columns); src/sse/handlers/chat.ts 2434->2450 (+16, registerMoonshotQuotaFetcher + startup node scan at the existing quota-fetcher registration chokepoint); src/sse/services/auth.ts 3427->3450 (+23, resolveDailyResetForProvider + dailyReset arg on checkFallbackError); open-sse/services/accountFallback.ts 2422->2461 (+39, compatible-node credits_exhausted carve-out + TPD node-clock lock); tests/unit/account-fallback-service.test.ts 2008->2056 (+48, TPD/empty-wallet cases). Wiring at existing chokepoints; Moonshot host predicates, daily reset clock, and the balance fetcher live in new leaves under cap. Covered by tests/unit/moonshot-*.test.ts + account-fallback-service.test.ts (135/135 focused).",
|
||||
"_rebaseline_2026_09_02_11786_seekai_provider": "PR #11786 (feat/11786-seekai-provider, closes #11786) own growth: src/shared/constants/providers/apikey/gateways.ts 1438->1458 (check-file-size split-newline=1459; the seekai APIKEY_PROVIDERS_GATEWAYS catalog entry plus authHint, additive data at the existing registry chokepoint, same god-file no-split rationale as prior gateways.ts rebaselines: #10987 logfare, #10531 freebuff). Covered by tests/unit/seekai-provider.test.ts.",
|
||||
@@ -423,7 +424,7 @@
|
||||
"open-sse/mcp-server/server.ts": 1572,
|
||||
"open-sse/services/accountFallback.ts": 2467,
|
||||
"open-sse/services/adobeFireflyBrowserLogin.ts": 1401,
|
||||
"open-sse/services/combo.ts": 4036,
|
||||
"open-sse/services/combo.ts": 4075,
|
||||
"open-sse/translator/response/openai-responses.ts": 1466,
|
||||
"open-sse/utils/cursorAgentProtobuf.ts": 1547,
|
||||
"open-sse/utils/proxyFetch.ts": 1271,
|
||||
@@ -556,7 +557,7 @@
|
||||
"open-sse/services/accountFallback.ts": "1978",
|
||||
"open-sse/services/adobeFireflyClient.ts": "2385",
|
||||
"open-sse/services/claudeCodeCompatible.ts": "1202",
|
||||
"open-sse/services/combo.ts": "3648",
|
||||
"open-sse/services/combo.ts": "4075",
|
||||
"open-sse/services/compression/strategySelector.ts": "1060",
|
||||
"open-sse/services/rateLimitManager.ts": "1167",
|
||||
"open-sse/translator/response/openai-responses.ts": "1204",
|
||||
|
||||
@@ -178,6 +178,7 @@ import {
|
||||
} from "./combo/validateQuality.ts";
|
||||
import {
|
||||
resolveComboCooldownWaitDecision,
|
||||
resolveCircuitOpenWaitDecision,
|
||||
ResolveComboCooldownDecisionResult,
|
||||
} from "./combo/comboCooldownRetry.ts";
|
||||
import {
|
||||
@@ -1133,6 +1134,8 @@ async function handleComboChatInner({
|
||||
let lastError: string | null = null;
|
||||
let earliestRetryAfter: ComboRetryAfter | null = null;
|
||||
let lastStatus: number | null = null;
|
||||
let skippedForCircuitOpen = false;
|
||||
let earliestCircuitOpenRetryMs = 0;
|
||||
// #11804: the loop-safety timer is armed per setTry iteration but must be
|
||||
// cleared on EVERY exit path, not just the happy one. Hoisted to function
|
||||
// scope so the `finally` at the end of this function always reaches it —
|
||||
@@ -1151,6 +1154,8 @@ async function handleComboChatInner({
|
||||
const exhaustedProviders = new Set<string>();
|
||||
const exhaustedConnections = new Set<string>();
|
||||
const transientRateLimitedProviders = new Set<string>();
|
||||
skippedForCircuitOpen = false;
|
||||
earliestCircuitOpenRetryMs = 0;
|
||||
if (setTry > 0) {
|
||||
log.info("COMBO", `All targets failed — retrying set (${setTry}/${maxSetRetries})`);
|
||||
await new Promise((resolve) => {
|
||||
@@ -1272,7 +1277,15 @@ async function handleComboChatInner({
|
||||
};
|
||||
|
||||
const cb = getCircuitBreaker(provider);
|
||||
if (cb.getStatus().state === "OPEN") {
|
||||
const cbStatus = cb.getStatus();
|
||||
if (cbStatus.state === "OPEN") {
|
||||
skippedForCircuitOpen = true;
|
||||
if (
|
||||
cbStatus.retryAfterMs > 0 &&
|
||||
(earliestCircuitOpenRetryMs === 0 || cbStatus.retryAfterMs < earliestCircuitOpenRetryMs)
|
||||
) {
|
||||
earliestCircuitOpenRetryMs = cbStatus.retryAfterMs;
|
||||
}
|
||||
log.info("COMBO", `Skipping ${modelStr} — circuit breaker OPEN for ${provider}`);
|
||||
recordComboDecision(traceInvocationId, {
|
||||
step: target.executionKey,
|
||||
@@ -2762,6 +2775,32 @@ async function handleComboChatInner({
|
||||
// Retry the entire set if more attempts remain
|
||||
if (setTry < maxSetRetries) continue;
|
||||
|
||||
if (!lastStatus && recordedAttempts === 0 && comboCooldownWaitEnabled) {
|
||||
const circuitOpenWait = resolveCircuitOpenWaitDecision({
|
||||
skippedForCircuitOpen,
|
||||
retryAfterMs: earliestCircuitOpenRetryMs,
|
||||
attempt: comboCooldownAttempt,
|
||||
budgetLeftMs: comboCooldownBudgetLeftMs,
|
||||
settings: resilienceSettings.comboCooldownWait,
|
||||
});
|
||||
if (circuitOpenWait.wait) {
|
||||
log.info(
|
||||
"COMBO",
|
||||
`${strategy} circuit-open wait: waiting ${Math.ceil(circuitOpenWait.waitMs / 1000)}s (reason=${circuitOpenWait.reason ?? "circuit_open"}) then retrying (attempt ${comboCooldownAttempt + 1}/${resilienceSettings.comboCooldownWait.maxAttempts})`
|
||||
);
|
||||
const completed = await waitForCooldownAwareRetry(circuitOpenWait.waitMs, signal);
|
||||
if (!completed) {
|
||||
return errorResponse(499, "Request aborted");
|
||||
}
|
||||
comboCooldownAttempt += 1;
|
||||
comboCooldownBudgetLeftMs = Math.max(
|
||||
0,
|
||||
comboCooldownBudgetLeftMs - circuitOpenWait.waitMs
|
||||
);
|
||||
return dispatchWithCooldownRetry();
|
||||
}
|
||||
}
|
||||
|
||||
// All set retries exhausted — return the final error
|
||||
// #10681: finalize the decision trace (all targets failed or skipped).
|
||||
finalizeComboTrace(traceInvocationId, orderedTargets);
|
||||
|
||||
@@ -56,6 +56,7 @@ export const COMBO_COOLDOWN_RETRYABLE_REASONS: ReadonlySet<string> = new Set([
|
||||
"transient",
|
||||
"overloaded",
|
||||
"server_error",
|
||||
"circuit_open",
|
||||
]);
|
||||
|
||||
export interface ComboCooldownWaitSettings {
|
||||
@@ -256,3 +257,36 @@ export function resolveComboCooldownWaitDecision(
|
||||
reason: typeof best.reason === "string" ? best.reason : null,
|
||||
};
|
||||
}
|
||||
|
||||
export interface ResolveCircuitOpenWaitInput {
|
||||
skippedForCircuitOpen: unknown;
|
||||
retryAfterMs: unknown;
|
||||
attempt: number;
|
||||
budgetLeftMs: number;
|
||||
settings: ComboCooldownWaitSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* When every combo target was pre-skipped because the whole-provider breaker is
|
||||
* OPEN, wait out a SHORT reset instead of crystallizing ALL_TARGETS_SKIPPED.
|
||||
* Same ceilings as model-lockout waits. Live incident 2026-09-03: offical-fable
|
||||
* (single claude target) returned 43ms 503 while the breaker reset was 60s.
|
||||
*/
|
||||
export function resolveCircuitOpenWaitDecision(
|
||||
input: ResolveCircuitOpenWaitInput
|
||||
): ResolveComboCooldownDecisionResult {
|
||||
if (input.settings.enabled !== true || input.skippedForCircuitOpen !== true) {
|
||||
return { wait: false, waitMs: 0, reason: null };
|
||||
}
|
||||
const retryAfterMs = toFiniteWaitMs(input.retryAfterMs);
|
||||
if (retryAfterMs <= 0) return { wait: false, waitMs: 0, reason: null };
|
||||
const waitMs = retryAfterMs + COMBO_COOLDOWN_WAIT_MARGIN_MS;
|
||||
const decision = shouldWaitForComboCooldown({
|
||||
reason: "circuit_open",
|
||||
waitMs,
|
||||
attempt: input.attempt,
|
||||
budgetLeftMs: input.budgetLeftMs,
|
||||
settings: input.settings,
|
||||
});
|
||||
return { ...decision, reason: "circuit_open" };
|
||||
}
|
||||
|
||||
@@ -11,7 +11,11 @@ import { remainingPercentFromQuotaWindows } from "../antigravityQuotaFamily.ts";
|
||||
import { errorResponse } from "../../utils/error.ts";
|
||||
import { parseModel } from "../model.ts";
|
||||
import { isSelfInflictedUpstreamTimeout } from "../../handlers/chatCore/cooldownClassification.ts";
|
||||
import { isLocalStreamLifecycleError, isLocalExecutionError } from "@/shared/utils/circuitBreaker";
|
||||
import {
|
||||
isLocalStreamLifecycleError,
|
||||
isLocalExecutionError,
|
||||
isModelCapacityOverloadError,
|
||||
} from "@/shared/utils/circuitBreaker";
|
||||
import { CONTEXT_OVERFLOW_PATTERNS, MODEL_ACCESS_DENIED_PATTERNS } from "../accountFallback.ts";
|
||||
import { isResourceNotFoundResponse } from "../errorClassifier.ts";
|
||||
import { getTrustedLocalRateLimitResponse } from "../rateLimitManager/errors.ts";
|
||||
@@ -213,6 +217,12 @@ export function shouldRecordProviderBreakerFailure(args: {
|
||||
}): boolean {
|
||||
return (
|
||||
(!args.isStreamReadinessFailure || args.isStreamEarlyEof === true) &&
|
||||
// Overloaded 502 (STREAM_EARLY_EOF wrapping "Overloaded") must not trip
|
||||
// the whole-provider breaker. The status=529 check is defense in depth:
|
||||
// 529 is not in PROVIDER_BREAKER_FAILURE_STATUSES today, but a later
|
||||
// addition of 529 to that set must still stay off the breaker.
|
||||
!isModelCapacityOverloadError(args.error) &&
|
||||
!isModelCapacityOverloadError(args.status) &&
|
||||
PROVIDER_BREAKER_FAILURE_STATUSES.has(args.status) &&
|
||||
(!args.sameProviderNext || args.isProxyUnreachable === true) &&
|
||||
!args.skipProviderBreaker &&
|
||||
@@ -441,10 +451,7 @@ export function quotaRemainingPercentFromQuota(
|
||||
|
||||
const windows = record.windows;
|
||||
if (windows && typeof windows === "object" && !Array.isArray(windows)) {
|
||||
const fromWindows = remainingPercentFromQuotaWindows(
|
||||
windows as Record<string, unknown>,
|
||||
scope
|
||||
);
|
||||
const fromWindows = remainingPercentFromQuotaWindows(windows as Record<string, unknown>, scope);
|
||||
if (fromWindows !== null) return fromWindows;
|
||||
}
|
||||
|
||||
|
||||
@@ -102,6 +102,30 @@ export function isLocalExecutionError(error: unknown): boolean {
|
||||
return LOCAL_EXECUTION_PATTERNS.some((p) => p.test(message));
|
||||
}
|
||||
|
||||
/**
|
||||
* Anthropic/Claude model-capacity overload (HTTP 529, body "Overloaded", or a
|
||||
* STREAM_EARLY_EOF that wraps that body as 502). This is one model being
|
||||
* capacity-throttled, not a whole-provider outage — the same account still
|
||||
* serves sibling models. Must not trip the provider circuit breaker.
|
||||
*
|
||||
* Accepts an error object/string OR a numeric HTTP status (529). Callers
|
||||
* pass both `error` and `status` at the two breaker predicates.
|
||||
*
|
||||
* Live incident 2026-09-03: STREAM_EARLY_EOF: Overloaded opened `claude` and
|
||||
* a single-target combo then pre-skipped with ALL_TARGETS_SKIPPED in ~43ms.
|
||||
*/
|
||||
export function isModelCapacityOverloadError(error: unknown): boolean {
|
||||
if (error === 529) return true;
|
||||
if (typeof error === "number") return false;
|
||||
if (!error) return false;
|
||||
const errObj = typeof error === "object" ? (error as Record<string, unknown>) : null;
|
||||
if (errObj && (errObj.status === 529 || errObj.statusCode === 529)) return true;
|
||||
const message =
|
||||
typeof error === "string" ? error : typeof errObj?.message === "string" ? errObj.message : "";
|
||||
if (!message) return false;
|
||||
return /\boverloaded(?:_error)?\b/i.test(message);
|
||||
}
|
||||
|
||||
export const STATE = {
|
||||
CLOSED: "CLOSED",
|
||||
DEGRADED: "DEGRADED",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
isLocalStreamLifecycleError,
|
||||
isLocalExecutionError,
|
||||
isModelCapacityOverloadError,
|
||||
} from "../../shared/utils/circuitBreaker";
|
||||
import { isRequestScopedUpstreamFailure } from "./comboFailureLogging";
|
||||
import { getTrustedLocalRateLimitResponse } from "@omniroute/open-sse/services/rateLimitManager/errors";
|
||||
@@ -39,6 +40,8 @@ export function shouldTripProviderBreakerForResult(
|
||||
result.errorCode !== "proxy_unreachable" &&
|
||||
result.errorCode !== "RATE_LIMIT_QUEUE_TIMEOUT" &&
|
||||
result.errorCode !== "RATE_LIMIT_QUEUE_WEDGED" &&
|
||||
!isModelCapacityOverloadError(result.error) &&
|
||||
!isModelCapacityOverloadError(result.status) &&
|
||||
PROVIDER_BREAKER_FAILURE_STATUSES.has(Number(result.status))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@
|
||||
"tests/unit/circuit-breaker-registry-cap.test.ts",
|
||||
"tests/unit/circuit-breaker-resolved-5xx-12254.test.ts",
|
||||
"tests/unit/circuit-breaker-stream-controller-4602.test.ts",
|
||||
"tests/unit/overloaded-not-provider-breaker.test.ts",
|
||||
"tests/unit/claude-code-parity.test.ts",
|
||||
"tests/unit/claude-effort-suffix-strip.test.ts",
|
||||
"tests/unit/claude-oauth-provider.test.ts",
|
||||
|
||||
@@ -78,7 +78,7 @@ test("forceLiveComboTest=true prevents breaker trip (combo will try next target)
|
||||
// `{ success: false, status: 5xx }` as a success.
|
||||
test("classifyProviderBreakerResult: a resolved 503 on the single-model path is a failure", () => {
|
||||
const outcome = classifyProviderBreakerResult(
|
||||
{ success: false, status: 503, errorCode: null, errorType: null, error: "overloaded" },
|
||||
{ success: false, status: 503, errorCode: null, errorType: null, error: "service unavailable" },
|
||||
false,
|
||||
false
|
||||
);
|
||||
|
||||
@@ -22,8 +22,12 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { shouldWaitForComboCooldown, resolveComboCooldownWaitDecision, COMBO_COOLDOWN_WAIT_MARGIN_MS } =
|
||||
await import("../../open-sse/services/combo/comboCooldownRetry.ts");
|
||||
const {
|
||||
shouldWaitForComboCooldown,
|
||||
resolveComboCooldownWaitDecision,
|
||||
resolveCircuitOpenWaitDecision,
|
||||
COMBO_COOLDOWN_WAIT_MARGIN_MS,
|
||||
} = await import("../../open-sse/services/combo/comboCooldownRetry.ts");
|
||||
|
||||
function baseSettings(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
return {
|
||||
@@ -77,6 +81,11 @@ test("missing/unknown reason (null) → no wait (only an explicit transient reas
|
||||
assert.equal(r.wait, false);
|
||||
});
|
||||
|
||||
test("reason circuit_open is eligible (whole-provider breaker OPEN is a short reset)", () => {
|
||||
const r = shouldWaitForComboCooldown(baseInput({ reason: "circuit_open" }) as never);
|
||||
assert.equal(r.wait, true);
|
||||
});
|
||||
|
||||
test("waitMs above the configured ceiling → no wait", () => {
|
||||
const r = shouldWaitForComboCooldown(
|
||||
baseInput({ waitMs: 5001, settings: baseSettings({ maxWaitMs: 5000 }) }) as never
|
||||
@@ -146,6 +155,7 @@ test("returned waitMs is clamped to a finite number (0 when input invalid)", ()
|
||||
// ── resolveComboCooldownWaitDecision (target resolution + hint/fallback) ──────
|
||||
|
||||
const M = COMBO_COOLDOWN_WAIT_MARGIN_MS;
|
||||
assert.equal(M, 50, "wait margin is a pinned production constant, not a free parameter");
|
||||
|
||||
function decisionInput(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
@@ -269,3 +279,75 @@ test("resolve: lock remaining above the ceiling → no wait (not a SHORT cooldow
|
||||
);
|
||||
assert.equal(r.wait, false);
|
||||
});
|
||||
|
||||
// Live incident 2026-09-03: a single-target combo pre-skipped every target because
|
||||
// the whole-provider breaker was OPEN (claude Overloaded STREAM_EARLY_EOF). That
|
||||
// path crystallized ALL_TARGETS_SKIPPED in ~43ms and never entered the cooldown
|
||||
// wait, even though the breaker resetTimeout is 60s and comboCooldownWait was on.
|
||||
|
||||
function circuitOpenInput(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
skippedForCircuitOpen: true,
|
||||
retryAfterMs: 30_000,
|
||||
attempt: 0,
|
||||
budgetLeftMs: 90_000,
|
||||
settings: baseSettings({ maxWaitMs: 90_000, budgetMs: 300_000, maxAttempts: 5 }),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test("circuit-open skip with a short breaker reset → wait", () => {
|
||||
const r = resolveCircuitOpenWaitDecision(circuitOpenInput() as never);
|
||||
assert.equal(r.wait, true);
|
||||
assert.equal(r.waitMs, 30_000 + M);
|
||||
assert.equal(r.reason, "circuit_open");
|
||||
});
|
||||
|
||||
test("circuit-open skip is ignored when no target was skipped for circuit_open", () => {
|
||||
const r = resolveCircuitOpenWaitDecision(
|
||||
circuitOpenInput({ skippedForCircuitOpen: false }) as never
|
||||
);
|
||||
assert.equal(r.wait, false);
|
||||
assert.equal(r.reason, null);
|
||||
});
|
||||
|
||||
test("circuit-open skip with zero retryAfter → no wait", () => {
|
||||
const r = resolveCircuitOpenWaitDecision(circuitOpenInput({ retryAfterMs: 0 }) as never);
|
||||
assert.equal(r.wait, false);
|
||||
});
|
||||
|
||||
test("circuit-open skip above the wait ceiling → no wait", () => {
|
||||
const r = resolveCircuitOpenWaitDecision(
|
||||
circuitOpenInput({
|
||||
retryAfterMs: 120_000,
|
||||
settings: baseSettings({ maxWaitMs: 90_000, budgetMs: 300_000, maxAttempts: 5 }),
|
||||
}) as never
|
||||
);
|
||||
assert.equal(r.wait, false);
|
||||
});
|
||||
|
||||
test("circuit-open skip honors attempt/budget the same as model-lockout waits", () => {
|
||||
assert.equal(
|
||||
resolveCircuitOpenWaitDecision(circuitOpenInput({ attempt: 5 }) as never).wait,
|
||||
false
|
||||
);
|
||||
assert.equal(
|
||||
resolveCircuitOpenWaitDecision(circuitOpenInput({ budgetLeftMs: 1_000 }) as never).wait,
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("circuit-open skip is off when comboCooldownWait.enabled is false", () => {
|
||||
const r = resolveCircuitOpenWaitDecision(
|
||||
circuitOpenInput({
|
||||
settings: baseSettings({
|
||||
enabled: false,
|
||||
maxWaitMs: 90_000,
|
||||
budgetMs: 300_000,
|
||||
maxAttempts: 5,
|
||||
}),
|
||||
}) as never
|
||||
);
|
||||
assert.equal(r.wait, false);
|
||||
assert.equal(r.reason, null);
|
||||
});
|
||||
|
||||
@@ -29,6 +29,7 @@ const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const testSettings = {
|
||||
resilienceSettings: {
|
||||
providerCooldown: { enabled: true, minRetryCooldownMs: 5000, maxRetryCooldownMs: 300000 },
|
||||
comboCooldownWait: { enabled: false },
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
169
tests/unit/overloaded-not-provider-breaker.test.ts
Normal file
169
tests/unit/overloaded-not-provider-breaker.test.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { isModelCapacityOverloadError } from "../../src/shared/utils/circuitBreaker.ts";
|
||||
import {
|
||||
shouldTripProviderBreakerForResult,
|
||||
classifyProviderBreakerResult,
|
||||
} from "../../src/sse/handlers/chatPredicates.ts";
|
||||
import { shouldRecordProviderBreakerFailure } from "../../open-sse/services/combo/comboPredicates.ts";
|
||||
|
||||
/**
|
||||
* Live incident 2026-09-03 (X500 offical-fable): Anthropic returned
|
||||
* STREAM_EARLY_EOF wrapping "Overloaded" as HTTP 502. That 502 opened the
|
||||
* whole-provider `claude` breaker. The single-target combo then pre-skipped
|
||||
* with ALL_TARGETS_SKIPPED in ~43ms even though the account, pin, and quota
|
||||
* were healthy. Model capacity (529 / Overloaded) is not a provider outage.
|
||||
*/
|
||||
|
||||
const OTHER_COMBO_ARGS = {
|
||||
isStreamReadinessFailure: false,
|
||||
sameProviderNext: false,
|
||||
skipProviderBreaker: false,
|
||||
requestScopedFailure: false,
|
||||
} as const;
|
||||
|
||||
const LIVE_EOF_OVERLOADED = "Stream ended before producing a non-ping SSE event: Overloaded";
|
||||
const PLAIN_EOF = "Stream ended before producing a non-ping SSE event";
|
||||
|
||||
test("isModelCapacityOverloadError: live STREAM_EARLY_EOF Overloaded text", () => {
|
||||
assert.equal(isModelCapacityOverloadError(LIVE_EOF_OVERLOADED), true);
|
||||
});
|
||||
|
||||
test("isModelCapacityOverloadError: bare Overloaded / HTTP 529", () => {
|
||||
assert.equal(isModelCapacityOverloadError("Overloaded"), true);
|
||||
assert.equal(isModelCapacityOverloadError("[529]: Overloaded"), true);
|
||||
assert.equal(isModelCapacityOverloadError(529), true);
|
||||
assert.equal(isModelCapacityOverloadError({ message: "overloaded_error" }), true);
|
||||
});
|
||||
|
||||
test("isModelCapacityOverloadError: a plain early EOF is NOT capacity", () => {
|
||||
assert.equal(isModelCapacityOverloadError(PLAIN_EOF), false);
|
||||
assert.equal(isModelCapacityOverloadError("502 Bad Gateway"), false);
|
||||
assert.equal(isModelCapacityOverloadError(null), false);
|
||||
assert.equal(isModelCapacityOverloadError(undefined), false);
|
||||
});
|
||||
|
||||
test("combo: STREAM_EARLY_EOF Overloaded 502 does NOT record a whole-provider breaker failure", () => {
|
||||
assert.equal(
|
||||
shouldRecordProviderBreakerFailure({
|
||||
...OTHER_COMBO_ARGS,
|
||||
isStreamReadinessFailure: true,
|
||||
isStreamEarlyEof: true,
|
||||
status: 502,
|
||||
error: LIVE_EOF_OVERLOADED,
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("combo: a plain STREAM_EARLY_EOF 502 still records a breaker failure", () => {
|
||||
assert.equal(
|
||||
shouldRecordProviderBreakerFailure({
|
||||
...OTHER_COMBO_ARGS,
|
||||
isStreamReadinessFailure: true,
|
||||
isStreamEarlyEof: true,
|
||||
status: 502,
|
||||
error: PLAIN_EOF,
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("combo: HTTP 529 Overloaded does not record even if someone later adds 529 to the status set", () => {
|
||||
assert.equal(
|
||||
shouldRecordProviderBreakerFailure({
|
||||
...OTHER_COMBO_ARGS,
|
||||
status: 529,
|
||||
error: "[529]: Overloaded",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("combo: HTTP 529 status alone does not record a breaker failure", () => {
|
||||
assert.equal(
|
||||
shouldRecordProviderBreakerFailure({
|
||||
...OTHER_COMBO_ARGS,
|
||||
status: 529,
|
||||
error: "upstream error",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("combo: a genuine 502 without Overloaded still records", () => {
|
||||
assert.equal(
|
||||
shouldRecordProviderBreakerFailure({
|
||||
...OTHER_COMBO_ARGS,
|
||||
status: 502,
|
||||
error: "upstream error",
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("single-model: STREAM_EARLY_EOF Overloaded 502 does NOT trip the provider breaker", () => {
|
||||
assert.equal(
|
||||
shouldTripProviderBreakerForResult(
|
||||
{
|
||||
status: 502,
|
||||
errorCode: "STREAM_EARLY_EOF",
|
||||
errorType: "stream_early_eof",
|
||||
error: LIVE_EOF_OVERLOADED,
|
||||
},
|
||||
false,
|
||||
false
|
||||
),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("single-model: a genuine 502 without Overloaded still trips", () => {
|
||||
assert.equal(
|
||||
shouldTripProviderBreakerForResult(
|
||||
{ status: 502, errorCode: null, errorType: null, error: "upstream error" },
|
||||
false,
|
||||
false
|
||||
),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("single-model: HTTP 529 does not trip", () => {
|
||||
assert.equal(
|
||||
shouldTripProviderBreakerForResult(
|
||||
{ status: 529, errorCode: null, errorType: null, error: "Overloaded" },
|
||||
false,
|
||||
false
|
||||
),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("classifyProviderBreakerResult: Overloaded 502 on the single-model path is ignore", () => {
|
||||
assert.equal(
|
||||
classifyProviderBreakerResult(
|
||||
{
|
||||
success: false,
|
||||
status: 502,
|
||||
errorCode: "STREAM_EARLY_EOF",
|
||||
errorType: "stream_early_eof",
|
||||
error: LIVE_EOF_OVERLOADED,
|
||||
},
|
||||
false,
|
||||
false
|
||||
),
|
||||
"ignore"
|
||||
);
|
||||
});
|
||||
|
||||
test("classifyProviderBreakerResult: Overloaded 529 on the single-model path is ignore", () => {
|
||||
assert.equal(
|
||||
classifyProviderBreakerResult(
|
||||
{ success: false, status: 529, errorCode: null, errorType: null, error: "Overloaded" },
|
||||
false,
|
||||
false
|
||||
),
|
||||
"ignore"
|
||||
);
|
||||
});
|
||||
@@ -70,7 +70,11 @@ test("#9630: combo returns truthful error, not false ALL_ACCOUNTS_INACTIVE, when
|
||||
},
|
||||
isModelAvailable: async () => true,
|
||||
log: { info: () => {}, warn: () => {}, debug: () => {}, error: () => {} },
|
||||
settings: null,
|
||||
settings: {
|
||||
resilienceSettings: {
|
||||
comboCooldownWait: { enabled: false },
|
||||
},
|
||||
},
|
||||
relayOptions: null,
|
||||
allCombos: null,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user