diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index e445fac026..157018d553 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -2333,6 +2333,7 @@ export async function handleComboChat({ log, tag: "COMBO", exhaustedLogLevel: "info", + structuredError, }); // #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely @@ -3240,6 +3241,7 @@ async function handleRoundRobinCombo({ log, tag: "COMBO-RR", exhaustedLogLevel: "debug", + structuredError, }); // Transient errors → mark in semaphore so round-robin stops stampeding this target. diff --git a/open-sse/services/combo/targetExhaustion.ts b/open-sse/services/combo/targetExhaustion.ts index bf3d69dc57..01f73d58db 100644 --- a/open-sse/services/combo/targetExhaustion.ts +++ b/open-sse/services/combo/targetExhaustion.ts @@ -15,7 +15,11 @@ * The only standardization is the log MESSAGE wording (round-robin previously dropped the * "on remaining targets" suffix) — diagnostic text only, same #code + provider info. */ -import { classifyErrorText, hasPerModelQuota, isProviderExhaustedReason } from "../accountFallback.ts"; +import { + classifyErrorText, + hasPerModelQuota, + isProviderExhaustedReason, +} from "../accountFallback.ts"; import { RateLimitReason } from "../../config/constants.ts"; import { isProviderCircuitOpenResult } from "./comboPredicates.ts"; import type { ComboLogger, ResolvedComboTarget } from "./types.ts"; @@ -51,6 +55,8 @@ export type ApplyComboTargetExhaustionOptions = { log: ComboLogger; tag: string; exhaustedLogLevel: "info" | "debug"; + /** Structured error object from upstream response — preferred over raw errorText for classification */ + structuredError?: { code?: string; type?: string; message?: string }; }; /** @@ -73,6 +79,7 @@ export function applyComboTargetExhaustion( log, tag, exhaustedLogLevel, + structuredError, } = opts; const { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders } = sets; const provider = target.provider; @@ -84,12 +91,15 @@ export function applyComboTargetExhaustion( Boolean(provider && provider !== "unknown") && !hasPerModelQuota(provider, rawModel) && (isProviderExhaustedReason(fallbackResult) || - classifyErrorText(errorText) === RateLimitReason.QUOTA_EXHAUSTED || + classifyErrorText(structuredError?.code || errorText) === RateLimitReason.QUOTA_EXHAUSTED || allAccountsRateLimited); if (providerExhausted) { exhaustedProviders.add(provider); const emit = exhaustedLogLevel === "debug" ? log.debug : log.info; - emit?.(tag, `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`); + emit?.( + tag, + `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)` + ); } else { if (result.status === 429 && !isTokenLimitBreach && provider && provider !== "unknown") { transientRateLimitedProviders.add(provider); diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index 81ae985a99..6ff84edda0 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -43,8 +43,7 @@ const QUOTA_PATTERNS: ReadonlyArray = [ /out of credits/i, /hard.?limit/i, /plan.*limit/i, - /resource.*exhaust/i, - /check.*quota/i, + // Antigravity / Cloud Code quota exhaustion ("Individual quota reached. // Contact your administrator to enable overages. Resets in 164h27m24s."). // None of the patterns above match it, so the 429 was misclassified as a diff --git a/tests/unit/combo/combo-target-exhaustion.test.ts b/tests/unit/combo/combo-target-exhaustion.test.ts index b5dcce432c..5d28703aa0 100644 --- a/tests/unit/combo/combo-target-exhaustion.test.ts +++ b/tests/unit/combo/combo-target-exhaustion.test.ts @@ -118,6 +118,40 @@ test("an unknown provider is never marked (guard)", () => { assert.equal(s.exhaustedConnections.size, 0); }); +test("structuredError.code takes precedence over raw errorText for exhaustion classification", () => { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Resource has been exhausted (e.g. check quota).", + result: { status: 429 }, + fallbackResult: {}, + sets: s, + structuredError: { code: "rate_limit_exceeded" }, + }); + assert.equal(exhausted, false, "rate_limit_exceeded should NOT mark provider exhausted"); + assert.ok( + s.transientRateLimitedProviders.has("test-dedup-provider"), + "should be in transientRateLimitedProviders" + ); +}); + +test("structuredError.code with non-matching value falls back to classifyErrorText behavior", () => { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Rate limit reached", + result: { status: 429 }, + fallbackResult: {}, + sets: s, + structuredError: { code: "some_unknown_code" }, + }); + assert.equal(exhausted, false, "unknown code + non-quota errorText → not exhausted"); + assert.ok( + s.transientRateLimitedProviders.has("test-dedup-provider"), + "should be in transientRateLimitedProviders" + ); +}); + test("a 200/benign status with no exhaustion mutates nothing and returns false", () => { const s = sets(); const exhausted = applyComboTargetExhaustion(target(), { @@ -127,5 +161,8 @@ test("a 200/benign status with no exhaustion mutates nothing and returns false", sets: s, }); assert.equal(exhausted, false); - assert.equal(s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, 0); + assert.equal( + s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, + 0 + ); });