From 5398793339ddc5ea268bd2e8ce06e8b199aafcab Mon Sep 17 00:00:00 2001 From: Choti Wongbussakorn <126886556+Chewji9875@users.noreply.github.com> Date: Tue, 30 Jun 2026 15:29:56 +0700 Subject: [PATCH] fix: model lockout not recording for 429 rate_limit_exceeded from Antigravity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem When Antigravity returns HTTP 429 with `rate_limit_exceeded` error code, the model lockout system never records the failure, so the model is not cooled down despite being rate-limited. ### Root Cause Antigravity's 429 error text is: `"Resource has been exhausted (e.g. check quota)."` The QUOTA_PATTERNS in `classify429.ts` contained overly broad regexes: - `/resource.*exhaust/i` — matches "Resource has been exhausted" - `/check.*quota/i` — matches "check quota" This caused `classifyErrorText()` to return `QUOTA_EXHAUSTED` (wrong), which set `providerExhausted = true` in the combo target exhaustion logic. With `providerExhausted`, the retry path was skipped entirely, and while the "done retrying" path should still record lockout, the misclassification cascaded into incorrect provider-level exhaustion state. Additionally, `targetExhaustion.ts` used the raw error text string instead of the structured error code (`rate_limit_exceeded`) that was already parsed from the response body. ## Fix 1. **classify429.ts** — Removed overly broad `/resource.*exhaust/i` and `/check.*quota/i` from QUOTA_PATTERNS. Antigravity's rate-limit wording is not a true quota exhaustion signal. 2. **targetExhaustion.ts** — Added optional `structuredError` to `ApplyComboTargetExhaustionOptions`. When available, the structured error code (e.g. `rate_limit_exceeded`) takes precedence over raw error text for exhaustion classification. 3. **combo.ts** — Passes `structuredError` to both `applyComboTargetExhaustion` call sites (dispatch path + retry-or-rotate path). ## Effect `structuredError.code = "rate_limit_exceeded"` → classified as rate-limit (not quota) → `providerExhausted = false` → retry proceeds → `recordModelLockoutFailure` called → model enters lockout with proper cooldown (120s base, exponential backoff). ## Tests Added 2 new tests for `structuredError.code` precedence in exhaustion classification. All 28 related tests pass. --- open-sse/services/combo.ts | 2 + open-sse/services/combo/targetExhaustion.ts | 16 ++++++-- src/shared/utils/classify429.ts | 3 +- .../combo/combo-target-exhaustion.test.ts | 39 ++++++++++++++++++- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index e445fac026..157018d553 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -2333,6 +2333,7 @@ export async function handleComboChat({ log, tag: "COMBO", exhaustedLogLevel: "info", + structuredError, }); // #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely @@ -3240,6 +3241,7 @@ async function handleRoundRobinCombo({ log, tag: "COMBO-RR", exhaustedLogLevel: "debug", + structuredError, }); // Transient errors → mark in semaphore so round-robin stops stampeding this target. diff --git a/open-sse/services/combo/targetExhaustion.ts b/open-sse/services/combo/targetExhaustion.ts index bf3d69dc57..01f73d58db 100644 --- a/open-sse/services/combo/targetExhaustion.ts +++ b/open-sse/services/combo/targetExhaustion.ts @@ -15,7 +15,11 @@ * The only standardization is the log MESSAGE wording (round-robin previously dropped the * "on remaining targets" suffix) — diagnostic text only, same #code + provider info. */ -import { classifyErrorText, hasPerModelQuota, isProviderExhaustedReason } from "../accountFallback.ts"; +import { + classifyErrorText, + hasPerModelQuota, + isProviderExhaustedReason, +} from "../accountFallback.ts"; import { RateLimitReason } from "../../config/constants.ts"; import { isProviderCircuitOpenResult } from "./comboPredicates.ts"; import type { ComboLogger, ResolvedComboTarget } from "./types.ts"; @@ -51,6 +55,8 @@ export type ApplyComboTargetExhaustionOptions = { log: ComboLogger; tag: string; exhaustedLogLevel: "info" | "debug"; + /** Structured error object from upstream response — preferred over raw errorText for classification */ + structuredError?: { code?: string; type?: string; message?: string }; }; /** @@ -73,6 +79,7 @@ export function applyComboTargetExhaustion( log, tag, exhaustedLogLevel, + structuredError, } = opts; const { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders } = sets; const provider = target.provider; @@ -84,12 +91,15 @@ export function applyComboTargetExhaustion( Boolean(provider && provider !== "unknown") && !hasPerModelQuota(provider, rawModel) && (isProviderExhaustedReason(fallbackResult) || - classifyErrorText(errorText) === RateLimitReason.QUOTA_EXHAUSTED || + classifyErrorText(structuredError?.code || errorText) === RateLimitReason.QUOTA_EXHAUSTED || allAccountsRateLimited); if (providerExhausted) { exhaustedProviders.add(provider); const emit = exhaustedLogLevel === "debug" ? log.debug : log.info; - emit?.(tag, `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`); + emit?.( + tag, + `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)` + ); } else { if (result.status === 429 && !isTokenLimitBreach && provider && provider !== "unknown") { transientRateLimitedProviders.add(provider); diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index 81ae985a99..6ff84edda0 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -43,8 +43,7 @@ const QUOTA_PATTERNS: ReadonlyArray = [ /out of credits/i, /hard.?limit/i, /plan.*limit/i, - /resource.*exhaust/i, - /check.*quota/i, + // Antigravity / Cloud Code quota exhaustion ("Individual quota reached. // Contact your administrator to enable overages. Resets in 164h27m24s."). // None of the patterns above match it, so the 429 was misclassified as a diff --git a/tests/unit/combo/combo-target-exhaustion.test.ts b/tests/unit/combo/combo-target-exhaustion.test.ts index b5dcce432c..5d28703aa0 100644 --- a/tests/unit/combo/combo-target-exhaustion.test.ts +++ b/tests/unit/combo/combo-target-exhaustion.test.ts @@ -118,6 +118,40 @@ test("an unknown provider is never marked (guard)", () => { assert.equal(s.exhaustedConnections.size, 0); }); +test("structuredError.code takes precedence over raw errorText for exhaustion classification", () => { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Resource has been exhausted (e.g. check quota).", + result: { status: 429 }, + fallbackResult: {}, + sets: s, + structuredError: { code: "rate_limit_exceeded" }, + }); + assert.equal(exhausted, false, "rate_limit_exceeded should NOT mark provider exhausted"); + assert.ok( + s.transientRateLimitedProviders.has("test-dedup-provider"), + "should be in transientRateLimitedProviders" + ); +}); + +test("structuredError.code with non-matching value falls back to classifyErrorText behavior", () => { + const s = sets(); + const exhausted = applyComboTargetExhaustion(target(), { + ...baseOpts, + errorText: "Rate limit reached", + result: { status: 429 }, + fallbackResult: {}, + sets: s, + structuredError: { code: "some_unknown_code" }, + }); + assert.equal(exhausted, false, "unknown code + non-quota errorText → not exhausted"); + assert.ok( + s.transientRateLimitedProviders.has("test-dedup-provider"), + "should be in transientRateLimitedProviders" + ); +}); + test("a 200/benign status with no exhaustion mutates nothing and returns false", () => { const s = sets(); const exhausted = applyComboTargetExhaustion(target(), { @@ -127,5 +161,8 @@ test("a 200/benign status with no exhaustion mutates nothing and returns false", sets: s, }); assert.equal(exhausted, false); - assert.equal(s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, 0); + assert.equal( + s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, + 0 + ); });