fix: model lockout not recording for 429 rate_limit_exceeded from Antigravity

## Problem

When Antigravity returns HTTP 429 with `rate_limit_exceeded` error code,
the model lockout system never records the failure, so the model is not
cooled down despite being rate-limited.

### Root Cause

Antigravity's 429 error text is: `"Resource has been exhausted (e.g. check
quota)."`

The QUOTA_PATTERNS in `classify429.ts` contained overly broad regexes:
- `/resource.*exhaust/i` — matches "Resource has been exhausted"
- `/check.*quota/i` — matches "check quota"

This caused `classifyErrorText()` to return `QUOTA_EXHAUSTED` (wrong),
which set `providerExhausted = true` in the combo target exhaustion logic.
With `providerExhausted`, the retry path was skipped entirely, and while
the "done retrying" path should still record lockout, the misclassification
cascaded into incorrect provider-level exhaustion state.

Additionally, `targetExhaustion.ts` used the raw error text string instead
of the structured error code (`rate_limit_exceeded`) that was already
parsed from the response body.

## Fix

1. **classify429.ts** — Removed overly broad `/resource.*exhaust/i` and
   `/check.*quota/i` from QUOTA_PATTERNS. Antigravity's rate-limit wording
   is not a true quota exhaustion signal.

2. **targetExhaustion.ts** — Added optional `structuredError` to
   `ApplyComboTargetExhaustionOptions`. When available, the structured
   error code (e.g. `rate_limit_exceeded`) takes precedence over raw error
   text for exhaustion classification.

3. **combo.ts** — Passes `structuredError` to both `applyComboTargetExhaustion`
   call sites (dispatch path + retry-or-rotate path).

## Effect

`structuredError.code = "rate_limit_exceeded"` → classified as rate-limit
(not quota) → `providerExhausted = false` → retry proceeds →
`recordModelLockoutFailure` called → model enters lockout with proper
cooldown (120s base, exponential backoff).

## Tests

Added 2 new tests for `structuredError.code` precedence in exhaustion
classification. All 28 related tests pass.
This commit is contained in:
Choti Wongbussakorn
2026-06-30 15:29:56 +07:00
committed by Diego Rodrigues de Sa e Souza
parent f2b665b7b4
commit 5398793339
4 changed files with 54 additions and 6 deletions

View File

@@ -2333,6 +2333,7 @@ export async function handleComboChat({
log,
tag: "COMBO",
exhaustedLogLevel: "info",
structuredError,
});
// #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely
@@ -3240,6 +3241,7 @@ async function handleRoundRobinCombo({
log,
tag: "COMBO-RR",
exhaustedLogLevel: "debug",
structuredError,
});
// Transient errors → mark in semaphore so round-robin stops stampeding this target.

View File

@@ -15,7 +15,11 @@
* The only standardization is the log MESSAGE wording (round-robin previously dropped the
* "on remaining targets" suffix) — diagnostic text only, same #code + provider info.
*/
import { classifyErrorText, hasPerModelQuota, isProviderExhaustedReason } from "../accountFallback.ts";
import {
classifyErrorText,
hasPerModelQuota,
isProviderExhaustedReason,
} from "../accountFallback.ts";
import { RateLimitReason } from "../../config/constants.ts";
import { isProviderCircuitOpenResult } from "./comboPredicates.ts";
import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
@@ -51,6 +55,8 @@ export type ApplyComboTargetExhaustionOptions = {
log: ComboLogger;
tag: string;
exhaustedLogLevel: "info" | "debug";
/** Structured error object from upstream response — preferred over raw errorText for classification */
structuredError?: { code?: string; type?: string; message?: string };
};
/**
@@ -73,6 +79,7 @@ export function applyComboTargetExhaustion(
log,
tag,
exhaustedLogLevel,
structuredError,
} = opts;
const { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders } = sets;
const provider = target.provider;
@@ -84,12 +91,15 @@ export function applyComboTargetExhaustion(
Boolean(provider && provider !== "unknown") &&
!hasPerModelQuota(provider, rawModel) &&
(isProviderExhaustedReason(fallbackResult) ||
classifyErrorText(errorText) === RateLimitReason.QUOTA_EXHAUSTED ||
classifyErrorText(structuredError?.code || errorText) === RateLimitReason.QUOTA_EXHAUSTED ||
allAccountsRateLimited);
if (providerExhausted) {
exhaustedProviders.add(provider);
const emit = exhaustedLogLevel === "debug" ? log.debug : log.info;
emit?.(tag, `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`);
emit?.(
tag,
`Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`
);
} else {
if (result.status === 429 && !isTokenLimitBreach && provider && provider !== "unknown") {
transientRateLimitedProviders.add(provider);

View File

@@ -43,8 +43,7 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
/out of credits/i,
/hard.?limit/i,
/plan.*limit/i,
/resource.*exhaust/i,
/check.*quota/i,
// Antigravity / Cloud Code quota exhaustion ("Individual quota reached.
// Contact your administrator to enable overages. Resets in 164h27m24s.").
// None of the patterns above match it, so the 429 was misclassified as a

View File

@@ -118,6 +118,40 @@ test("an unknown provider is never marked (guard)", () => {
assert.equal(s.exhaustedConnections.size, 0);
});
test("structuredError.code takes precedence over raw errorText for exhaustion classification", () => {
const s = sets();
const exhausted = applyComboTargetExhaustion(target(), {
...baseOpts,
errorText: "Resource has been exhausted (e.g. check quota).",
result: { status: 429 },
fallbackResult: {},
sets: s,
structuredError: { code: "rate_limit_exceeded" },
});
assert.equal(exhausted, false, "rate_limit_exceeded should NOT mark provider exhausted");
assert.ok(
s.transientRateLimitedProviders.has("test-dedup-provider"),
"should be in transientRateLimitedProviders"
);
});
test("structuredError.code with non-matching value falls back to classifyErrorText behavior", () => {
const s = sets();
const exhausted = applyComboTargetExhaustion(target(), {
...baseOpts,
errorText: "Rate limit reached",
result: { status: 429 },
fallbackResult: {},
sets: s,
structuredError: { code: "some_unknown_code" },
});
assert.equal(exhausted, false, "unknown code + non-quota errorText → not exhausted");
assert.ok(
s.transientRateLimitedProviders.has("test-dedup-provider"),
"should be in transientRateLimitedProviders"
);
});
test("a 200/benign status with no exhaustion mutates nothing and returns false", () => {
const s = sets();
const exhausted = applyComboTargetExhaustion(target(), {
@@ -127,5 +161,8 @@ test("a 200/benign status with no exhaustion mutates nothing and returns false",
sets: s,
});
assert.equal(exhausted, false);
assert.equal(s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, 0);
assert.equal(
s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size,
0
);
});