mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 04:12:10 +03:00
fix: model lockout not recording for 429 rate_limit_exceeded from Antigravity
## Problem When Antigravity returns HTTP 429 with `rate_limit_exceeded` error code, the model lockout system never records the failure, so the model is not cooled down despite being rate-limited. ### Root Cause Antigravity's 429 error text is: `"Resource has been exhausted (e.g. check quota)."` The QUOTA_PATTERNS in `classify429.ts` contained overly broad regexes: - `/resource.*exhaust/i` — matches "Resource has been exhausted" - `/check.*quota/i` — matches "check quota" This caused `classifyErrorText()` to return `QUOTA_EXHAUSTED` (wrong), which set `providerExhausted = true` in the combo target exhaustion logic. With `providerExhausted`, the retry path was skipped entirely, and while the "done retrying" path should still record lockout, the misclassification cascaded into incorrect provider-level exhaustion state. Additionally, `targetExhaustion.ts` used the raw error text string instead of the structured error code (`rate_limit_exceeded`) that was already parsed from the response body. ## Fix 1. **classify429.ts** — Removed overly broad `/resource.*exhaust/i` and `/check.*quota/i` from QUOTA_PATTERNS. Antigravity's rate-limit wording is not a true quota exhaustion signal. 2. **targetExhaustion.ts** — Added optional `structuredError` to `ApplyComboTargetExhaustionOptions`. When available, the structured error code (e.g. `rate_limit_exceeded`) takes precedence over raw error text for exhaustion classification. 3. **combo.ts** — Passes `structuredError` to both `applyComboTargetExhaustion` call sites (dispatch path + retry-or-rotate path). ## Effect `structuredError.code = "rate_limit_exceeded"` → classified as rate-limit (not quota) → `providerExhausted = false` → retry proceeds → `recordModelLockoutFailure` called → model enters lockout with proper cooldown (120s base, exponential backoff). ## Tests Added 2 new tests for `structuredError.code` precedence in exhaustion classification. All 28 related tests pass.
This commit is contained in:
committed by
Diego Rodrigues de Sa e Souza
parent
f2b665b7b4
commit
5398793339
@@ -2333,6 +2333,7 @@ export async function handleComboChat({
|
||||
log,
|
||||
tag: "COMBO",
|
||||
exhaustedLogLevel: "info",
|
||||
structuredError,
|
||||
});
|
||||
|
||||
// #2101: Prevent infinite fallback loops with 400 Bad Request errors that are genuinely
|
||||
@@ -3240,6 +3241,7 @@ async function handleRoundRobinCombo({
|
||||
log,
|
||||
tag: "COMBO-RR",
|
||||
exhaustedLogLevel: "debug",
|
||||
structuredError,
|
||||
});
|
||||
|
||||
// Transient errors → mark in semaphore so round-robin stops stampeding this target.
|
||||
|
||||
@@ -15,7 +15,11 @@
|
||||
* The only standardization is the log MESSAGE wording (round-robin previously dropped the
|
||||
* "on remaining targets" suffix) — diagnostic text only, same #code + provider info.
|
||||
*/
|
||||
import { classifyErrorText, hasPerModelQuota, isProviderExhaustedReason } from "../accountFallback.ts";
|
||||
import {
|
||||
classifyErrorText,
|
||||
hasPerModelQuota,
|
||||
isProviderExhaustedReason,
|
||||
} from "../accountFallback.ts";
|
||||
import { RateLimitReason } from "../../config/constants.ts";
|
||||
import { isProviderCircuitOpenResult } from "./comboPredicates.ts";
|
||||
import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
|
||||
@@ -51,6 +55,8 @@ export type ApplyComboTargetExhaustionOptions = {
|
||||
log: ComboLogger;
|
||||
tag: string;
|
||||
exhaustedLogLevel: "info" | "debug";
|
||||
/** Structured error object from upstream response — preferred over raw errorText for classification */
|
||||
structuredError?: { code?: string; type?: string; message?: string };
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -73,6 +79,7 @@ export function applyComboTargetExhaustion(
|
||||
log,
|
||||
tag,
|
||||
exhaustedLogLevel,
|
||||
structuredError,
|
||||
} = opts;
|
||||
const { exhaustedProviders, exhaustedConnections, transientRateLimitedProviders } = sets;
|
||||
const provider = target.provider;
|
||||
@@ -84,12 +91,15 @@ export function applyComboTargetExhaustion(
|
||||
Boolean(provider && provider !== "unknown") &&
|
||||
!hasPerModelQuota(provider, rawModel) &&
|
||||
(isProviderExhaustedReason(fallbackResult) ||
|
||||
classifyErrorText(errorText) === RateLimitReason.QUOTA_EXHAUSTED ||
|
||||
classifyErrorText(structuredError?.code || errorText) === RateLimitReason.QUOTA_EXHAUSTED ||
|
||||
allAccountsRateLimited);
|
||||
if (providerExhausted) {
|
||||
exhaustedProviders.add(provider);
|
||||
const emit = exhaustedLogLevel === "debug" ? log.debug : log.info;
|
||||
emit?.(tag, `Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`);
|
||||
emit?.(
|
||||
tag,
|
||||
`Provider ${provider} quota exhausted — marking for skip on remaining targets (#1731)`
|
||||
);
|
||||
} else {
|
||||
if (result.status === 429 && !isTokenLimitBreach && provider && provider !== "unknown") {
|
||||
transientRateLimitedProviders.add(provider);
|
||||
|
||||
@@ -43,8 +43,7 @@ const QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
|
||||
/out of credits/i,
|
||||
/hard.?limit/i,
|
||||
/plan.*limit/i,
|
||||
/resource.*exhaust/i,
|
||||
/check.*quota/i,
|
||||
|
||||
// Antigravity / Cloud Code quota exhaustion ("Individual quota reached.
|
||||
// Contact your administrator to enable overages. Resets in 164h27m24s.").
|
||||
// None of the patterns above match it, so the 429 was misclassified as a
|
||||
|
||||
@@ -118,6 +118,40 @@ test("an unknown provider is never marked (guard)", () => {
|
||||
assert.equal(s.exhaustedConnections.size, 0);
|
||||
});
|
||||
|
||||
test("structuredError.code takes precedence over raw errorText for exhaustion classification", () => {
|
||||
const s = sets();
|
||||
const exhausted = applyComboTargetExhaustion(target(), {
|
||||
...baseOpts,
|
||||
errorText: "Resource has been exhausted (e.g. check quota).",
|
||||
result: { status: 429 },
|
||||
fallbackResult: {},
|
||||
sets: s,
|
||||
structuredError: { code: "rate_limit_exceeded" },
|
||||
});
|
||||
assert.equal(exhausted, false, "rate_limit_exceeded should NOT mark provider exhausted");
|
||||
assert.ok(
|
||||
s.transientRateLimitedProviders.has("test-dedup-provider"),
|
||||
"should be in transientRateLimitedProviders"
|
||||
);
|
||||
});
|
||||
|
||||
test("structuredError.code with non-matching value falls back to classifyErrorText behavior", () => {
|
||||
const s = sets();
|
||||
const exhausted = applyComboTargetExhaustion(target(), {
|
||||
...baseOpts,
|
||||
errorText: "Rate limit reached",
|
||||
result: { status: 429 },
|
||||
fallbackResult: {},
|
||||
sets: s,
|
||||
structuredError: { code: "some_unknown_code" },
|
||||
});
|
||||
assert.equal(exhausted, false, "unknown code + non-quota errorText → not exhausted");
|
||||
assert.ok(
|
||||
s.transientRateLimitedProviders.has("test-dedup-provider"),
|
||||
"should be in transientRateLimitedProviders"
|
||||
);
|
||||
});
|
||||
|
||||
test("a 200/benign status with no exhaustion mutates nothing and returns false", () => {
|
||||
const s = sets();
|
||||
const exhausted = applyComboTargetExhaustion(target(), {
|
||||
@@ -127,5 +161,8 @@ test("a 200/benign status with no exhaustion mutates nothing and returns false",
|
||||
sets: s,
|
||||
});
|
||||
assert.equal(exhausted, false);
|
||||
assert.equal(s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size, 0);
|
||||
assert.equal(
|
||||
s.exhaustedProviders.size + s.exhaustedConnections.size + s.transientRateLimitedProviders.size,
|
||||
0
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user