diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index a238f0feb5..74393938ed 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -1388,7 +1388,14 @@ export class AntigravityExecutor extends BaseExecutor { try { const errorBody = await response.clone().text(); const errorJson = JSON.parse(errorBody); - const errorMessage = errorJson?.error?.message || errorJson?.message || ""; + let errorMessage = errorJson?.error?.message || errorJson?.message || ""; + if (errorJson?.error?.details && Array.isArray(errorJson.error.details)) { + for (const detail of errorJson.error.details) { + if (detail?.reason) { + errorMessage += ` ${detail.reason}`; + } + } + } // 1. Try to parse explicit retry time from message const parsedRetryMs = this.parseRetryFromErrorMessage(errorMessage); diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 4139845c2e..0efddc1762 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -1296,6 +1296,7 @@ export function checkFallbackError( * provider itself). Callers should apply connection cooldown only — do NOT record a provider * circuit-breaker failure when this flag is set. */ skipProviderBreaker?: boolean; + quotaResetHintMs?: number; } { // G-02: detect embedded service supervisor failures (X-Omni-Fallback-Hint: connection_cooldown). // These are NOT upstream AI provider failures — they are local supervisor state changes. @@ -1478,11 +1479,13 @@ export function checkFallbackError( // profile.useUpstreamRetryHints. const hintMs = getUpstreamRetryHintMs(); const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour + const bodyHint = parseRetryFromErrorText(errorStr); return { shouldFallback: true, cooldownMs: hintMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS, reason: RateLimitReason.QUOTA_EXHAUSTED, usedUpstreamRetryHint: Boolean(hintMs), + quotaResetHintMs: bodyHint ?? undefined, }; } @@ -1492,7 +1495,11 @@ export function checkFallbackError( quotaResetHintMs && classifyErrorText(errorStr) === RateLimitReason.QUOTA_EXHAUSTED ) { - return buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED); + const fallbackResult = buildRetryableFallback(RateLimitReason.QUOTA_EXHAUSTED); + return { + ...fallbackResult, + quotaResetHintMs, + }; } // #2929: A route-restriction 403 (e.g. Fireworks Fire Pass keys returning diff --git a/open-sse/services/antigravity429Engine.ts b/open-sse/services/antigravity429Engine.ts index 909ed8adff..a29b0dba2e 100644 --- a/open-sse/services/antigravity429Engine.ts +++ b/open-sse/services/antigravity429Engine.ts @@ -21,10 +21,7 @@ export type Category = "unknown" | "rate_limited" | "quota_exhausted" | "soft_rate_limit"; export type DecisionKind = - | "soft_retry" - | "instant_retry_same_auth" - | "short_cooldown_switch_auth" - | "full_quota_exhausted"; + "soft_retry" | "instant_retry_same_auth" | "short_cooldown_switch_auth" | "full_quota_exhausted"; export interface Decision { kind: DecisionKind; @@ -53,6 +50,8 @@ const CREDITS_EXHAUSTED_KEYWORDS = [ "minimumcreditamountforusage", "minimum credit amount for usage", "minimum credit", + "insufficient_g1_credits_balance", + "g1_credits", ]; const SHORT_COOLDOWN_MS = 5 * 60 * 1000; // 5 minutes diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index 6ff84edda0..ff6f21ab31 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -52,6 +52,7 @@ const QUOTA_PATTERNS: ReadonlyArray = [ // per-minute limits like "request quota reached, retry in 60s". /individual quota reached/i, /enable overages/i, + /INSUFFICIENT_G1_CREDITS_BALANCE/i, ]; /** diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 5cd4ba6750..38148e3edd 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -2020,7 +2020,9 @@ export async function markAccountUnavailable( { ...modelLockoutOptions, exactCooldownMs: - fallbackResult.usedUpstreamRetryHint === true ? fallbackResult.cooldownMs : null, + fallbackResult.usedUpstreamRetryHint === true + ? fallbackResult.cooldownMs + : (fallbackResult.quotaResetHintMs ?? null), maxCooldownMs: mlSettings.maxCooldownMs, } ); diff --git a/tests/unit/antigravity-429-quota-cooldown.test.ts b/tests/unit/antigravity-429-quota-cooldown.test.ts index f0776c9694..eab627215b 100644 --- a/tests/unit/antigravity-429-quota-cooldown.test.ts +++ b/tests/unit/antigravity-429-quota-cooldown.test.ts @@ -55,6 +55,10 @@ test("classify429: AG 'Individual quota reached' message → quota_exhausted", ( assert.equal(classify429(msg), "quota_exhausted"); }); +test("classify429: AG G1 Credits Exhausted message → quota_exhausted", () => { + assert.equal(classify429("insufficient_g1_credits_balance"), "quota_exhausted"); +}); + test("classify429: standard Gemini rate limit 'resource has been exhausted' -> rate_limited or unknown, not quota_exhausted", () => { const msg = "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. queries per minute limit was reached)."; diff --git a/tests/unit/antigravity-429-quota-tdd.test.ts b/tests/unit/antigravity-429-quota-tdd.test.ts new file mode 100644 index 0000000000..c085adf2c3 --- /dev/null +++ b/tests/unit/antigravity-429-quota-tdd.test.ts @@ -0,0 +1,88 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { classify429 as classify429AG } from "../../open-sse/services/antigravity429Engine.ts"; +import { classify429 as classify429Shared } from "../../src/shared/utils/classify429.ts"; +import { + parseRetryFromErrorText, + checkFallbackError, +} from "../../open-sse/services/accountFallback.ts"; + +test("TDD S1: classify429 (Antigravity engine) detects INSUFFICIENT_G1_CREDITS_BALANCE", () => { + const msg = JSON.stringify({ + error: { + code: 429, + message: "Resource has been exhausted (e.g. check quota).", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + reason: "INSUFFICIENT_G1_CREDITS_BALANCE", + }, + ], + }, + }); + + const category = classify429AG(msg); + assert.equal(category, "quota_exhausted"); +}); + +test("TDD S1: classify429 (Shared utility) detects INSUFFICIENT_G1_CREDITS_BALANCE", () => { + const body = { + error: { + code: 429, + message: "Resource has been exhausted (e.g. check quota).", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + reason: "INSUFFICIENT_G1_CREDITS_BALANCE", + }, + ], + }, + }; + + const kind = classify429Shared({ status: 429, body }); + assert.equal(kind, "quota_exhausted"); +}); + +test("TDD S2: Regression: standard Gemini rate limit 'queries per minute limit was reached' -> rate_limit (shared) and rate_limited (AG)", () => { + const msg = + "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. queries per minute limit was reached)."; + assert.notEqual(classify429AG(msg), "quota_exhausted"); + assert.equal(classify429Shared({ status: 429, body: msg }), "rate_limit"); +}); + +test("TDD S3: parseRetryFromErrorText parses resets in 5h and resets in 164h27m24s", () => { + // Antigravity returns: "Individual quota reached. Contact your administrator to enable overages. Resets in 5h." + const msg5h = + "Individual quota reached. Contact your administrator to enable overages. Resets in 5h."; + const msgWeekly = + "Individual quota reached. Contact your administrator to enable overages. Resets in 164h27m24s."; + + const val5h = parseRetryFromErrorText(msg5h); + assert.equal(val5h, 5 * 3600 * 1000); + + const valWeekly = parseRetryFromErrorText(msgWeekly); + assert.equal(valWeekly, 164 * 3600 * 1000 + 27 * 60 * 1000 + 24 * 1000); +}); + +test("TDD S3: checkFallbackError extracts retry hint for oauth providers even if useUpstreamRetryHints is false", () => { + const errorText = + "Individual quota reached. Contact your administrator to enable overages. Resets in 5h."; + const res = checkFallbackError( + 429, + errorText, + 0, + "gemini-3.5-flash", + "antigravity", // which uses oauth provider profile (useUpstreamRetryHints: false) + null + ); + + assert.equal(res.shouldFallback, true); + assert.equal(res.usedUpstreamRetryHint, false); + // Connection cooldown should be the default/scaled backoff cooldown (e.g. ~5000 ms) because useUpstreamRetryHints is false + assert.notEqual(res.cooldownMs, 5 * 3600 * 1000); + // But quotaResetHintMs MUST be the precise parsed reset time (5h = 18,000,000 ms) + assert.equal(res.quotaResetHintMs, 5 * 3600 * 1000); +}); diff --git a/tests/unit/classify429.test.ts b/tests/unit/classify429.test.ts index 1a37a1d720..668de125df 100644 --- a/tests/unit/classify429.test.ts +++ b/tests/unit/classify429.test.ts @@ -41,10 +41,25 @@ test("classify429: Antigravity 'Individual quota reached' body returns 'quota_ex "Resets in 164h27m24s."; assert.equal(looksLikeQuotaExhausted(body), true); assert.equal(classify429({ status: 429, body }), "quota_exhausted"); - assert.equal( - classify429({ status: 429, body: { error: { message: body } } }), - "quota_exhausted" - ); + assert.equal(classify429({ status: 429, body: { error: { message: body } } }), "quota_exhausted"); +}); + +test("classify429: Antigravity INSUFFICIENT_G1_CREDITS_BALANCE body returns 'quota_exhausted'", () => { + const body = { + error: { + code: 429, + message: "Resource has been exhausted (e.g. check quota).", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + reason: "INSUFFICIENT_G1_CREDITS_BALANCE", + }, + ], + }, + }; + assert.equal(looksLikeQuotaExhausted(body), true); + assert.equal(classify429({ status: 429, body }), "quota_exhausted"); }); test("classify429: Antigravity quota patterns do not over-match plain rate limits", () => {