diff --git a/src/shared/utils/classify429.ts b/src/shared/utils/classify429.ts index 531a5bae67..9910a92ed0 100644 --- a/src/shared/utils/classify429.ts +++ b/src/shared/utils/classify429.ts @@ -123,14 +123,123 @@ export function looksLikeQuotaExhausted(body: unknown): boolean { return QUOTA_PATTERNS.some((pat) => pat.test(text)); } +/** + * A declared upstream retry window at or beyond this is treated as + * long-period exhaustion. One hour mirrors the circuit breaker's + * `quota_exhausted` cooldown bucket (`cooldownByKind`, wired in + * src/sse/handlers/chat.ts, chatHelpers.ts and + * open-sse/services/accountFallback.ts): the long bucket is only the right + * lock when the upstream's own window is at least that long. + */ +const QUOTA_SCALE_RETRY_DELAY_SECONDS = 3600; + +/** + * Quota signals that stay terminal no matter what retry hint accompanies + * them. Credits/billing exhaustion does not clear on a timer, so a short + * upstream hint must never downgrade these to a 60s retry loop. + */ +const TERMINAL_QUOTA_PATTERNS: ReadonlyArray = [ + /INSUFFICIENT_G1_CREDITS_BALANCE/i, + /credit.*exhaust/i, + /out of credits/i, + /billing.*cap/i, + /insufficient.*quota/i, + /individual quota reached/i, + /enable overages/i, + /daily free allocation/i, +]; + +/** + * Parse an upstream delay string ("38s", "26.66s", "1500ms", "2m", "1h", + * or a bare number of seconds) into seconds. + * + * Deliberately mirrors `parseDelayString` in + * open-sse/services/retryAfterJson.ts (#7940) rather than importing it: + * open-sse already imports this module (accountFallback.ts), so the + * reverse import would close a dependency cycle. Keep the two grammars in + * step when either changes. + */ +function parseDelaySeconds(value: unknown): number | null { + if (!value) return null; + const str = String(value).trim(); + const ms = /^(\d+(?:\.\d+)?)\s*ms$/i.exec(str); + if (ms) return Number.parseFloat(ms[1]) / 1000; + const sec = /^(\d+(?:\.\d+)?)\s*s$/i.exec(str); + if (sec) return Number.parseFloat(sec[1]); + const min = /^(\d+(?:\.\d+)?)\s*m$/i.exec(str); + if (min) return Number.parseFloat(min[1]) * 60; + const hr = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str); + if (hr) return Number.parseFloat(hr[1]) * 3600; + const bare = Number.parseFloat(str); + return Number.isFinite(bare) ? bare : null; +} + +/** + * Upstream-declared retry window in seconds, when the 429 carries one. + * + * Google APIs (Gemini `generativelanguage`, Vertex) attach a + * `google.rpc.RetryInfo` detail whose `retryDelay` Duration states exactly + * how long the throttle lasts, and repeat the same hint in the human + * message ("Please retry in 38.922534355s"). Gemini free-tier + * per-minute/per-token 429s open with the same "You exceeded your current + * quota, please check your plan and billing details" preamble as genuine + * long-window exhaustion, so `QUOTA_PATTERNS` cannot tell them apart — + * even the PerDay-named `quotaId` ships retryDelay values of ~30-50s + * (#9504). The declared window is the authoritative signal. + * + * Both carriers are read because the two live call paths deliver different + * shapes: `accountFallback` classifies the parsed body (details intact), + * while `chat.ts` classifies `result.rawMessage`, which + * `parseUpstreamError` has already reduced to `error.message` text. + * Structural matching keeps an unrelated `retryDelay` key from triggering + * the hint; the text form is anchored on Google's exact phrasing, matching + * the precedent in accountFallback's cooldown parser. + */ +function upstreamRetryDelaySeconds(body: unknown): number | null { + let root: unknown = body; + if (typeof body === "string") { + const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(body); + if (phrase) return Number.parseFloat(phrase[1]); + try { + root = JSON.parse(body); + } catch { + return null; + } + } + if (root === null || typeof root !== "object") return null; + const error = (root as { error?: unknown }).error; + const errorRecord = + error !== null && typeof error === "object" ? (error as Record) : {}; + const details = errorRecord.details ?? (root as Record).details; + for (const detail of Array.isArray(details) ? details : []) { + if (detail === null || typeof detail !== "object") continue; + const entry = detail as Record; + if (!String(entry["@type"] ?? "").includes("RetryInfo")) continue; + const seconds = parseDelaySeconds(entry.retryDelay); + if (seconds !== null && seconds >= 0) return seconds; + } + const message = errorRecord.message; + if (typeof message === "string") { + const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(message); + if (phrase) return Number.parseFloat(phrase[1]); + } + return null; +} + /** * Classify a 429 (or any) response into a `FailureKind`. * * Decision order: * 1. status !== 429 → `"transient"` (don't pretend to know more than * the caller does about non-429 failures). - * 2. body matches a quota keyword → `"quota_exhausted"`. - * 3. otherwise → `"rate_limit"` (default for 429 — even without + * 2. body carries a terminal credits/billing signal → `"quota_exhausted"` + * regardless of any retry hint: those do not clear on a timer. + * 3. body declares a sub-hour retry window → `"rate_limit"` even when + * generic quota keywords match: the upstream said the throttle clears + * in seconds, so the long lockout bucket would overshoot its own reset + * by 60-360x (#9504). + * 4. body matches a quota keyword → `"quota_exhausted"`. + * 5. otherwise → `"rate_limit"` (default for 429 — even without * Retry-After, a 429 is per definition a rate-limit signal). * * @param response - the upstream response with status, optional headers, @@ -143,6 +252,14 @@ export function classify429(response: { body?: unknown; }): FailureKind { if (response.status !== 429) return "transient"; + const text = bodyToText(response.body); + if (text && TERMINAL_QUOTA_PATTERNS.some((pat) => pat.test(text))) { + return "quota_exhausted"; + } + const declaredDelay = upstreamRetryDelaySeconds(response.body); + if (declaredDelay !== null && declaredDelay < QUOTA_SCALE_RETRY_DELAY_SECONDS) { + return "rate_limit"; + } if (looksLikeQuotaExhausted(response.body)) return "quota_exhausted"; return "rate_limit"; } diff --git a/tests/unit/classify429.test.ts b/tests/unit/classify429.test.ts index 08ba94de12..70f7bae9a5 100644 --- a/tests/unit/classify429.test.ts +++ b/tests/unit/classify429.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { classify429, looksLikeQuotaExhausted, + classify429FromError, parseRetryAfter, retryAfterFromResponse, type FailureKind, @@ -194,8 +195,14 @@ test("classify429: Modal-hosted endpoint 'usage limit reached' body returns 'quo "quota_exhausted" ); // Trailing punctuation/whitespace must still match. - assert.equal(classify429({ status: 429, body: { error: "usage limit reached." } }), "quota_exhausted"); - assert.equal(classify429({ status: 429, body: { error: "usage limit reached " } }), "quota_exhausted"); + assert.equal( + classify429({ status: 429, body: { error: "usage limit reached." } }), + "quota_exhausted" + ); + assert.equal( + classify429({ status: 429, body: { error: "usage limit reached " } }), + "quota_exhausted" + ); }); test("classify429: qualified transient 'usage limit reached' messages stay rate_limit", () => { @@ -277,3 +284,155 @@ test("retryAfterFromResponse: case-insensitive header lookup", () => { assert.equal(retryAfterFromResponse({ headers: {} }), null); assert.equal(retryAfterFromResponse({}), null); }); + +// --- Gemini free-tier 429s carrying google.rpc.RetryInfo (#9504) --- + +/** Real captured Gemini free-tier 429 (issue #9504), parameterized by quotaId/delay. */ +function geminiFreeTier429(quotaId: string, retryDelay: string) { + return { + error: { + code: 429, + message: + "You exceeded your current quota, please check your plan and billing details. " + + "For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. " + + "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, " + + "limit: 15, model: gemini-3.5-flash-lite\nPlease retry in 38.922534355s.", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.QuotaFailure", + violations: [ + { + quotaMetric: "generativelanguage.googleapis.com/generate_content_free_tier_requests", + quotaId, + quotaValue: "15", + }, + ], + }, + { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay }, + ], + }, + }; +} + +test("classify429: Gemini free-tier 429 with a short RetryInfo window is a rate limit", () => { + // The generic "exceeded your current quota ... check your plan" preamble + // matches three QUOTA_PATTERNS, but Google's own RetryInfo says the window + // clears in seconds. Every quotaId variant captured in #9504 ships a short + // retryDelay, including the confusingly day-named one. + const cases = [ + ["GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "38s"], + ["GenerateRequestsPerDayPerProjectPerModel-FreeTier", "29s"], + ["GenerateContentInputTokensPerModelPerMinute-FreeTier", "0s"], + ["GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "38.922534355s"], + ] as const; + for (const [quotaId, retryDelay] of cases) { + const body = geminiFreeTier429(quotaId, retryDelay); + assert.equal( + classify429({ status: 429, body }), + "rate_limit", + `${quotaId} retryDelay=${retryDelay}` + ); + } +}); + +test("classify429FromError: the production message-only shape is a rate limit", () => { + // This is the shape the live path actually delivers: parseUpstreamError + // reduces the upstream body to error.message, and chat.ts classifies + // classify429FromError({ status, message }). The RetryInfo details are + // already gone by then, so the hint must be read from Google's phrasing. + const message = + "You exceeded your current quota, please check your plan and billing details. " + + "For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\n" + + "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, " + + "limit: 15, model: gemini-3.5-flash-lite\nPlease retry in 38.922534355s."; + assert.equal(classify429FromError({ status: 429, message }), "rate_limit"); + assert.equal(classify429({ status: 429, body: message }), "rate_limit"); +}); + +test("classify429: short RetryInfo window wins for string bodies too", () => { + // The account-fallback path classifies the parsed body, so the same + // payload may arrive as pre-stringified JSON. + const body = JSON.stringify( + geminiFreeTier429("GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "14s") + ); + assert.equal(classify429({ status: 429, body }), "rate_limit"); +}); + +test("classify429: the bare RetryInfo @type and non-second units are honored", () => { + // Repo fixtures carry the short "@type": "google.rpc.RetryInfo" form, and + // the shared delay grammar (#7940) accepts ms/m/h as well as seconds. + const cases = [ + ["google.rpc.RetryInfo", "45s", "rate_limit"], + ["type.googleapis.com/google.rpc.RetryInfo", "1500ms", "rate_limit"], + ["type.googleapis.com/google.rpc.RetryInfo", "30m", "rate_limit"], + ["type.googleapis.com/google.rpc.RetryInfo", "3h", "quota_exhausted"], + ] as const; + for (const [type, retryDelay, expected] of cases) { + const body = { + error: { + message: "You exceeded your current quota, please check your plan and billing details.", + details: [{ "@type": type, retryDelay }], + }, + }; + assert.equal(classify429({ status: 429, body }), expected, `${type} ${retryDelay}`); + } +}); + +test("classify429: a terminal credits signal is never downgraded by a retry hint", () => { + // Credits/billing exhaustion does not clear on a timer, so a short + // upstream hint must not flip it into a 60s retry loop. + const cases = [ + "Individual quota reached. Contact your administrator to enable overages. Resets in 164h27m24s.", + "INSUFFICIENT_G1_CREDITS_BALANCE", + "Out of credits - top up your account.", + "you have used up your daily free allocation of 10,000 neurons", + ]; + for (const message of cases) { + const body = { + error: { + message, + details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "20s" }], + }, + }; + assert.equal(classify429({ status: 429, body }), "quota_exhausted", message.slice(0, 40)); + } +}); + +test("classify429: an hours-scale RetryInfo window keeps the quota classification", () => { + const body = geminiFreeTier429("GenerateRequestsPerDayPerProjectPerModel-FreeTier", "7200s"); + assert.equal(classify429({ status: 429, body }), "quota_exhausted"); +}); + +test("classify429: quota keywords with no retry hint at all stay quota exhausted", () => { + // Neither a RetryInfo detail nor Google's "Please retry in Ns" phrasing: + // with no declared window there is nothing to contradict the keywords. + const body = { + error: { + code: 429, + message: + "You exceeded your current quota, please check your plan and billing details. " + + "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests.", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.QuotaFailure", + violations: [{ quotaId: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" }], + }, + ], + }, + }; + assert.equal(classify429({ status: 429, body }), "quota_exhausted"); +}); + +test("classify429: retryDelay outside a RetryInfo detail is ignored", () => { + // The field name alone must not trigger the short-window path when it is + // not an upstream google.rpc.RetryInfo declaration. + const body = { + error: { + message: "You exceeded your current quota, please check your plan and billing details.", + retryDelay: "10s", + }, + }; + assert.equal(classify429({ status: 429, body }), "quota_exhausted"); +});