/** * 429 response classifier — distinguish rate-limit from quota-exhausted. * * Most LLM providers return HTTP 429 for two semantically different reasons: * * 1. **Rate-limit**: short transient back-off ("too many requests in * the last minute"). Fix: wait the Retry-After window and retry. * 2. **Quota-exhausted**: long-period cap hit ("daily/monthly limit * reached"). Fix: wait until the period rolls over (could be hours * or days). Retrying every 60s wastes calls and burns alerts. * * The HTTP status alone cannot disambiguate. This helper inspects the * response body and headers to return a `FailureKind` the circuit * breaker can use to pick the right cooldown. * * Companion to OmniRoute issue #2100. * * @module shared/utils/classify429 */ export type FailureKind = "rate_limit" | "quota_exhausted" | "transient"; /** * Heuristic regexes for "explicit quota exhausted" vs "rate-limited" * detection in 429 error bodies. A 429 alone never implies quota * exhausted — only an explicit keyword does. * * Patterns observed across OpenAI, Anthropic, Groq, Cerebras, Mistral, * Google Gemini, and OpenRouter free-tier responses. */ const QUOTA_PATTERNS: ReadonlyArray = [ /daily.*limit/i, /daily.*quota/i, /per.?day.*limit/i, /monthly.*limit/i, /monthly.*quota/i, /per.?month.*limit/i, /quota.*exceed/i, /exceed.*quota/i, /insufficient.*quota/i, /billing.*cap/i, /credit.*exhaust/i, /out of credits/i, /hard.?limit/i, /plan.*limit/i, // Antigravity / Cloud Code quota exhaustion ("Individual quota reached. // Contact your administrator to enable overages. Resets in 164h27m24s."). // None of the patterns above match it, so the 429 was misclassified as a // transient rate-limit and locked for only ~5s instead of the real window. // Keep these specific: a bare /quota reached/ would also flag transient // per-minute limits like "request quota reached, retry in 60s". /individual quota reached/i, /enable overages/i, /INSUFFICIENT_G1_CREDITS_BALANCE/i, // Google APIs return this generic RESOURCE_EXHAUSTED message when a // billing-period quota has been consumed. Keep the reset-window qualifier // so transient Google rate limits are not treated as long-term exhaustion. /resource has been exhausted.*reset after/i, // Cloudflare Workers AI daily neuron exhaustion (Issue #6980). // Body: "you have used up your daily free allocation of 10,000 neurons, // please upgrade to Cloudflare's Workers Paid plan..." // No existing pattern matches "daily free allocation" — without this, // the 429 is misclassified as transient rate_limit and retried every // ~60s against a budget that only resets at UTC midnight. /daily free allocation/i, // OmniRoute auth-layer synthetic 429 (Issue #9269). // Body: "All antigravity accounts have exhausted their quota (reset after 5m)" // Produced by auth.ts line 1477 when every account for a provider has // exhausted its quota. Without this pattern, the message is classified as // a transient rate-limit and the combo loop burns retries against the // same provider instead of falling back to a healthy one. /have exhausted their quota/i, // Modal-hosted OpenAI-compatible endpoints (e.g. self-hosted Kimi K3). // Body: {"error":"usage limit reached"}, no nested "message"/"quota"/ // "daily" wording. Without this pattern the 429 falls through to // "rate_limit" (short cooldown), so combo round-robin's per-conversation // session stickiness (#3825) keeps re-targeting the same exhausted // connection every turn instead of a long lockout that lets the sticky // target fail over to another account. // // Matches the "error" JSON key with "usage limit reached" as its value. // Extra sibling fields (e.g. {"error":"usage limit reached", "code":"..."}) // still match. A different key like {"detail":"..."} or a qualified value // like {"error":"Per-minute usage limit reached"} does NOT match. Bare // string bodies without a JSON wrapper also do NOT match. // Trailing punctuation/whitespace before the closing quote is tolerated // because real API responses may include a period or trailing space. /"error"\s*:\s*"usage limit reached[.\s]*"/i, ]; /** * Best-effort case-insensitive header lookup. */ function getHeader(headers: Record | undefined, name: string): string | undefined { if (!headers) return undefined; const target = name.toLowerCase(); for (const [k, v] of Object.entries(headers)) { if (k.toLowerCase() === target) return v; } return undefined; } /** * Coerce a body of unknown shape to a string for keyword scanning. * - string: returned as-is * - object: JSON-stringified (so nested error.message gets scanned) * - undefined/null: empty string */ function bodyToText(body: unknown): string { if (typeof body === "string") return body; if (body == null) return ""; try { return JSON.stringify(body); } catch { return ""; } } /** * Returns true if the body looks like an explicit quota-exhausted * error — i.e. the upstream is telling us a long-period cap was hit. */ export function looksLikeQuotaExhausted(body: unknown): boolean { const text = bodyToText(body); if (!text) return false; return QUOTA_PATTERNS.some((pat) => pat.test(text)); } /** * A declared upstream retry window at or beyond this is treated as * long-period exhaustion. One hour mirrors the circuit breaker's * `quota_exhausted` cooldown bucket (`cooldownByKind`, wired in * src/sse/handlers/chat.ts, chatHelpers.ts and * open-sse/services/accountFallback.ts): the long bucket is only the right * lock when the upstream's own window is at least that long. */ const QUOTA_SCALE_RETRY_DELAY_SECONDS = 3600; /** * Quota signals that stay terminal no matter what retry hint accompanies * them. Credits/billing exhaustion does not clear on a timer, so a short * upstream hint must never downgrade these to a 60s retry loop. */ const TERMINAL_QUOTA_PATTERNS: ReadonlyArray = [ /INSUFFICIENT_G1_CREDITS_BALANCE/i, /credit.*exhaust/i, /out of credits/i, /billing.*cap/i, /insufficient.*quota/i, /individual quota reached/i, /enable overages/i, /daily free allocation/i, ]; /** * Parse an upstream delay string ("38s", "26.66s", "1500ms", "2m", "1h", * or a bare number of seconds) into seconds. * * Deliberately mirrors `parseDelayString` in * open-sse/services/retryAfterJson.ts (#7940) rather than importing it: * open-sse already imports this module (accountFallback.ts), so the * reverse import would close a dependency cycle. Keep the two grammars in * step when either changes. */ function parseDelaySeconds(value: unknown): number | null { if (!value) return null; const str = String(value).trim(); const ms = /^(\d+(?:\.\d+)?)\s*ms$/i.exec(str); if (ms) return Number.parseFloat(ms[1]) / 1000; const sec = /^(\d+(?:\.\d+)?)\s*s$/i.exec(str); if (sec) return Number.parseFloat(sec[1]); const min = /^(\d+(?:\.\d+)?)\s*m$/i.exec(str); if (min) return Number.parseFloat(min[1]) * 60; const hr = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str); if (hr) return Number.parseFloat(hr[1]) * 3600; const bare = Number.parseFloat(str); return Number.isFinite(bare) ? bare : null; } /** * Upstream-declared retry window in seconds, when the 429 carries one. * * Google APIs (Gemini `generativelanguage`, Vertex) attach a * `google.rpc.RetryInfo` detail whose `retryDelay` Duration states exactly * how long the throttle lasts, and repeat the same hint in the human * message ("Please retry in 38.922534355s"). Gemini free-tier * per-minute/per-token 429s open with the same "You exceeded your current * quota, please check your plan and billing details" preamble as genuine * long-window exhaustion, so `QUOTA_PATTERNS` cannot tell them apart — * even the PerDay-named `quotaId` ships retryDelay values of ~30-50s * (#9504). The declared window is the authoritative signal. * * Both carriers are read because the two live call paths deliver different * shapes: `accountFallback` classifies the parsed body (details intact), * while `chat.ts` classifies `result.rawMessage`, which * `parseUpstreamError` has already reduced to `error.message` text. * Structural matching keeps an unrelated `retryDelay` key from triggering * the hint; the text form is anchored on Google's exact phrasing, matching * the precedent in accountFallback's cooldown parser. */ function upstreamRetryDelaySeconds(body: unknown): number | null { let root: unknown = body; if (typeof body === "string") { const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(body); if (phrase) return Number.parseFloat(phrase[1]); try { root = JSON.parse(body); } catch { return null; } } if (root === null || typeof root !== "object") return null; const error = (root as { error?: unknown }).error; const errorRecord = error !== null && typeof error === "object" ? (error as Record) : {}; const details = errorRecord.details ?? (root as Record).details; for (const detail of Array.isArray(details) ? details : []) { if (detail === null || typeof detail !== "object") continue; const entry = detail as Record; if (!String(entry["@type"] ?? "").includes("RetryInfo")) continue; const seconds = parseDelaySeconds(entry.retryDelay); if (seconds !== null && seconds >= 0) return seconds; } const message = errorRecord.message; if (typeof message === "string") { const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(message); if (phrase) return Number.parseFloat(phrase[1]); } return null; } /** * Classify a 429 (or any) response into a `FailureKind`. * * Decision order: * 1. status !== 429 → `"transient"` (don't pretend to know more than * the caller does about non-429 failures). * 2. body carries a terminal credits/billing signal → `"quota_exhausted"` * regardless of any retry hint: those do not clear on a timer. * 3. body declares a sub-hour retry window → `"rate_limit"` even when * generic quota keywords match: the upstream said the throttle clears * in seconds, so the long lockout bucket would overshoot its own reset * by 60-360x (#9504). * 4. body matches a quota keyword → `"quota_exhausted"`. * 5. otherwise → `"rate_limit"` (default for 429 — even without * Retry-After, a 429 is per definition a rate-limit signal). * * @param response - the upstream response with status, optional headers, * optional body. Headers are looked up * case-insensitively. */ export function classify429(response: { status: number; headers?: Record; body?: unknown; }): FailureKind { if (response.status !== 429) return "transient"; const text = bodyToText(response.body); if (text && TERMINAL_QUOTA_PATTERNS.some((pat) => pat.test(text))) { return "quota_exhausted"; } const declaredDelay = upstreamRetryDelaySeconds(response.body); if (declaredDelay !== null && declaredDelay < QUOTA_SCALE_RETRY_DELAY_SECONDS) { return "rate_limit"; } if (looksLikeQuotaExhausted(response.body)) return "quota_exhausted"; return "rate_limit"; } /** * Parse a `Retry-After` header value into seconds. * * Accepts: * - integer seconds: `"60"` * - HTTP date: `"Wed, 08 May 2026 03:00:00 GMT"` * - Groq-style relative: `"60s"`, `"5m"`, `"2h"` * * Returns `null` if unparseable. * * Note: integer seconds vs Groq relative units are easy to confuse — * `parseInt("5m", 10)` returns `5` (parses leading digits and ignores * trailing). This helper checks the relative-unit pattern FIRST. */ export function parseRetryAfter(headerValue: string | undefined): number | null { if (!headerValue) return null; const trimmed = headerValue.trim(); if (!trimmed) return null; // Groq-style relative: must check BEFORE plain int parse. const relMatch = trimmed.match(/^(\d+)([smh])$/i); if (relMatch) { const n = Number(relMatch[1]); const unit = relMatch[2].toLowerCase(); if (Number.isFinite(n)) { if (unit === "s") return n; if (unit === "m") return n * 60; if (unit === "h") return n * 3600; } } // Pure integer seconds. if (/^\d+$/.test(trimmed)) { const n = Number(trimmed); return Number.isFinite(n) ? n : null; } // HTTP date. const ts = Date.parse(trimmed); if (Number.isFinite(ts)) { return Math.max(0, Math.floor((ts - Date.now()) / 1000)); } return null; } /** * Convenience wrapper: pull the Retry-After from a response's headers * and parse it to seconds. Returns null if absent or unparseable. */ export function retryAfterFromResponse(response: { headers?: Record; }): number | null { return parseRetryAfter(getHeader(response.headers, "retry-after")); } /** * Normalize an unknown headers-like value into a plain `Record`. * Native `Headers` (from `fetch`) does NOT respond to `Object.entries` — it * exposes `.entries()` instead. Without this normalization, `getHeader` would * silently miss every header on a Headers instance. */ function normalizeHeaders(raw: unknown): Record | undefined { if (raw === null || typeof raw !== "object") return undefined; const maybeIter = (raw as { entries?: unknown }).entries; if (typeof maybeIter === "function") { try { return Object.fromEntries((raw as { entries: () => Iterable<[string, string]> }).entries()); } catch { // fall through to plain-object treatment } } return raw as Record; } /** * Adapter that takes an error thrown by an HTTP client (fetch wrapper, axios, * upstream SDK, etc.) and produces a {@link FailureKind} suitable for the * `classifyError` option of the circuit breaker. * * Recognises the common error shapes: * - `err.status` + `err.headers` + `err.body` (low-level fetch wrapper) * - `err.response.status` + `err.response.headers` + `err.response.data` (axios-style) * - `err.message` (last-resort body for keyword scan) * * Returns `undefined` when the error doesn't carry enough information to * classify, so the breaker can decide what to do without a kind tag. * * Companion to issue #2100 follow-up. */ export function classify429FromError(err: unknown): FailureKind | undefined { if (err === null || typeof err !== "object") return undefined; const e = err as Record; let status: number | undefined; let headers: Record | undefined; let body: unknown; if (typeof e.status === "number") { status = e.status; } if (typeof e.statusCode === "number" && status === undefined) { status = e.statusCode; } if (e.response && typeof e.response === "object") { const resp = e.response as Record; if (typeof resp.status === "number" && status === undefined) { status = resp.status; } if (resp.headers && typeof resp.headers === "object") { headers = normalizeHeaders(resp.headers); } if (resp.data !== undefined) { body = resp.data; } else if (typeof resp.body !== "undefined") { body = resp.body; } } if (headers === undefined && e.headers && typeof e.headers === "object") { headers = normalizeHeaders(e.headers); } if (body === undefined) { if (typeof e.body !== "undefined") { body = e.body; } else if (typeof e.message === "string") { body = e.message; } } if (typeof status !== "number") return undefined; return classify429({ status, headers, body }); }