diff --git a/changelog.d/fixes/7940-gemini-429-model-lockout.md b/changelog.d/fixes/7940-gemini-429-model-lockout.md new file mode 100644 index 0000000000..cac486caf9 --- /dev/null +++ b/changelog.d/fixes/7940-gemini-429-model-lockout.md @@ -0,0 +1 @@ +- fix(sse): parse Gemini 429 RetryInfo.retryDelay / "please retry in Ns" so model lockout honors the short upstream hint instead of falling back to a ~19h until-midnight cooldown (#7940) diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index bd1dbabe66..7617e5054d 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -36,7 +36,11 @@ import { getCodexModelScope } from "../config/codexQuotaScopes.ts"; import { getQuotaScopedModelForProvider } from "./antigravityQuotaFamily.ts"; import { isRpdExhausted, isRpmExhausted } from "./geminiRateLimitTracker.ts"; import { setConnectionRateLimitUntil } from "@/lib/db/providers"; -import { parseRetryHintFromJsonBody } from "./retryAfterJson.ts"; +import { + parseRetryHintFromJsonBody, + parseDelayString, + MAX_SHORT_RETRY_HINT_MS, +} from "./retryAfterJson.ts"; import { isSubscriptionQuotaText, buildSubscriptionQuotaFallback, @@ -1036,24 +1040,8 @@ export function parseRetryAfterFromBody(responseBody: unknown): { return { retryAfterMs: null, reason }; } -/** - * Parse delay strings like "33s", "2m", "1h", "1500ms" - */ -function parseDelayString(value: unknown): number | null { - if (!value) return null; - const str = String(value).trim(); - const msMatch = /^(\d+)\s*ms$/i.exec(str); - if (msMatch) return Number.parseInt(msMatch[1], 10); - const secMatch = /^(\d+)\s*s$/i.exec(str); - if (secMatch) return Number.parseInt(secMatch[1], 10) * 1000; - const minMatch = /^(\d+)\s*m$/i.exec(str); - if (minMatch) return Number.parseInt(minMatch[1], 10) * 60 * 1000; - const hrMatch = /^(\d+)\s*h$/i.exec(str); - if (hrMatch) return Number.parseInt(hrMatch[1], 10) * 3600 * 1000; - // Bare number → seconds - const num = Number.parseInt(str, 10); - return Number.isNaN(num) ? null : num * 1000; -} +// parseDelayString now lives in ./retryAfterJson.ts (shared with parseRetryHintFromJsonBody's +// Gemini RetryInfo.retryDelay parsing, #7940) — see the import at the top of this file. // T07: parse retry time from error text body with combined "XhYmZs" format. export function parseRetryFromErrorText(errorText: unknown): number | null { @@ -1063,6 +1051,14 @@ export function parseRetryFromErrorText(errorText: unknown): number | null { const bodyHintMs = parseRetryHintFromJsonBody(msg, MAX_PROVIDER_COOLDOWN_MS); if (bodyHintMs !== null) return bodyHintMs; + // Gemini free-tier text fallback (no parseable JSON details present): + // "Please retry in 26.660853464s." Short throttle hint — capped independently of + // MAX_PROVIDER_COOLDOWN_MS, mirroring the JSON RetryInfo.retryDelay cap (#7940). + const pleaseRetryMs = parseDelayString(/please retry in\s+([\d.]+\s*s)/i.exec(msg)?.[1]); + if (pleaseRetryMs !== null && pleaseRetryMs > 0) { + return Math.min(pleaseRetryMs, MAX_SHORT_RETRY_HINT_MS); + } + // Issue #2321: parse embedded absolute ISO retry timestamps. const isoMatch = /\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i.exec( diff --git a/open-sse/services/retryAfterJson.ts b/open-sse/services/retryAfterJson.ts index 1426fc3447..1c4b89f30a 100644 --- a/open-sse/services/retryAfterJson.ts +++ b/open-sse/services/retryAfterJson.ts @@ -18,6 +18,47 @@ function futureTimestampMs(value: unknown, maxMs: number): number | null { return waitMs > 0 ? Math.min(waitMs, maxMs) : null; } +// RetryInfo.retryDelay / "please retry in Ns" are short per-request throttling +// hints (Gemini free-tier RPM/TPM), not long-lived quota resets like Antigravity's +// "Resets in 160h" — cap them independently of the caller's maxMs so a malformed or +// adversarial upstream value cannot masquerade as a multi-day reset (#7940). +export const MAX_SHORT_RETRY_HINT_MS = 24 * 60 * 60 * 1000; // 24h + +/** + * Parse delay strings like "33s", "26.660853464s", "2m", "1h", "1500ms", or a bare + * number of seconds. Shared by `parseRetryAfterFromBody` (rateLimitManager wiring) + * and `parseRetryHintFromJsonBody` (model-lockout wiring) so both honor the same + * upstream `RetryInfo.retryDelay` grammar (#7940). + */ +export function parseDelayString(value: unknown): number | null { + if (!value) return null; + const str = String(value).trim(); + const msMatch = /^(\d+(?:\.\d+)?)\s*ms$/i.exec(str); + if (msMatch) return Math.round(Number.parseFloat(msMatch[1])); + const secMatch = /^(\d+(?:\.\d+)?)\s*s$/i.exec(str); + if (secMatch) return Math.round(Number.parseFloat(secMatch[1]) * 1000); + const minMatch = /^(\d+(?:\.\d+)?)\s*m$/i.exec(str); + if (minMatch) return Math.round(Number.parseFloat(minMatch[1]) * 60 * 1000); + const hrMatch = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str); + if (hrMatch) return Math.round(Number.parseFloat(hrMatch[1]) * 3600 * 1000); + // Bare number → seconds + const num = Number.parseFloat(str); + return Number.isFinite(num) ? Math.round(num * 1000) : null; +} + +// Gemini/Google RPC 429 bodies embed the short throttle hint as +// `error.details[].{"@type": ".../google.rpc.RetryInfo", "retryDelay": "26s"}`. +function retryInfoDetailsMs(details: unknown): number | null { + for (const detail of Array.isArray(details) ? details : []) { + const detailRecord = objectRecord(detail); + const type = String(detailRecord["@type"] ?? ""); + if (!type.includes("RetryInfo")) continue; + const ms = parseDelayString(detailRecord.retryDelay); + if (ms !== null && ms > 0) return Math.min(ms, MAX_SHORT_RETRY_HINT_MS); + } + return null; +} + /** * Parse Retry-After hints from a 429 JSON response body. Providers use both * top-level and nested `error` fields for ISO timestamps and millisecond values. @@ -34,6 +75,9 @@ export function parseRetryHintFromJsonBody(body: string, maxMs: number): number if (!Object.keys(root).length) return null; const errorObj = objectRecord(root.error); + const retryInfoMs = retryInfoDetailsMs(errorObj.details ?? root.details); + if (retryInfoMs !== null) return retryInfoMs; + const isoHint = futureTimestampMs(errorObj.retryAfter ?? root.retryAfter, maxMs); if (isoHint !== null) return isoHint; diff --git a/stryker.conf.json b/stryker.conf.json index e93b99e852..4926210178 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -70,6 +70,7 @@ "tests/unit/auto-combo-context-advertising.test.ts", "tests/unit/auto-combo-engine.test.ts", "tests/unit/auto-combo-scoring-clamp.test.ts", + "tests/unit/bug-7940-gemini-retrydelay.test.ts", "tests/unit/build/check-circular-deps.test.ts", "tests/unit/cache-sweeps.test.ts", "tests/unit/cc-bridge-openai-image-7777.test.ts", diff --git a/tests/unit/bug-7940-gemini-retrydelay.test.ts b/tests/unit/bug-7940-gemini-retrydelay.test.ts new file mode 100644 index 0000000000..c86cfdec37 --- /dev/null +++ b/tests/unit/bug-7940-gemini-retrydelay.test.ts @@ -0,0 +1,71 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + parseRetryFromErrorText, + recordModelLockoutFailure, + clearAllModelLockouts, + getModelLockoutInfo, +} from "../../open-sse/services/accountFallback.ts"; + +const GEMINI_429_BODY = JSON.stringify({ + error: { + code: 429, + message: + "You exceeded your current quota, please check your plan and billing details. " + + "Please retry in 26.660853464s.", + status: "RESOURCE_EXHAUSTED", + details: [ + { + "@type": "type.googleapis.com/google.rpc.QuotaFailure", + violations: [ + { quotaMetric: "generativelanguage.googleapis.com/generate_content_free_tier_requests" }, + ], + }, + { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "26s" }, + ], + }, +}); + +test("parseRetryFromErrorText extracts Gemini RetryInfo.retryDelay (26s), not null", () => { + const parsedMs = parseRetryFromErrorText(GEMINI_429_BODY); + assert.notEqual(parsedMs, null, "expected the 26s RetryInfo hint to be parsed, got null"); + assert.ok(parsedMs! >= 25_000 && parsedMs! <= 28_000, `expected ~26000ms, got ${parsedMs}ms`); +}); + +test("recordModelLockoutFailure quota_exhausted with a short upstream hint must NOT fall back to midnight", () => { + clearAllModelLockouts(); + const provider = "gemini", + connectionId = "conn-7940", + model = "gemini-2.5-flash"; + const quotaResetHintMs = parseRetryFromErrorText(GEMINI_429_BODY) ?? undefined; + const result = recordModelLockoutFailure(provider, connectionId, model, "quota_exhausted", 429, 0, null, { + exactCooldownMs: quotaResetHintMs ?? null, + }); + const oneHourMs = 60 * 60 * 1000; + assert.ok(result.cooldownMs < oneHourMs, `expected ~26s cooldown, got ${result.cooldownMs}ms`); + clearAllModelLockouts(); +}); + +test("parseRetryFromErrorText falls back to 'please retry in Ns' text when no JSON details are present", () => { + const plainText = + "429 RESOURCE_EXHAUSTED: You exceeded your current quota. Please retry in 12.5s."; + const parsedMs = parseRetryFromErrorText(plainText); + assert.notEqual(parsedMs, null, "expected the 12.5s text hint to be parsed, got null"); + assert.ok(parsedMs! >= 12_000 && parsedMs! <= 13_500, `expected ~12500ms, got ${parsedMs}ms`); +}); + +test("getModelLockoutInfo reflects the short lockout, not a multi-hour one", () => { + clearAllModelLockouts(); + const provider = "gemini", + connectionId = "conn-7940b", + model = "gemini-2.5-flash"; + const quotaResetHintMs = parseRetryFromErrorText(GEMINI_429_BODY) ?? undefined; + recordModelLockoutFailure(provider, connectionId, model, "quota_exhausted", 429, 0, null, { + exactCooldownMs: quotaResetHintMs ?? null, + }); + const info = getModelLockoutInfo(provider, connectionId, model); + assert.ok(info, "expected an active lockout entry"); + const oneHourMs = 60 * 60 * 1000; + assert.ok(info!.remainingMs < oneHourMs, `expected <1h remaining, got ${info!.remainingMs}ms`); + clearAllModelLockouts(); +});