fix(classify): honor upstream retry windows on Gemini free-tier 429s (#9513)

Merge-train validated (tip 6ce4effef8). Vitest failures confirmed as base-red (#9679).

src/shared/utils/classify429.ts

@@ -123,14 +123,123 @@ export function looksLikeQuotaExhausted(body: unknown): boolean {
   return QUOTA_PATTERNS.some((pat) => pat.test(text));
 }
 
+/**
+ * A declared upstream retry window at or beyond this is treated as
+ * long-period exhaustion. One hour mirrors the circuit breaker's
+ * `quota_exhausted` cooldown bucket (`cooldownByKind`, wired in
+ * src/sse/handlers/chat.ts, chatHelpers.ts and
+ * open-sse/services/accountFallback.ts): the long bucket is only the right
+ * lock when the upstream's own window is at least that long.
+ */
+const QUOTA_SCALE_RETRY_DELAY_SECONDS = 3600;
+
+/**
+ * Quota signals that stay terminal no matter what retry hint accompanies
+ * them. Credits/billing exhaustion does not clear on a timer, so a short
+ * upstream hint must never downgrade these to a 60s retry loop.
+ */
+const TERMINAL_QUOTA_PATTERNS: ReadonlyArray<RegExp> = [
+  /INSUFFICIENT_G1_CREDITS_BALANCE/i,
+  /credit.*exhaust/i,
+  /out of credits/i,
+  /billing.*cap/i,
+  /insufficient.*quota/i,
+  /individual quota reached/i,
+  /enable overages/i,
+  /daily free allocation/i,
+];
+
+/**
+ * Parse an upstream delay string ("38s", "26.66s", "1500ms", "2m", "1h",
+ * or a bare number of seconds) into seconds.
+ *
+ * Deliberately mirrors `parseDelayString` in
+ * open-sse/services/retryAfterJson.ts (#7940) rather than importing it:
+ * open-sse already imports this module (accountFallback.ts), so the
+ * reverse import would close a dependency cycle. Keep the two grammars in
+ * step when either changes.
+ */
+function parseDelaySeconds(value: unknown): number | null {
+  if (!value) return null;
+  const str = String(value).trim();
+  const ms = /^(\d+(?:\.\d+)?)\s*ms$/i.exec(str);
+  if (ms) return Number.parseFloat(ms[1]) / 1000;
+  const sec = /^(\d+(?:\.\d+)?)\s*s$/i.exec(str);
+  if (sec) return Number.parseFloat(sec[1]);
+  const min = /^(\d+(?:\.\d+)?)\s*m$/i.exec(str);
+  if (min) return Number.parseFloat(min[1]) * 60;
+  const hr = /^(\d+(?:\.\d+)?)\s*h$/i.exec(str);
+  if (hr) return Number.parseFloat(hr[1]) * 3600;
+  const bare = Number.parseFloat(str);
+  return Number.isFinite(bare) ? bare : null;
+}
+
+/**
+ * Upstream-declared retry window in seconds, when the 429 carries one.
+ *
+ * Google APIs (Gemini `generativelanguage`, Vertex) attach a
+ * `google.rpc.RetryInfo` detail whose `retryDelay` Duration states exactly
+ * how long the throttle lasts, and repeat the same hint in the human
+ * message ("Please retry in 38.922534355s"). Gemini free-tier
+ * per-minute/per-token 429s open with the same "You exceeded your current
+ * quota, please check your plan and billing details" preamble as genuine
+ * long-window exhaustion, so `QUOTA_PATTERNS` cannot tell them apart —
+ * even the PerDay-named `quotaId` ships retryDelay values of ~30-50s
+ * (#9504). The declared window is the authoritative signal.
+ *
+ * Both carriers are read because the two live call paths deliver different
+ * shapes: `accountFallback` classifies the parsed body (details intact),
+ * while `chat.ts` classifies `result.rawMessage`, which
+ * `parseUpstreamError` has already reduced to `error.message` text.
+ * Structural matching keeps an unrelated `retryDelay` key from triggering
+ * the hint; the text form is anchored on Google's exact phrasing, matching
+ * the precedent in accountFallback's cooldown parser.
+ */
+function upstreamRetryDelaySeconds(body: unknown): number | null {
+  let root: unknown = body;
+  if (typeof body === "string") {
+    const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(body);
+    if (phrase) return Number.parseFloat(phrase[1]);
+    try {
+      root = JSON.parse(body);
+    } catch {
+      return null;
+    }
+  }
+  if (root === null || typeof root !== "object") return null;
+  const error = (root as { error?: unknown }).error;
+  const errorRecord =
+    error !== null && typeof error === "object" ? (error as Record<string, unknown>) : {};
+  const details = errorRecord.details ?? (root as Record<string, unknown>).details;
+  for (const detail of Array.isArray(details) ? details : []) {
+    if (detail === null || typeof detail !== "object") continue;
+    const entry = detail as Record<string, unknown>;
+    if (!String(entry["@type"] ?? "").includes("RetryInfo")) continue;
+    const seconds = parseDelaySeconds(entry.retryDelay);
+    if (seconds !== null && seconds >= 0) return seconds;
+  }
+  const message = errorRecord.message;
+  if (typeof message === "string") {
+    const phrase = /please retry in (\d+(?:\.\d+)?)\s*s/i.exec(message);
+    if (phrase) return Number.parseFloat(phrase[1]);
+  }
+  return null;
+}
+
 /**
  * Classify a 429 (or any) response into a `FailureKind`.
  *
  * Decision order:
  * 1. status !== 429 → `"transient"` (don't pretend to know more than
  *    the caller does about non-429 failures).
- * 2. body matches a quota keyword → `"quota_exhausted"`.
- * 3. otherwise → `"rate_limit"` (default for 429 — even without
+ * 2. body carries a terminal credits/billing signal → `"quota_exhausted"`
+ *    regardless of any retry hint: those do not clear on a timer.
+ * 3. body declares a sub-hour retry window → `"rate_limit"` even when
+ *    generic quota keywords match: the upstream said the throttle clears
+ *    in seconds, so the long lockout bucket would overshoot its own reset
+ *    by 60-360x (#9504).
+ * 4. body matches a quota keyword → `"quota_exhausted"`.
+ * 5. otherwise → `"rate_limit"` (default for 429 — even without
  *    Retry-After, a 429 is per definition a rate-limit signal).
  *
  * @param response - the upstream response with status, optional headers,
@@ -143,6 +252,14 @@ export function classify429(response: {
   body?: unknown;
 }): FailureKind {
   if (response.status !== 429) return "transient";
+  const text = bodyToText(response.body);
+  if (text && TERMINAL_QUOTA_PATTERNS.some((pat) => pat.test(text))) {
+    return "quota_exhausted";
+  }
+  const declaredDelay = upstreamRetryDelaySeconds(response.body);
+  if (declaredDelay !== null && declaredDelay < QUOTA_SCALE_RETRY_DELAY_SECONDS) {
+    return "rate_limit";
+  }
   if (looksLikeQuotaExhausted(response.body)) return "quota_exhausted";
   return "rate_limit";
 }

tests/unit/classify429.test.ts

@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
 import {
   classify429,
   looksLikeQuotaExhausted,
+  classify429FromError,
   parseRetryAfter,
   retryAfterFromResponse,
   type FailureKind,
@@ -194,8 +195,14 @@ test("classify429: Modal-hosted endpoint 'usage limit reached' body returns 'quo
     "quota_exhausted"
   );
   // Trailing punctuation/whitespace must still match.
-  assert.equal(classify429({ status: 429, body: { error: "usage limit reached." } }), "quota_exhausted");
-  assert.equal(classify429({ status: 429, body: { error: "usage limit reached " } }), "quota_exhausted");
+  assert.equal(
+    classify429({ status: 429, body: { error: "usage limit reached." } }),
+    "quota_exhausted"
+  );
+  assert.equal(
+    classify429({ status: 429, body: { error: "usage limit reached " } }),
+    "quota_exhausted"
+  );
 });
 
 test("classify429: qualified transient 'usage limit reached' messages stay rate_limit", () => {
@@ -277,3 +284,155 @@ test("retryAfterFromResponse: case-insensitive header lookup", () => {
   assert.equal(retryAfterFromResponse({ headers: {} }), null);
   assert.equal(retryAfterFromResponse({}), null);
 });
+
+// --- Gemini free-tier 429s carrying google.rpc.RetryInfo (#9504) ---
+
+/** Real captured Gemini free-tier 429 (issue #9504), parameterized by quotaId/delay. */
+function geminiFreeTier429(quotaId: string, retryDelay: string) {
+  return {
+    error: {
+      code: 429,
+      message:
+        "You exceeded your current quota, please check your plan and billing details. " +
+        "For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. " +
+        "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, " +
+        "limit: 15, model: gemini-3.5-flash-lite\nPlease retry in 38.922534355s.",
+      status: "RESOURCE_EXHAUSTED",
+      details: [
+        {
+          "@type": "type.googleapis.com/google.rpc.QuotaFailure",
+          violations: [
+            {
+              quotaMetric: "generativelanguage.googleapis.com/generate_content_free_tier_requests",
+              quotaId,
+              quotaValue: "15",
+            },
+          ],
+        },
+        { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay },
+      ],
+    },
+  };
+}
+
+test("classify429: Gemini free-tier 429 with a short RetryInfo window is a rate limit", () => {
+  // The generic "exceeded your current quota ... check your plan" preamble
+  // matches three QUOTA_PATTERNS, but Google's own RetryInfo says the window
+  // clears in seconds. Every quotaId variant captured in #9504 ships a short
+  // retryDelay, including the confusingly day-named one.
+  const cases = [
+    ["GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "38s"],
+    ["GenerateRequestsPerDayPerProjectPerModel-FreeTier", "29s"],
+    ["GenerateContentInputTokensPerModelPerMinute-FreeTier", "0s"],
+    ["GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "38.922534355s"],
+  ] as const;
+  for (const [quotaId, retryDelay] of cases) {
+    const body = geminiFreeTier429(quotaId, retryDelay);
+    assert.equal(
+      classify429({ status: 429, body }),
+      "rate_limit",
+      `${quotaId} retryDelay=${retryDelay}`
+    );
+  }
+});
+
+test("classify429FromError: the production message-only shape is a rate limit", () => {
+  // This is the shape the live path actually delivers: parseUpstreamError
+  // reduces the upstream body to error.message, and chat.ts classifies
+  // classify429FromError({ status, message }). The RetryInfo details are
+  // already gone by then, so the hint must be read from Google's phrasing.
+  const message =
+    "You exceeded your current quota, please check your plan and billing details. " +
+    "For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits.\n" +
+    "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, " +
+    "limit: 15, model: gemini-3.5-flash-lite\nPlease retry in 38.922534355s.";
+  assert.equal(classify429FromError({ status: 429, message }), "rate_limit");
+  assert.equal(classify429({ status: 429, body: message }), "rate_limit");
+});
+
+test("classify429: short RetryInfo window wins for string bodies too", () => {
+  // The account-fallback path classifies the parsed body, so the same
+  // payload may arrive as pre-stringified JSON.
+  const body = JSON.stringify(
+    geminiFreeTier429("GenerateRequestsPerMinutePerProjectPerModel-FreeTier", "14s")
+  );
+  assert.equal(classify429({ status: 429, body }), "rate_limit");
+});
+
+test("classify429: the bare RetryInfo @type and non-second units are honored", () => {
+  // Repo fixtures carry the short "@type": "google.rpc.RetryInfo" form, and
+  // the shared delay grammar (#7940) accepts ms/m/h as well as seconds.
+  const cases = [
+    ["google.rpc.RetryInfo", "45s", "rate_limit"],
+    ["type.googleapis.com/google.rpc.RetryInfo", "1500ms", "rate_limit"],
+    ["type.googleapis.com/google.rpc.RetryInfo", "30m", "rate_limit"],
+    ["type.googleapis.com/google.rpc.RetryInfo", "3h", "quota_exhausted"],
+  ] as const;
+  for (const [type, retryDelay, expected] of cases) {
+    const body = {
+      error: {
+        message: "You exceeded your current quota, please check your plan and billing details.",
+        details: [{ "@type": type, retryDelay }],
+      },
+    };
+    assert.equal(classify429({ status: 429, body }), expected, `${type} ${retryDelay}`);
+  }
+});
+
+test("classify429: a terminal credits signal is never downgraded by a retry hint", () => {
+  // Credits/billing exhaustion does not clear on a timer, so a short
+  // upstream hint must not flip it into a 60s retry loop.
+  const cases = [
+    "Individual quota reached. Contact your administrator to enable overages. Resets in 164h27m24s.",
+    "INSUFFICIENT_G1_CREDITS_BALANCE",
+    "Out of credits - top up your account.",
+    "you have used up your daily free allocation of 10,000 neurons",
+  ];
+  for (const message of cases) {
+    const body = {
+      error: {
+        message,
+        details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "20s" }],
+      },
+    };
+    assert.equal(classify429({ status: 429, body }), "quota_exhausted", message.slice(0, 40));
+  }
+});
+
+test("classify429: an hours-scale RetryInfo window keeps the quota classification", () => {
+  const body = geminiFreeTier429("GenerateRequestsPerDayPerProjectPerModel-FreeTier", "7200s");
+  assert.equal(classify429({ status: 429, body }), "quota_exhausted");
+});
+
+test("classify429: quota keywords with no retry hint at all stay quota exhausted", () => {
+  // Neither a RetryInfo detail nor Google's "Please retry in Ns" phrasing:
+  // with no declared window there is nothing to contradict the keywords.
+  const body = {
+    error: {
+      code: 429,
+      message:
+        "You exceeded your current quota, please check your plan and billing details. " +
+        "* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests.",
+      status: "RESOURCE_EXHAUSTED",
+      details: [
+        {
+          "@type": "type.googleapis.com/google.rpc.QuotaFailure",
+          violations: [{ quotaId: "GenerateRequestsPerDayPerProjectPerModel-FreeTier" }],
+        },
+      ],
+    },
+  };
+  assert.equal(classify429({ status: 429, body }), "quota_exhausted");
+});
+
+test("classify429: retryDelay outside a RetryInfo detail is ignored", () => {
+  // The field name alone must not trigger the short-window path when it is
+  // not an upstream google.rpc.RetryInfo declaration.
+  const body = {
+    error: {
+      message: "You exceeded your current quota, please check your plan and billing details.",
+      retryDelay: "10s",
+    },
+  };
+  assert.equal(classify429({ status: 429, body }), "quota_exhausted");
+});