fix(resilience): lock permanently retired models instead of short backoff (Gemini ban prevention) (#11762)

Root-caused via a real Gemini-ban incident log: deprecated-model 404/410s (e.g. gemini-2.5-flash "no longer available to new users") fell through checkFallbackError's generic transient-cooldown branch, so combo/auto-routing kept re-selecting a permanently dead model every cooldown window forever — the hammering that got the account flagged as abusive.

Fix: `MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS` + `isModelPermanentlyUnavailable()` classify these as a 24h lockout instead, surfaced via `quotaResetHintMs` so combo's per-request model-lockout honors it in full.

Validated: 6/6 new tests + 133/133 existing accountFallback/error-classification tests, no regressions. Thanks for tracing this end-to-end with real production logs!

open-sse/services/accountFallback.ts

@@ -249,6 +249,28 @@ export const OAUTH_INVALID_TOKEN_SIGNALS = [
   "invalid credentials",
 ];
 
+// A model that upstream has permanently retired — Gemini's deprecated-model 404
+// ("This model models/gemini-2.5-flash is no longer available to new users...")
+// and Fireworks/OpenAI-compatible "end of life" 410s ("has reached its end of
+// life ... and is no longer available") — will 404/410 on EVERY future request;
+// no cooldown short enough to retry soon is ever correct. Without this check
+// these fall through to the generic "all other errors" branch at the bottom of
+// checkFallbackError, which only applies a short (seconds-to-minutes) transient
+// cooldown, so combo/auto-routing keeps re-selecting the dead model roughly
+// every cooldown window, forever — wasted upstream calls that, at volume, look
+// like abusive traffic to the provider (observed: a Gemini free-tier key
+// retried `gemini-2.5-flash`/`gemini-2.5-flash-lite` every ~15-45 minutes for a
+// full day). Matched independent of MODEL_ACCESS_DENIED_PATTERNS below because
+// those only fire for status 400; this needs to catch the far more common
+// 404/410 status a retired model actually returns.
+export const MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS = [
+  /\bno longer available\b/i,
+  /\bno longer supported\b/i,
+  /\bhas reached (?:its |the )?end.?of.?life\b/i,
+  /\bmodel[\s\S]{0,40}?\b(?:deprecated|retired|discontinued|decommissioned)\b/i,
+  /\b(?:deprecated|retired|discontinued|decommissioned)[\s\S]{0,40}?\bmodel\b/i,
+];
+
 // Context overflow patterns — the prompt exceeds the model's maximum context length.
 // Different providers phrase this differently. Used to decide whether a 400 error
 // should trigger combo fallback (a different model may have a larger context window).
@@ -421,6 +443,15 @@ export function isCreditsExhausted(errorText: string): boolean {
   return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig));
 }
 
+/**
+ * Returns true if the response body indicates the requested model has been
+ * permanently retired by the provider (see MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS).
+ */
+export function isModelPermanentlyUnavailable(errorText: string): boolean {
+  const text = String(errorText || "");
+  return MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS.some((p) => p.test(text));
+}
+
 /**
  * T11: Returns true if response body indicates OAuth token is invalid/expired.
  * This is different from permanent account deactivation - token refresh can recover.
@@ -1755,6 +1786,28 @@ export function checkFallbackError(
       };
     }
 
+    // A retired model (Gemini deprecated-model 404, Fireworks/etc. end-of-life 410)
+    // will fail identically on every future request — lock it for a long, fixed
+    // window instead of falling through to the generic transient-error branch's
+    // short backoff, which would otherwise keep re-selecting a permanently dead
+    // model roughly every cooldown window, all day, hammering the provider with
+    // guaranteed-to-fail requests (see MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS).
+    // `quotaResetHintMs` flows into combo.ts's per-request model-lockout as an
+    // upstream-verified reset, so it is honored in full and not clamped to the
+    // normal ~20min model-lockout ceiling.
+    if (
+      (status === HTTP_STATUS.NOT_FOUND || status === HTTP_STATUS.GONE) &&
+      isModelPermanentlyUnavailable(errorStr)
+    ) {
+      const cooldownMs = 24 * 60 * 60 * 1000; // 24h
+      return {
+        shouldFallback: true,
+        cooldownMs,
+        reason: "not_found",
+        quotaResetHintMs: cooldownMs,
+      };
+    }
+
     // Gemini-specific check — MUST run before isCreditsExhausted/
     // isDailyQuotaExhausted/the generic text classifier below: Gemini's free-
     // tier 429 boilerplate literally says "You exceeded your current quota,

stryker.conf.json

@@ -247,6 +247,7 @@
       "tests/unit/format-provider-error-cause.test.ts",
       "tests/unit/forwarded-header-budget.test.ts",
       "tests/unit/fusion-vision-panel-3378.test.ts",
+      "tests/unit/gemini-deprecated-model-lockout.test.ts",
       "tests/unit/gemini-web-capabilities-9356.test.ts",
       "tests/unit/gemini-web-missing-browser-3516.test.ts",
       "tests/unit/grok-cli-oauth.test.ts",

tests/unit/gemini-deprecated-model-lockout.test.ts

@@ -0,0 +1,72 @@
+/**
+ * Regression guard: a permanently retired model (Gemini's deprecated-model 404,
+ * "This model models/gemini-2.5-flash is no longer available to new users...",
+ * or a Fireworks/etc. end-of-life 410) must get a long, fixed lockout instead of
+ * falling through to the generic transient-error branch's short backoff.
+ *
+ * Without this classification, combo/auto-routing kept re-selecting the dead
+ * model roughly every cooldown window (a few minutes, escalating to ~20min max)
+ * for as long as the model stayed in the registry — all day, every day — sending
+ * guaranteed-to-fail requests to the provider. At volume this looks like abusive
+ * traffic and was implicated in a Gemini free-tier API key getting banned.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { checkFallbackError, isModelPermanentlyUnavailable } =
+  await import("../../open-sse/services/accountFallback.ts");
+
+const GEMINI_DEPRECATED_404 =
+  "[404]: This model models/gemini-2.5-flash is no longer available to new users. " +
+  "Please update your code to use models/gemini-3.6-flash for the latest features and improvements.";
+
+const END_OF_LIFE_410 =
+  '[410]: {"type":"about:blank","title":"Gone","status":410,"detail":"The model ' +
+  "'minimaxai/minimax-m2.7' has reached its end of life on 2026-07-27T00:00:00Z and is no longer available.\"}\n";
+
+test("isModelPermanentlyUnavailable matches Gemini's deprecated-model phrasing", () => {
+  assert.equal(isModelPermanentlyUnavailable(GEMINI_DEPRECATED_404), true);
+});
+
+test("isModelPermanentlyUnavailable matches end-of-life phrasing", () => {
+  assert.equal(isModelPermanentlyUnavailable(END_OF_LIFE_410), true);
+});
+
+test("isModelPermanentlyUnavailable does not match an ordinary 404", () => {
+  assert.equal(
+    isModelPermanentlyUnavailable("[404]: Model not found, inaccessible, and/or not deployed"),
+    false
+  );
+});
+
+test("checkFallbackError locks a deprecated Gemini model for 24h, not a short backoff", () => {
+  const result = checkFallbackError(404, GEMINI_DEPRECATED_404, 0, "gemini-2.5-flash", "gemini");
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, "not_found");
+  assert.equal(result.cooldownMs, 24 * 60 * 60 * 1000);
+  // Feeds combo.ts's per-request model-lockout as an upstream-verified reset,
+  // so it bypasses the normal ~20min model-lockout ceiling.
+  assert.equal(result.quotaResetHintMs, 24 * 60 * 60 * 1000);
+});
+
+test("checkFallbackError locks an end-of-life model (410) for 24h too", () => {
+  const result = checkFallbackError(410, END_OF_LIFE_410, 0, "minimaxai/minimax-m2.7", "nvidia");
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, "not_found");
+  assert.equal(result.cooldownMs, 24 * 60 * 60 * 1000);
+});
+
+test("a generic 404 (not a deprecation message) still falls through to the short transient cooldown", () => {
+  const result = checkFallbackError(
+    404,
+    "[404]: Model not found, inaccessible, and/or not deployed",
+    0,
+    "some-model",
+    "openrouter"
+  );
+
+  assert.equal(result.reason, "unknown");
+  assert.notEqual(result.cooldownMs, 24 * 60 * 60 * 1000);
+});