diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 8d1c3a88a1..b4056d2e2b 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -249,6 +249,28 @@ export const OAUTH_INVALID_TOKEN_SIGNALS = [ "invalid credentials", ]; +// A model that upstream has permanently retired — Gemini's deprecated-model 404 +// ("This model models/gemini-2.5-flash is no longer available to new users...") +// and Fireworks/OpenAI-compatible "end of life" 410s ("has reached its end of +// life ... and is no longer available") — will 404/410 on EVERY future request; +// no cooldown short enough to retry soon is ever correct. Without this check +// these fall through to the generic "all other errors" branch at the bottom of +// checkFallbackError, which only applies a short (seconds-to-minutes) transient +// cooldown, so combo/auto-routing keeps re-selecting the dead model roughly +// every cooldown window, forever — wasted upstream calls that, at volume, look +// like abusive traffic to the provider (observed: a Gemini free-tier key +// retried `gemini-2.5-flash`/`gemini-2.5-flash-lite` every ~15-45 minutes for a +// full day). Matched independent of MODEL_ACCESS_DENIED_PATTERNS below because +// those only fire for status 400; this needs to catch the far more common +// 404/410 status a retired model actually returns. +export const MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS = [ + /\bno longer available\b/i, + /\bno longer supported\b/i, + /\bhas reached (?:its |the )?end.?of.?life\b/i, + /\bmodel[\s\S]{0,40}?\b(?:deprecated|retired|discontinued|decommissioned)\b/i, + /\b(?:deprecated|retired|discontinued|decommissioned)[\s\S]{0,40}?\bmodel\b/i, +]; + // Context overflow patterns — the prompt exceeds the model's maximum context length. // Different providers phrase this differently. Used to decide whether a 400 error // should trigger combo fallback (a different model may have a larger context window). @@ -421,6 +443,15 @@ export function isCreditsExhausted(errorText: string): boolean { return CREDITS_EXHAUSTED_SIGNALS.some((sig) => lower.includes(sig)); } +/** + * Returns true if the response body indicates the requested model has been + * permanently retired by the provider (see MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS). + */ +export function isModelPermanentlyUnavailable(errorText: string): boolean { + const text = String(errorText || ""); + return MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS.some((p) => p.test(text)); +} + /** * T11: Returns true if response body indicates OAuth token is invalid/expired. * This is different from permanent account deactivation - token refresh can recover. @@ -1755,6 +1786,28 @@ export function checkFallbackError( }; } + // A retired model (Gemini deprecated-model 404, Fireworks/etc. end-of-life 410) + // will fail identically on every future request — lock it for a long, fixed + // window instead of falling through to the generic transient-error branch's + // short backoff, which would otherwise keep re-selecting a permanently dead + // model roughly every cooldown window, all day, hammering the provider with + // guaranteed-to-fail requests (see MODEL_PERMANENTLY_UNAVAILABLE_PATTERNS). + // `quotaResetHintMs` flows into combo.ts's per-request model-lockout as an + // upstream-verified reset, so it is honored in full and not clamped to the + // normal ~20min model-lockout ceiling. + if ( + (status === HTTP_STATUS.NOT_FOUND || status === HTTP_STATUS.GONE) && + isModelPermanentlyUnavailable(errorStr) + ) { + const cooldownMs = 24 * 60 * 60 * 1000; // 24h + return { + shouldFallback: true, + cooldownMs, + reason: "not_found", + quotaResetHintMs: cooldownMs, + }; + } + // Gemini-specific check — MUST run before isCreditsExhausted/ // isDailyQuotaExhausted/the generic text classifier below: Gemini's free- // tier 429 boilerplate literally says "You exceeded your current quota, diff --git a/stryker.conf.json b/stryker.conf.json index 6486c81f4c..d9fb56523d 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -247,6 +247,7 @@ "tests/unit/format-provider-error-cause.test.ts", "tests/unit/forwarded-header-budget.test.ts", "tests/unit/fusion-vision-panel-3378.test.ts", + "tests/unit/gemini-deprecated-model-lockout.test.ts", "tests/unit/gemini-web-capabilities-9356.test.ts", "tests/unit/gemini-web-missing-browser-3516.test.ts", "tests/unit/grok-cli-oauth.test.ts", diff --git a/tests/unit/gemini-deprecated-model-lockout.test.ts b/tests/unit/gemini-deprecated-model-lockout.test.ts new file mode 100644 index 0000000000..c3134d52c8 --- /dev/null +++ b/tests/unit/gemini-deprecated-model-lockout.test.ts @@ -0,0 +1,72 @@ +/** + * Regression guard: a permanently retired model (Gemini's deprecated-model 404, + * "This model models/gemini-2.5-flash is no longer available to new users...", + * or a Fireworks/etc. end-of-life 410) must get a long, fixed lockout instead of + * falling through to the generic transient-error branch's short backoff. + * + * Without this classification, combo/auto-routing kept re-selecting the dead + * model roughly every cooldown window (a few minutes, escalating to ~20min max) + * for as long as the model stayed in the registry — all day, every day — sending + * guaranteed-to-fail requests to the provider. At volume this looks like abusive + * traffic and was implicated in a Gemini free-tier API key getting banned. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { checkFallbackError, isModelPermanentlyUnavailable } = + await import("../../open-sse/services/accountFallback.ts"); + +const GEMINI_DEPRECATED_404 = + "[404]: This model models/gemini-2.5-flash is no longer available to new users. " + + "Please update your code to use models/gemini-3.6-flash for the latest features and improvements."; + +const END_OF_LIFE_410 = + '[410]: {"type":"about:blank","title":"Gone","status":410,"detail":"The model ' + + "'minimaxai/minimax-m2.7' has reached its end of life on 2026-07-27T00:00:00Z and is no longer available.\"}\n"; + +test("isModelPermanentlyUnavailable matches Gemini's deprecated-model phrasing", () => { + assert.equal(isModelPermanentlyUnavailable(GEMINI_DEPRECATED_404), true); +}); + +test("isModelPermanentlyUnavailable matches end-of-life phrasing", () => { + assert.equal(isModelPermanentlyUnavailable(END_OF_LIFE_410), true); +}); + +test("isModelPermanentlyUnavailable does not match an ordinary 404", () => { + assert.equal( + isModelPermanentlyUnavailable("[404]: Model not found, inaccessible, and/or not deployed"), + false + ); +}); + +test("checkFallbackError locks a deprecated Gemini model for 24h, not a short backoff", () => { + const result = checkFallbackError(404, GEMINI_DEPRECATED_404, 0, "gemini-2.5-flash", "gemini"); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, "not_found"); + assert.equal(result.cooldownMs, 24 * 60 * 60 * 1000); + // Feeds combo.ts's per-request model-lockout as an upstream-verified reset, + // so it bypasses the normal ~20min model-lockout ceiling. + assert.equal(result.quotaResetHintMs, 24 * 60 * 60 * 1000); +}); + +test("checkFallbackError locks an end-of-life model (410) for 24h too", () => { + const result = checkFallbackError(410, END_OF_LIFE_410, 0, "minimaxai/minimax-m2.7", "nvidia"); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, "not_found"); + assert.equal(result.cooldownMs, 24 * 60 * 60 * 1000); +}); + +test("a generic 404 (not a deprecation message) still falls through to the short transient cooldown", () => { + const result = checkFallbackError( + 404, + "[404]: Model not found, inaccessible, and/or not deployed", + 0, + "some-model", + "openrouter" + ); + + assert.equal(result.reason, "unknown"); + assert.notEqual(result.cooldownMs, 24 * 60 * 60 * 1000); +});