fix(providers): classify per-model-quota 403 and DEGRADED 400 as model-unhealthy in checkFallbackError (#8247, #8248) (#8323)

changelog.d/fixes/8247-8248-accountfallback-model-unhealthy.md

@@ -0,0 +1 @@
+- fix(providers): classify per-model-quota 403 and DEGRADED 400 as model-unhealthy in checkFallbackError (#8247, #8248)

config/quality/file-size-baseline.json

@@ -193,7 +193,8 @@
     "_rebaseline_2026_06_27_5193_antigravity_basered": "Base-red (pre-existing release drift, fast-gate PR->release skips check:file-size): accountFallback.ts 1773->1777 and src/app/api/providers/[id]/test/route.ts 924->940 were already over their frozen caps on release/v3.8.39 independent of any antigravity change. Owner chose to rebaseline (keep the documented issue-reference comments #1846/#1449/#347 etc.) rather than accept the contributor comment-stripping in #5200/#5198. Reverted #5200 to restore the comments; bumped these two frozen caps to the actual base sizes. No logic change.",
     "_rebaseline_2026_07_22_8213_gemini_tpm_quota_cooldown_wait": "PR #8213 (hartmark, fix/gemini-tpm-quota-cooldown-wait) own growth: open-sse/services/accountFallback.ts 1857->1932 on the merged tip (release 1892 incl #8050 +35, plus this PR own growth +40); measured against the PR own merge-base was 1857->1898 (+41 — the release tip separately carries an unrelated +34 from #8050's antigravity 404 model-not-found lockout scoping, which this PR's branch does not include and this entry does not cover). Own growth is the Gemini TPM-ceiling classification + cooldown-wait wiring feeding into the combo cooldown-wait state machine (rate-limit wedge recovery) introduced by this PR's commit series. Irreducible additions at the existing account-fallback/model-lockout chokepoint. Covered by the PR's own gemini-rate-limit-tracker and TPM-ceiling benchmark test additions.",
     "_rebaseline_2026_07_23_8252_combo_400_advance": "#8252 (@RaviTharuma) own growth: accountFallback.ts 1932->1940 (+8) + combo.ts 3604->3630 (+26) — advance combo on model-scoped 400s wrapped as invalid/Bad-Request. Irreducible wiring at existing account-fallback + combo dispatch chokepoints. Covered by combo-model-scoped-400-advance.test.ts.",
-    "open-sse/services/accountFallback.ts": 1940,
+    "_rebaseline_2026_07_23_8247_8248_model_unhealthy": "#8247+#8248 own growth: accountFallback.ts 1940->1941 (+1, irreducible import statement only — the substantive #8248 DEGRADED-pattern classifier was extracted into open-sse/config/errorConfig.ts, which has ample headroom, instead of growing this frozen file; #8247's fix is a single existing-line condition change, net zero lines). Scoping the credits-exhausted 403/429 branch to isCompatibleProvider() (per-model-quota openai/anthropic-compatible-* nicknames) so it stays model-scoped instead of terminalling the whole connection, and classifying NVIDIA NIM 'Function ... DEGRADED' 400 bodies as model-access-denied instead of a raw passthrough 400. Covered by tests/unit/8247-accountfallback-model-unhealthy.test.ts and tests/unit/8248-accountfallback-nvidia-degraded.test.ts.",
+    "open-sse/services/accountFallback.ts": 1941,
     "open-sse/services/adobeFireflyClient.ts": 1958,
     "open-sse/services/batchProcessor.ts": 915,
     "open-sse/services/browserBackedChat.ts": 850,

open-sse/config/errorConfig.ts

@@ -200,6 +200,19 @@ export function findMatchingErrorRule(statusCode: number, message: unknown): Err
   return matchErrorRuleByText(message) || matchErrorRuleByStatus(statusCode);
 }
 
+// #8248: NVIDIA NIM function-state DEGRADED — some NIM deployments signal a non-standard
+// HTTP 400 whose body reports the backing "function" is DEGRADED (e.g. `Function id "<uuid>"
+// submitted for inference is DEGRADED`) instead of a clean model-not-found/5xx. Bounded
+// lookahead ({0,80}) — ReDoS-safe, no nested quantifiers.
+const NIM_FUNCTION_DEGRADED_PATTERNS = [
+  /\bfunction\b[\s\S]{0,80}?\bDEGRADED\b/i,
+  /\bDEGRADED\b[\s\S]{0,80}?\bfunction\b/i,
+];
+
+export function isNimFunctionDegraded(errorText: string): boolean {
+  return NIM_FUNCTION_DEGRADED_PATTERNS.some((p) => p.test(errorText));
+}
+
 export interface ServiceSupervisorCooldown {
   shouldFallback: true;
   cooldownMs: number;

open-sse/services/accountFallback.ts

@@ -12,6 +12,7 @@ import {
   matchErrorRuleByText,
   matchErrorRuleByStatus,
   serviceSupervisorCooldown,
+  isNimFunctionDegraded,
 } from "../config/errorConfig.ts";
 import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts";
 import * as rot from "./rotationConfig.ts";
@@ -1519,8 +1520,8 @@ export function checkFallbackError(
       }
     }
 
-    // T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit
-    if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) {
+    // T10 (sub2api #1169) + #8247: credits/quota exhausted; *-compatible-* nicknames stay model-scoped.
+    if (shouldUseQuotaSignal && isCreditsExhausted(errorStr) && !isCompatibleProvider(provider)) {
       return {
         shouldFallback: true,
         cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown
@@ -1697,8 +1698,8 @@ export function checkFallbackError(
     const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr));
     const isParamValidation = PARAM_VALIDATION_PATTERNS.some((p) => p.test(errorStr));
     const isModelAccessDenied = isModelAccessDeniedStructured || matchesModelAccessPattern;
-
-    if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied) {
+    const isNimDegraded = isNimFunctionDegraded(errorStr);
+    if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied || isNimDegraded) {
       return {
         shouldFallback: true,
         cooldownMs: 0,

tests/unit/8247-accountfallback-model-unhealthy.test.ts

@@ -0,0 +1,44 @@
+// Regression guard for #8247: checkFallbackError() must not classify a 403
+// insufficient_quota body as connection-wide creditsExhausted when the
+// provider is a per-model-quota provider (e.g. openai-compatible-*). The
+// whole point of hasPerModelQuota() is to keep such failures model-scoped
+// instead of terminalling the entire connection.
+import test from "node:test";
+import assert from "node:assert/strict";
+import { checkFallbackError, hasPerModelQuota } from "../../open-sse/services/accountFallback.ts";
+
+const PROVIDER = "openai-compatible-cegp";
+const MODEL = "gpt-5.6-luna";
+const UPSTREAM_BODY = JSON.stringify({
+  error: {
+    code: "insufficient_quota",
+    type: "insufficient_quota",
+    message: "You have exceeded your quota, reset after 24s",
+  },
+});
+
+test("#8247: per-model-quota provider 403 insufficient_quota is NOT connection-wide creditsExhausted", () => {
+  assert.equal(hasPerModelQuota(PROVIDER, MODEL), true);
+  const result = checkFallbackError(403, UPSTREAM_BODY, 0, MODEL, PROVIDER);
+  assert.ok(
+    !result.creditsExhausted,
+    "per-model-quota providers must not terminal the whole connection on a per-model 403 " +
+      `(got creditsExhausted=${result.creditsExhausted})`
+  );
+  assert.equal(result.shouldFallback, true, "the failure must still be fallback-worthy");
+  assert.equal(
+    result.reason,
+    "quota_exhausted",
+    "should still be classified as quota-exhausted, just model-scoped instead of connection-wide"
+  );
+});
+
+test("#8247: non-per-model-quota provider (plain openai apikey) still terminals on 403 insufficient_quota", () => {
+  assert.equal(hasPerModelQuota("openai", MODEL), false);
+  const result = checkFallbackError(403, UPSTREAM_BODY, 0, MODEL, "openai");
+  assert.equal(
+    result.creditsExhausted,
+    true,
+    "original sub2api-ported behavior for single-model-per-connection providers must not regress"
+  );
+});

tests/unit/8248-accountfallback-nvidia-degraded.test.ts

@@ -0,0 +1,35 @@
+// Regression guard for #8248: checkFallbackError() must classify NVIDIA NIM's
+// "Function <uuid> ... DEGRADED" function-state 400 body as fallback-worthy
+// (model-unhealthy), instead of falling through to the generic
+// shouldFallback:false/UNKNOWN branch.
+import test from "node:test";
+import assert from "node:assert/strict";
+import { checkFallbackError } from "../../open-sse/services/accountFallback.ts";
+
+test("#8248: nvidia NIM DEGRADED function-state 400 is classified as fallback-worthy (model unhealthy)", () => {
+  const body =
+    'Function id "d290f1ee-6c54-4b01-90e6-d701748f0851" submitted for inference is DEGRADED';
+  const res = checkFallbackError(400, body, 0, null, "nvidia");
+  assert.equal(
+    res.shouldFallback,
+    true,
+    `expected DEGRADED NIM function-state 400 to be classified as fallback-worthy, got shouldFallback=${res.shouldFallback} reason=${res.reason}`
+  );
+});
+
+test("#8248: unrelated generic 400 body is still not fallback-worthy (no regression)", () => {
+  const res = checkFallbackError(400, "some unrelated generic 400 body text", 0, null, "nvidia");
+  assert.equal(res.shouldFallback, false);
+});
+
+test("#8248: existing malformed-request 400 branch still wins over the new DEGRADED pattern", () => {
+  const res = checkFallbackError(
+    400,
+    "messages must alternate between user and assistant",
+    0,
+    null,
+    "nvidia"
+  );
+  assert.equal(res.shouldFallback, true);
+  assert.equal(res.reason, "model_capacity");
+});