From 7202654f81577eadc7b0e8ba7620a2dd76066e7e Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 24 Jul 2026 09:36:34 -0300 Subject: [PATCH] fix(providers): classify per-model-quota 403 and DEGRADED 400 as model-unhealthy in checkFallbackError (#8247, #8248) (#8323) --- ...47-8248-accountfallback-model-unhealthy.md | 1 + config/quality/file-size-baseline.json | 3 +- open-sse/config/errorConfig.ts | 13 ++++++ open-sse/services/accountFallback.ts | 9 ++-- ...47-accountfallback-model-unhealthy.test.ts | 44 +++++++++++++++++++ ...48-accountfallback-nvidia-degraded.test.ts | 35 +++++++++++++++ 6 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 changelog.d/fixes/8247-8248-accountfallback-model-unhealthy.md create mode 100644 tests/unit/8247-accountfallback-model-unhealthy.test.ts create mode 100644 tests/unit/8248-accountfallback-nvidia-degraded.test.ts diff --git a/changelog.d/fixes/8247-8248-accountfallback-model-unhealthy.md b/changelog.d/fixes/8247-8248-accountfallback-model-unhealthy.md new file mode 100644 index 0000000000..42288526f4 --- /dev/null +++ b/changelog.d/fixes/8247-8248-accountfallback-model-unhealthy.md @@ -0,0 +1 @@ +- fix(providers): classify per-model-quota 403 and DEGRADED 400 as model-unhealthy in checkFallbackError (#8247, #8248) diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 3991a2188e..0ff32a4ea2 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -193,7 +193,8 @@ "_rebaseline_2026_06_27_5193_antigravity_basered": "Base-red (pre-existing release drift, fast-gate PR->release skips check:file-size): accountFallback.ts 1773->1777 and src/app/api/providers/[id]/test/route.ts 924->940 were already over their frozen caps on release/v3.8.39 independent of any antigravity change. Owner chose to rebaseline (keep the documented issue-reference comments #1846/#1449/#347 etc.) rather than accept the contributor comment-stripping in #5200/#5198. Reverted #5200 to restore the comments; bumped these two frozen caps to the actual base sizes. No logic change.", "_rebaseline_2026_07_22_8213_gemini_tpm_quota_cooldown_wait": "PR #8213 (hartmark, fix/gemini-tpm-quota-cooldown-wait) own growth: open-sse/services/accountFallback.ts 1857->1932 on the merged tip (release 1892 incl #8050 +35, plus this PR own growth +40); measured against the PR own merge-base was 1857->1898 (+41 — the release tip separately carries an unrelated +34 from #8050's antigravity 404 model-not-found lockout scoping, which this PR's branch does not include and this entry does not cover). Own growth is the Gemini TPM-ceiling classification + cooldown-wait wiring feeding into the combo cooldown-wait state machine (rate-limit wedge recovery) introduced by this PR's commit series. Irreducible additions at the existing account-fallback/model-lockout chokepoint. Covered by the PR's own gemini-rate-limit-tracker and TPM-ceiling benchmark test additions.", "_rebaseline_2026_07_23_8252_combo_400_advance": "#8252 (@RaviTharuma) own growth: accountFallback.ts 1932->1940 (+8) + combo.ts 3604->3630 (+26) — advance combo on model-scoped 400s wrapped as invalid/Bad-Request. Irreducible wiring at existing account-fallback + combo dispatch chokepoints. Covered by combo-model-scoped-400-advance.test.ts.", - "open-sse/services/accountFallback.ts": 1940, + "_rebaseline_2026_07_23_8247_8248_model_unhealthy": "#8247+#8248 own growth: accountFallback.ts 1940->1941 (+1, irreducible import statement only — the substantive #8248 DEGRADED-pattern classifier was extracted into open-sse/config/errorConfig.ts, which has ample headroom, instead of growing this frozen file; #8247's fix is a single existing-line condition change, net zero lines). Scoping the credits-exhausted 403/429 branch to isCompatibleProvider() (per-model-quota openai/anthropic-compatible-* nicknames) so it stays model-scoped instead of terminalling the whole connection, and classifying NVIDIA NIM 'Function ... DEGRADED' 400 bodies as model-access-denied instead of a raw passthrough 400. Covered by tests/unit/8247-accountfallback-model-unhealthy.test.ts and tests/unit/8248-accountfallback-nvidia-degraded.test.ts.", + "open-sse/services/accountFallback.ts": 1941, "open-sse/services/adobeFireflyClient.ts": 1958, "open-sse/services/batchProcessor.ts": 915, "open-sse/services/browserBackedChat.ts": 850, diff --git a/open-sse/config/errorConfig.ts b/open-sse/config/errorConfig.ts index f5ac29fde3..d3e520cccf 100644 --- a/open-sse/config/errorConfig.ts +++ b/open-sse/config/errorConfig.ts @@ -200,6 +200,19 @@ export function findMatchingErrorRule(statusCode: number, message: unknown): Err return matchErrorRuleByText(message) || matchErrorRuleByStatus(statusCode); } +// #8248: NVIDIA NIM function-state DEGRADED — some NIM deployments signal a non-standard +// HTTP 400 whose body reports the backing "function" is DEGRADED (e.g. `Function id "" +// submitted for inference is DEGRADED`) instead of a clean model-not-found/5xx. Bounded +// lookahead ({0,80}) — ReDoS-safe, no nested quantifiers. +const NIM_FUNCTION_DEGRADED_PATTERNS = [ + /\bfunction\b[\s\S]{0,80}?\bDEGRADED\b/i, + /\bDEGRADED\b[\s\S]{0,80}?\bfunction\b/i, +]; + +export function isNimFunctionDegraded(errorText: string): boolean { + return NIM_FUNCTION_DEGRADED_PATTERNS.some((p) => p.test(errorText)); +} + export interface ServiceSupervisorCooldown { shouldFallback: true; cooldownMs: number; diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 6e8bb22a44..8f0ca7d8f3 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -12,6 +12,7 @@ import { matchErrorRuleByText, matchErrorRuleByStatus, serviceSupervisorCooldown, + isNimFunctionDegraded, } from "../config/errorConfig.ts"; import { getProviderErrorRuleMatch } from "../config/providerErrorRules.ts"; import * as rot from "./rotationConfig.ts"; @@ -1519,8 +1520,8 @@ export function checkFallbackError( } } - // T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit - if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) { + // T10 (sub2api #1169) + #8247: credits/quota exhausted; *-compatible-* nicknames stay model-scoped. + if (shouldUseQuotaSignal && isCreditsExhausted(errorStr) && !isCompatibleProvider(provider)) { return { shouldFallback: true, cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown @@ -1697,8 +1698,8 @@ export function checkFallbackError( const isMalformed = MALFORMED_REQUEST_PATTERNS.some((p) => p.test(errorStr)); const isParamValidation = PARAM_VALIDATION_PATTERNS.some((p) => p.test(errorStr)); const isModelAccessDenied = isModelAccessDeniedStructured || matchesModelAccessPattern; - - if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied) { + const isNimDegraded = isNimFunctionDegraded(errorStr); + if (isOverflow || isMalformed || isParamValidation || isModelAccessDenied || isNimDegraded) { return { shouldFallback: true, cooldownMs: 0, diff --git a/tests/unit/8247-accountfallback-model-unhealthy.test.ts b/tests/unit/8247-accountfallback-model-unhealthy.test.ts new file mode 100644 index 0000000000..d70d3c00b9 --- /dev/null +++ b/tests/unit/8247-accountfallback-model-unhealthy.test.ts @@ -0,0 +1,44 @@ +// Regression guard for #8247: checkFallbackError() must not classify a 403 +// insufficient_quota body as connection-wide creditsExhausted when the +// provider is a per-model-quota provider (e.g. openai-compatible-*). The +// whole point of hasPerModelQuota() is to keep such failures model-scoped +// instead of terminalling the entire connection. +import test from "node:test"; +import assert from "node:assert/strict"; +import { checkFallbackError, hasPerModelQuota } from "../../open-sse/services/accountFallback.ts"; + +const PROVIDER = "openai-compatible-cegp"; +const MODEL = "gpt-5.6-luna"; +const UPSTREAM_BODY = JSON.stringify({ + error: { + code: "insufficient_quota", + type: "insufficient_quota", + message: "You have exceeded your quota, reset after 24s", + }, +}); + +test("#8247: per-model-quota provider 403 insufficient_quota is NOT connection-wide creditsExhausted", () => { + assert.equal(hasPerModelQuota(PROVIDER, MODEL), true); + const result = checkFallbackError(403, UPSTREAM_BODY, 0, MODEL, PROVIDER); + assert.ok( + !result.creditsExhausted, + "per-model-quota providers must not terminal the whole connection on a per-model 403 " + + `(got creditsExhausted=${result.creditsExhausted})` + ); + assert.equal(result.shouldFallback, true, "the failure must still be fallback-worthy"); + assert.equal( + result.reason, + "quota_exhausted", + "should still be classified as quota-exhausted, just model-scoped instead of connection-wide" + ); +}); + +test("#8247: non-per-model-quota provider (plain openai apikey) still terminals on 403 insufficient_quota", () => { + assert.equal(hasPerModelQuota("openai", MODEL), false); + const result = checkFallbackError(403, UPSTREAM_BODY, 0, MODEL, "openai"); + assert.equal( + result.creditsExhausted, + true, + "original sub2api-ported behavior for single-model-per-connection providers must not regress" + ); +}); diff --git a/tests/unit/8248-accountfallback-nvidia-degraded.test.ts b/tests/unit/8248-accountfallback-nvidia-degraded.test.ts new file mode 100644 index 0000000000..fcf3a736e3 --- /dev/null +++ b/tests/unit/8248-accountfallback-nvidia-degraded.test.ts @@ -0,0 +1,35 @@ +// Regression guard for #8248: checkFallbackError() must classify NVIDIA NIM's +// "Function ... DEGRADED" function-state 400 body as fallback-worthy +// (model-unhealthy), instead of falling through to the generic +// shouldFallback:false/UNKNOWN branch. +import test from "node:test"; +import assert from "node:assert/strict"; +import { checkFallbackError } from "../../open-sse/services/accountFallback.ts"; + +test("#8248: nvidia NIM DEGRADED function-state 400 is classified as fallback-worthy (model unhealthy)", () => { + const body = + 'Function id "d290f1ee-6c54-4b01-90e6-d701748f0851" submitted for inference is DEGRADED'; + const res = checkFallbackError(400, body, 0, null, "nvidia"); + assert.equal( + res.shouldFallback, + true, + `expected DEGRADED NIM function-state 400 to be classified as fallback-worthy, got shouldFallback=${res.shouldFallback} reason=${res.reason}` + ); +}); + +test("#8248: unrelated generic 400 body is still not fallback-worthy (no regression)", () => { + const res = checkFallbackError(400, "some unrelated generic 400 body text", 0, null, "nvidia"); + assert.equal(res.shouldFallback, false); +}); + +test("#8248: existing malformed-request 400 branch still wins over the new DEGRADED pattern", () => { + const res = checkFallbackError( + 400, + "messages must alternate between user and assistant", + 0, + null, + "nvidia" + ); + assert.equal(res.shouldFallback, true); + assert.equal(res.reason, "model_capacity"); +});