diff --git a/changelog.d/fixes/6773-6773-nim-404.md b/changelog.d/fixes/6773-6773-nim-404.md new file mode 100644 index 0000000000..ba6d5e1543 --- /dev/null +++ b/changelog.d/fixes/6773-6773-nim-404.md @@ -0,0 +1 @@ +- fix(providers): scope nvidia NIM 404s to the single failing model instead of cooling down the whole connection (#6773) diff --git a/open-sse/config/providers/registry/nvidia/index.ts b/open-sse/config/providers/registry/nvidia/index.ts index c42f8da703..dd2516a4a1 100644 --- a/open-sse/config/providers/registry/nvidia/index.ts +++ b/open-sse/config/providers/registry/nvidia/index.ts @@ -8,6 +8,13 @@ export const nvidiaProvider: RegistryEntry = { baseUrl: "https://integrate.api.nvidia.com/v1/chat/completions", authType: "apikey", authHeader: "bearer", + // #6773: nvidia multiplexes 17 models from 9 different upstream vendors + // (z-ai/, minimaxai/, deepseek-ai/, qwen/, mistralai/, stepfun-ai/, + // moonshotai/, openai/, nvidia/) behind ONE connection — mark it passthrough + // so a single stale/renamed model's 404 locks out only that model instead + // of cooling down the whole connection (see accountFallback.ts + // hasPerModelQuota doc comment; matches modelscope/synthetic/kilo-gateway). + passthroughModels: true, models: [ // #6108: z-ai/glm-5.1 EOL'd 2026-07-02 (direct probe returns 410) — dropped. { id: "z-ai/glm-5.2", name: "GLM 5.2" }, diff --git a/stryker.conf.json b/stryker.conf.json index 7cf803c4e4..ffa4f9f17f 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -196,6 +196,7 @@ "tests/unit/model-lockout-max-cooldown.test.ts", "tests/unit/no-memory-header.test.ts", "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts", + "tests/unit/nvidia-passthrough-models-6773.test.ts", "tests/unit/oauth-providers-config.test.ts", "tests/unit/oauth-redirect-uri-mismatch.test.ts", "tests/unit/observability-fase04.test.ts", diff --git a/tests/unit/nvidia-passthrough-models-6773.test.ts b/tests/unit/nvidia-passthrough-models-6773.test.ts new file mode 100644 index 0000000000..915b541413 --- /dev/null +++ b/tests/unit/nvidia-passthrough-models-6773.test.ts @@ -0,0 +1,79 @@ +/** + * Regression test for #6773 — NVIDIA NIM models listed available:true but 404 at router. + * + * Root cause: the `nvidia` provider registry entry multiplexes many distinct + * third-party vendor models (z-ai/, minimaxai/, deepseek-ai/, qwen/, + * mistralai/, stepfun-ai/, moonshotai/, openai/, nvidia/) behind ONE base URL + * and ONE API key connection — architecturally identical to `modelscope`, + * `synthetic`, and `kilo-gateway`, which all set `passthroughModels: true` so + * that a single model's 404/429 stays scoped to that model instead of cooling + * down the whole connection (see accountFallback.ts `hasPerModelQuota` doc + * comment). Without the flag, a single upstream 404 for one (possibly + * stale/renamed) model poisons ALL nvidia models for the connection-cooldown + * duration — matching the issue's "All 17 behave the same" symptom. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const accountFallback = await import("../../open-sse/services/accountFallback.ts"); +const providerRegistry = await import("../../open-sse/config/providerRegistry.ts"); + +test("#6773: nvidia registry entry sets passthroughModels", () => { + const entry = providerRegistry.getRegistryEntry("nvidia"); + assert.equal( + entry?.passthroughModels, + true, + "nvidia multiplexes many third-party vendor models behind one connection " + + "(z-ai/, minimaxai/, deepseek-ai/, qwen/, mistralai/, stepfun-ai/, " + + "moonshotai/, openai/, nvidia/) — it should set passthroughModels: true " + + "like modelscope/synthetic/kilo-gateway, so a single stale model 404 " + + "does not cool down the whole connection for all other models" + ); +}); + +test("#6773: hasPerModelQuota('nvidia') is true, so a 404 on one nvidia model is model-scoped", () => { + assert.equal( + accountFallback.hasPerModelQuota("nvidia", "z-ai/glm-5.2"), + true, + "expected nvidia to use per-model lockout (like gemini/github/codex/compatible " + + "providers) so a 404 on one model doesn't cool down the other nvidia models" + ); +}); + +test("#6773: checkFallbackError + lockModelIfPerModelQuota scope a single-model 404 to just that model for nvidia", () => { + // A plain upstream 404 (e.g. one stale/renamed nvidia model id) falls through + // checkFallbackError's generic catch-all: shouldFallback=true with a non-zero + // connection cooldown. With hasPerModelQuota=true, lockModelIfPerModelQuota + // now scopes that cooldown to just the one failing model instead of the + // whole connection. + const result = accountFallback.checkFallbackError( + 404, + "Not Found", + 0, + "z-ai/glm-5.2", + "nvidia", + null, + null, + null + ); + assert.equal(result.shouldFallback, true, "404 triggers a connection-level fallback/cooldown"); + assert.ok( + (result.cooldownMs ?? 0) > 0, + "the connection-level cooldown is non-zero, so it also blocks the other nvidia models" + + " unless it gets scoped to just this model below" + ); + + const locked = accountFallback.lockModelIfPerModelQuota( + "nvidia", + "conn-6773", + "z-ai/glm-5.2", + "unknown", + result.cooldownMs ?? 30_000 + ); + assert.equal( + locked, + true, + "expected the 404 to be scoped to just this one model (per-model lockout), " + + "not the whole connection" + ); +});