diff --git a/changelog.d/features/6846-nvidia-nim-quota.md b/changelog.d/features/6846-nvidia-nim-quota.md new file mode 100644 index 0000000000..df3897244e --- /dev/null +++ b/changelog.d/features/6846-nvidia-nim-quota.md @@ -0,0 +1 @@ +- feat(sse): add a static local RPM budget (default 40/min, operator-overridable), per-model 429 scoping (already covered by #6773's `passthroughModels`), and a per-connection concurrency cap (default 6, operator-overridable) for the `nvidia` (NVIDIA NIM) provider, which sends no rate-limit headers and has no usage API — Phase 1 of client-side quota tracking; adaptive AIMD learning and the dashboard quota card are deferred follow-ups (#6846). diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 5586cc71ba..44091f15fb 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -1,4 +1,5 @@ { + "_rebaseline_2026_07_19_6846_nvidia_concurrency_gate": "Issue #6846 Phase 1 (nvidia NIM local RPM budget + per-model lockout + per-connection concurrency cap) own growth: open-sse/executors/default.ts 877->890 (+13 = the irreducible call-site wiring at DefaultExecutor.execute(), the only place nvidia requests dispatch through — the existing session-pool body was extracted verbatim into a new private executeWithSessionPool() so the outer execute() can wrap it in the nvidia concurrency-gate acquire/finally-release). All actual gating logic (semaphore key + cap resolution) lives in the new leaf open-sse/executors/default/nvidiaConcurrencyGate.ts (not frozen, well under cap). Covered by tests/unit/nvidia-quota-phase1.test.ts.", "_rebaseline_2026_07_18_v3849_provider_detail_wiring": "Merge campaign R2/R3 (2026-07-18): three authorized PRs each add irreducible call-site wiring to ProviderDetailPageClient.tsx — #7360 +5 (ProviderQuotaVisibilityToggle render, component extracted), #7419 +4 (NoAuthProviderControls wiring), #7062 +3 (Dahl provider hook) = 786->798. All three follow the extracted-component pattern (AgentrouterConsoleFields precedent); the frozen file only takes the wiring. Structural shrink tracked in #3501.", "_rebaseline_2026_07_18_pr7653_chat_tracker_import": "PR #7653 merge-interaction growth: release moved chat.ts to its 1796 cap while this PR adds the single side-effect import 'quotaTrackersBatch.ts' (line 130) — chat.ts IS the canonical quota-fetcher registration point (codex/bailian/deepseek/openrouter/opencode/generic all import+register there), so the +1 is irreducible call-site wiring. 1796->1797. Covered by tests/unit/{agentrouter,v0,freemodel}-quota-fetcher.test.ts.", "_rebaseline_2026_07_17_pr7653_agentrouter_console_fields": "PR #7653 own growth (missing acceptance criterion: the AgentRouter quota tracker (#6850) read providerSpecificData.consoleApiKey/newApiUserId but neither field had dashboard UI for provider agentrouter — consoleApiKey was gated to bailian-coding-plan only and newApiUserId had zero UI). AddApiKeyModal.tsx 961->967 (+6) and EditConnectionModal.tsx 1278->1286 (+8) = import + a single render call plus the newApiUserId formData init field. The actual Input rendering (both consoleApiKey reuse + the new newApiUserId field) was EXTRACTED into a new leaf src/app/(dashboard)/dashboard/providers/[id]/components/modals/AgentrouterConsoleFields.tsx (48 LOC, ` — excess requests wait in the queue instead of + * firing doomed parallel calls. + * + * No-op for every other provider and for a missing `connectionId` — returns + * `null`, and callers should skip the `finally` release when the return value + * is `null`. + */ +import * as semaphore from "../../services/rateLimitSemaphore.ts"; +import { getProviderConcurrencyCap } from "../../services/providerDefaultRateLimit.ts"; + +const NVIDIA_DEFAULT_CONCURRENCY_CAP = 6; +const NVIDIA_ACQUIRE_TIMEOUT_MS = 30_000; + +/** + * Acquire a concurrency slot for an nvidia request. Resolves to a release + * function that MUST be called (typically in a `finally`) once the request + * completes, or `null` when the gate does not apply (non-nvidia provider, or + * no connection id to scope the gate to). Rejects with a + * `SEMAPHORE_TIMEOUT`-coded error if the queue wait exceeds the timeout — + * propagates like any other executor failure (caller/combo fallback handles it). + */ +export function acquireNvidiaConcurrencySlot( + provider: string, + connectionId: string | null | undefined +): Promise<(() => void) | null> { + if (provider !== "nvidia" || !connectionId) return Promise.resolve(null); + const maxConcurrency = getProviderConcurrencyCap(provider, NVIDIA_DEFAULT_CONCURRENCY_CAP); + const key = `${provider}:${connectionId}`; + return semaphore.acquire(key, { maxConcurrency, timeoutMs: NVIDIA_ACQUIRE_TIMEOUT_MS }); +} diff --git a/open-sse/services/providerDefaultRateLimit.ts b/open-sse/services/providerDefaultRateLimit.ts index 2749a37342..5560c71e76 100644 --- a/open-sse/services/providerDefaultRateLimit.ts +++ b/open-sse/services/providerDefaultRateLimit.ts @@ -6,8 +6,10 @@ * un-throttled. For such a provider with a *documented* fixed cap, an operator can * declare a default here and a sliding window (burst-free, unlike Bottleneck's * fixed-window reservoir which refills in one burst every interval) enforces it - * proactively. The map is EMPTY by default → zero behavior change for every existing - * provider; the whole path is a no-op until an entry is added. + * proactively. `nvidia` is the first (and currently only) real entry — NVIDIA NIM's + * free tier is documented as "~40 RPM" and exposes no rate-limit headers or usage + * API (#6846 Phase 1). Every other provider still gets zero behavior change; the + * whole path is a no-op unless an entry (or a resolved override, see below) exists. * * Wired as a pre-schedule gate in `withRateLimit` (rateLimitManager.ts). Bottleneck * still applies on top — this only adds a floor for header-less providers. @@ -16,11 +18,29 @@ import { SlidingWindowLimiter, type RateLimitWindow } from "./slidingWindowLimit // Opt-in per-provider caps. Example shape (commented — add real entries as needed): // "some-headerless-provider": { requests: 60, windowMs: 60_000 }, -const PROVIDER_DEFAULT_RATE_LIMITS: Record = {}; +const PROVIDER_DEFAULT_RATE_LIMITS: Record = { + nvidia: { requests: 40, windowMs: 60_000 }, +}; + +/** #6846 Phase 1: default per-connection concurrency cap for providers whose static + * budget is enforced via `open-sse/services/rateLimitSemaphore.ts` (nvidia only, + * today). Mid-point of the issue's suggested 4-8 range. */ +export const PROVIDER_DEFAULT_CONCURRENCY_CAP: Record = { + nvidia: 6, +}; let providerDefaultOverrides: Record | null = null; const limiter = new SlidingWindowLimiter(); +/** + * Operator-configured overrides, resolved from `ResilienceSettings.providerQuotaOverrides` + * (see `src/lib/resilience/settings.ts`) by `rateLimitManager.ts` on init/refresh. + * `rpm` overrides the static sliding-window budget; `concurrency` overrides the + * default per-connection concurrency cap. A 0/missing field falls back to the + * static default for that field. + */ +let providerQuotaOverrides: Record | null = null; + /** Test hook: override the provider-default map and clear accumulated history. */ export function __setProviderDefaultRateLimitsForTests( map: Record | null @@ -29,11 +49,38 @@ export function __setProviderDefaultRateLimitsForTests( limiter.reset(); } +/** + * Set (or clear, with `null`) the operator-configured per-provider RPM/concurrency + * overrides. Called by `rateLimitManager.ts::initializeRateLimitProtection()` and + * `applyRequestQueueSettings()` after resolving `ResilienceSettings`. + */ +export function setProviderQuotaOverrides( + map: Record | null +): void { + providerQuotaOverrides = map; +} + export function getProviderDefaultRateLimit(provider: string): RateLimitWindow | undefined { if (!provider) return undefined; + const rpmOverride = providerQuotaOverrides?.[provider]?.rpm; + if (typeof rpmOverride === "number" && rpmOverride > 0) { + return { requests: rpmOverride, windowMs: 60_000 }; + } return (providerDefaultOverrides ?? PROVIDER_DEFAULT_RATE_LIMITS)[provider]; } +/** + * Resolve a provider's per-connection concurrency cap: operator override → + * `PROVIDER_DEFAULT_CONCURRENCY_CAP[provider]` → `fallback`. Used by the nvidia + * executor concurrency gate (`open-sse/executors/default/nvidiaConcurrencyGate.ts`). + */ +export function getProviderConcurrencyCap(provider: string, fallback: number): number { + const override = providerQuotaOverrides?.[provider]?.concurrency; + if (typeof override === "number" && override > 0) return override; + const staticDefault = PROVIDER_DEFAULT_CONCURRENCY_CAP[provider]; + return typeof staticDefault === "number" && staticDefault > 0 ? staticDefault : fallback; +} + /** * Consume one slot from the provider's opt-in sliding-window default. * Returns 0 when a slot was taken (proceed) or no default is configured; otherwise the diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index 5e92751b00..fe5bce878d 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -12,7 +12,7 @@ import Bottleneck from "bottleneck"; import { parseRetryAfterFromBody } from "./accountFallback.ts"; import { getProviderCategory } from "../config/providerRegistry.ts"; import { getCodexRateLimitKey } from "../executors/codex.ts"; -import { awaitProviderDefaultSlot } from "./providerDefaultRateLimit.ts"; +import { awaitProviderDefaultSlot, setProviderQuotaOverrides } from "./providerDefaultRateLimit.ts"; import { DEFAULT_RESILIENCE_SETTINGS, resolveResilienceSettings, @@ -345,6 +345,10 @@ export async function initializeRateLimits() { const [connections, settings] = await Promise.all([getProviderConnections(), getSettings()]); const resilience = resolveResilienceSettings(settings); currentRequestQueueSettings = { ...resilience.requestQueue }; + // #6846 Phase 1: operator overrides for header-less providers' static RPM + // budget + concurrency cap (nvidia today). No-op for every provider without + // an entry in either providerQuotaOverrides or PROVIDER_DEFAULT_RATE_LIMITS. + setProviderQuotaOverrides(resilience.providerQuotaOverrides); const { explicitCount, autoCount } = reconcileEnabledConnections( connections as unknown[], currentRequestQueueSettings diff --git a/src/lib/resilience/settings.ts b/src/lib/resilience/settings.ts index e5aad43d75..5995ad88a6 100644 --- a/src/lib/resilience/settings.ts +++ b/src/lib/resilience/settings.ts @@ -15,6 +15,7 @@ import { normalizeProviderCooldownSettings, normalizeQuotaPreflightSettings, normalizeStreamRecoverySettings, + normalizeProviderQuotaOverrides, } from "./settings/normalize"; // Re-export the settings shape (moved to ./settings/types) so this module's @@ -29,6 +30,7 @@ export type { ProviderCooldownSettings, QuotaPreflightSettings, StreamRecoverySettings, + ProviderQuotaOverrideSettings, ResilienceSettings, ResilienceSettingsPatch, } from "./settings/types"; @@ -140,6 +142,10 @@ export const DEFAULT_RESILIENCE_SETTINGS: ResilienceSettings = { (process.env.STREAM_RECOVERY_MIDSTREAM_ENABLED || "").trim().toLowerCase() ), }, + // #6846 Phase 1: empty by default — nvidia (and any future header-less + // provider registered in providerDefaultRateLimit.ts) uses its static + // default until an operator adds an override here. + providerQuotaOverrides: {}, }; function buildLegacyFallback(settings: JsonRecord): ResilienceSettings { @@ -233,6 +239,7 @@ function buildLegacyFallback(settings: JsonRecord): ResilienceSettings { providerCooldown: DEFAULT_RESILIENCE_SETTINGS.providerCooldown, quotaPreflight: DEFAULT_RESILIENCE_SETTINGS.quotaPreflight, streamRecovery: streamRecoveryDefaults, + providerQuotaOverrides: DEFAULT_RESILIENCE_SETTINGS.providerQuotaOverrides, }; } @@ -312,6 +319,10 @@ export function resolveResilienceSettings( current.streamRecovery, fallback.streamRecovery ), + providerQuotaOverrides: normalizeProviderQuotaOverrides( + current.providerQuotaOverrides, + fallback.providerQuotaOverrides + ), }; } @@ -359,6 +370,10 @@ export function mergeResilienceSettings( ), quotaPreflight: normalizeQuotaPreflightSettings(updates.quotaPreflight, current.quotaPreflight), streamRecovery: normalizeStreamRecoverySettings(updates.streamRecovery, current.streamRecovery), + providerQuotaOverrides: normalizeProviderQuotaOverrides( + updates.providerQuotaOverrides, + current.providerQuotaOverrides + ), }; } diff --git a/src/lib/resilience/settings/normalize.ts b/src/lib/resilience/settings/normalize.ts index 405777e9f8..43c98b0400 100644 --- a/src/lib/resilience/settings/normalize.ts +++ b/src/lib/resilience/settings/normalize.ts @@ -20,6 +20,7 @@ import type { ProviderCooldownSettings, QuotaPreflightSettings, StreamRecoverySettings, + ProviderQuotaOverrideSettings, } from "./types"; export function asRecord(value: unknown): JsonRecord { @@ -362,3 +363,38 @@ export function normalizeStreamRecoverySettings( continueMidStream: toBoolean(record.continueMidStream, fallback.continueMidStream), }; } + +// #6846 Phase 1: parse a single provider's { rpm?, concurrency? } override entry, +// dropping non-positive/non-finite fields so a malformed value falls back to that +// provider's static default rather than disabling its budget outright (0 would +// mean "no cap" for rpm, mirroring the resolveRpm/resolveMaxConcurrent convention +// elsewhere in this file). +function normalizeProviderQuotaOverrideEntry(raw: unknown): ProviderQuotaOverrideSettings | null { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null; + const record = raw as Record; + const out: ProviderQuotaOverrideSettings = {}; + const rpm = typeof record.rpm === "number" ? record.rpm : Number(record.rpm); + if (Number.isFinite(rpm) && rpm > 0) out.rpm = Math.trunc(rpm); + const concurrency = typeof record.concurrency === "number" ? record.concurrency : Number(record.concurrency); + if (Number.isFinite(concurrency) && concurrency > 0) out.concurrency = Math.trunc(concurrency); + return Object.keys(out).length > 0 ? out : null; +} + +/** + * #6846 Phase 1: per-provider RPM/concurrency override map (currently only + * consumed for `nvidia`). Accepts an explicit object or falls back; drops + * providers whose value isn't a valid override object. + */ +export function normalizeProviderQuotaOverrides( + next: unknown, + fallback: Record +): Record { + const rawProviders = asRecord(next ?? fallback); + const out: Record = {}; + for (const [provider, entry] of Object.entries(rawProviders)) { + if (!provider) continue; + const normalized = normalizeProviderQuotaOverrideEntry(entry); + if (normalized) out[provider] = normalized; + } + return out; +} diff --git a/src/lib/resilience/settings/types.ts b/src/lib/resilience/settings/types.ts index 61e2031906..0564937fc7 100644 --- a/src/lib/resilience/settings/types.ts +++ b/src/lib/resilience/settings/types.ts @@ -143,6 +143,22 @@ export interface QuotaPreflightSettings { providerWindowDefaults: Record>; } +/** + * #6846 Phase 1: per-provider operator overrides for the header-less "provider + * default" static budget (`open-sse/services/providerDefaultRateLimit.ts`) and its + * companion per-connection concurrency cap (`rateLimitSemaphore.ts`). Keyed by + * provider id (e.g. `"nvidia"`). A missing/0 field falls back to that provider's + * static default — this is a ceiling override, not a new rate-limit mechanism. + * Empty by default; only providers with a registered static default (currently + * only `nvidia`) read from this map. + */ +export interface ProviderQuotaOverrideSettings { + /** Overrides the static sliding-window requests-per-minute budget. */ + rpm?: number; + /** Overrides the static per-connection concurrency cap. */ + concurrency?: number; +} + export interface StreamRecoverySettings { /** * Opt-in transparent recovery of truncated upstream streams (free-claude-code port). @@ -174,6 +190,7 @@ export interface ResilienceSettings { providerCooldown: ProviderCooldownSettings; quotaPreflight: QuotaPreflightSettings; streamRecovery: StreamRecoverySettings; + providerQuotaOverrides: Record; } export interface ResilienceSettingsPatch { @@ -186,4 +203,5 @@ export interface ResilienceSettingsPatch { providerCooldown?: Partial; quotaPreflight?: Partial; streamRecovery?: Partial; + providerQuotaOverrides?: Record>; } diff --git a/tests/unit/nvidia-quota-phase1.test.ts b/tests/unit/nvidia-quota-phase1.test.ts new file mode 100644 index 0000000000..ef09091d01 --- /dev/null +++ b/tests/unit/nvidia-quota-phase1.test.ts @@ -0,0 +1,237 @@ +/** + * #6846 Phase 1 — nvidia NIM static local RPM budget, per-model 429 lockout, and + * per-connection concurrency cap. Mocked/synthetic fixtures only — no live network + * calls against NVIDIA NIM (matches the issue's own stated test plan). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +import { + acquireProviderDefaultSlot, + getProviderDefaultRateLimit, + getProviderConcurrencyCap, + setProviderQuotaOverrides, + __setProviderDefaultRateLimitsForTests, +} from "../../open-sse/services/providerDefaultRateLimit.ts"; +import { + hasPerModelQuota, + lockModelIfPerModelQuota, + isModelLocked, +} from "../../open-sse/services/accountFallback.ts"; +import * as semaphore from "../../open-sse/services/rateLimitSemaphore.ts"; +import { acquireNvidiaConcurrencySlot } from "../../open-sse/executors/default/nvidiaConcurrencyGate.ts"; + +// ── RPM budget ────────────────────────────────────────────────────────────── + +test("nvidia has a real default RPM budget of 40/60s (source-of-truth constant)", () => { + // No test override active — resolves the REAL PROVIDER_DEFAULT_RATE_LIMITS entry, + // so an accidental future edit to the literal 40 is caught here too. + const cfg = getProviderDefaultRateLimit("nvidia"); + assert.ok(cfg, "nvidia must have a registered default"); + assert.equal(cfg?.requests, 40); + assert.equal(cfg?.windowMs, 60_000); +}); + +test("nvidia default RPM budget: 41st request in-window is throttled", () => { + __setProviderDefaultRateLimitsForTests({ nvidia: { requests: 40, windowMs: 60_000 } }); + try { + for (let i = 0; i < 40; i++) { + assert.equal( + acquireProviderDefaultSlot("nvidia", "conn-rpm"), + 0, + `request ${i + 1}/40 proceeds` + ); + } + const wait = acquireProviderDefaultSlot("nvidia", "conn-rpm"); + assert.ok(wait > 0, "41st request in the same 60s window is throttled"); + assert.ok(wait <= 60_000, "wait never exceeds the window"); + } finally { + __setProviderDefaultRateLimitsForTests(null); + } +}); + +test("per-provider RPM override takes precedence over the static default", () => { + setProviderQuotaOverrides({ nvidia: { rpm: 2 } }); + __setProviderDefaultRateLimitsForTests(null); + try { + const cfg = getProviderDefaultRateLimit("nvidia"); + assert.equal(cfg?.requests, 2, "override replaces the static 40 default"); + assert.equal(acquireProviderDefaultSlot("nvidia", "conn-override"), 0); + assert.equal(acquireProviderDefaultSlot("nvidia", "conn-override"), 0); + const wait = acquireProviderDefaultSlot("nvidia", "conn-override"); + assert.ok(wait > 0, "3rd request exceeds the overridden 2/min budget"); + } finally { + setProviderQuotaOverrides(null); + } +}); + +test("RPM override does not mutate PROVIDER_DEFAULT_RATE_LIMITS for other providers", () => { + setProviderQuotaOverrides({ nvidia: { rpm: 2 } }); + try { + assert.equal(getProviderDefaultRateLimit("openai"), undefined, "unrelated provider untouched"); + } finally { + setProviderQuotaOverrides(null); + } +}); + +// ── Per-model 429 lockout ──────────────────────────────────────────────────── +// +// Plan Step 3 turned out to be already satisfied: issue #6773 (landed after this +// plan's research) already set `passthroughModels: true` on nvidia's provider +// registry entry (open-sse/config/providers/registry/nvidia/index.ts) so that a +// single model's 404/429 stays scoped to that model — `hasPerModelQuota()` +// already returns `true` for nvidia via its generic +// `getPassthroughProviders().has(provider)` branch, with zero new code needed +// here. This PR does NOT add a redundant `provider === "nvidia"` branch; these +// tests are a regression guard proving the requirement holds. + +test('hasPerModelQuota("nvidia", ) returns true (via #6773 passthroughModels)', () => { + assert.equal(hasPerModelQuota("nvidia"), true); + assert.equal(hasPerModelQuota("nvidia", "kimi-k2.6"), true); + assert.equal(hasPerModelQuota("nvidia", "glm-4.7"), true); +}); + +test("429 on model A does not lock model B on the same nvidia connection", () => { + const connectionId = "nvidia-conn-lockout"; + const locked = lockModelIfPerModelQuota( + "nvidia", + connectionId, + "kimi-k2.6", + "429 rate limit", + 5_000 + ); + assert.equal(locked, true, "lockModelIfPerModelQuota locks the offending model"); + assert.equal(isModelLocked("nvidia", connectionId, "kimi-k2.6"), true); + assert.equal( + isModelLocked("nvidia", connectionId, "glm-4.7"), + false, + "a different model on the same connection stays unlocked" + ); +}); + +// ── Per-connection concurrency cap ────────────────────────────────────────── + +test("concurrency cap queues the 3rd concurrent request at cap=2, releases correctly", async () => { + setProviderQuotaOverrides({ nvidia: { concurrency: 2 } }); + try { + const release1 = await acquireNvidiaConcurrencySlot("nvidia", "conn-cap"); + const release2 = await acquireNvidiaConcurrencySlot("nvidia", "conn-cap"); + assert.ok(release1 && release2, "first two acquisitions resolve immediately"); + + let thirdResolved = false; + const thirdPromise = acquireNvidiaConcurrencySlot("nvidia", "conn-cap").then((release) => { + thirdResolved = true; + return release; + }); + + // Give the microtask queue a tick — the 3rd acquire must still be queued. + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(thirdResolved, false, "3rd concurrent request queues instead of rejecting"); + + release1?.(); + const release3 = await thirdPromise; + assert.equal(thirdResolved, true, "3rd request resolves once a slot frees"); + release2?.(); + release3?.(); + } finally { + setProviderQuotaOverrides(null); + } +}); + +test("concurrency cap is scoped per-connection, not shared across two nvidia connections", async () => { + setProviderQuotaOverrides({ nvidia: { concurrency: 1 } }); + try { + const releaseA = await acquireNvidiaConcurrencySlot("nvidia", "conn-a"); + assert.ok(releaseA, "connection A gets its slot"); + + let releaseBResolved = false; + const releaseBPromise = acquireNvidiaConcurrencySlot("nvidia", "conn-b").then((release) => { + releaseBResolved = true; + return release; + }); + const releaseB = await releaseBPromise; + assert.equal( + releaseBResolved, + true, + "a different connection is unaffected by connection A's saturated gate" + ); + + releaseA?.(); + releaseB?.(); + } finally { + setProviderQuotaOverrides(null); + } +}); + +test("the concurrency gate is a no-op for every non-nvidia provider", async () => { + const release = await acquireNvidiaConcurrencySlot("openai", "some-connection"); + assert.equal(release, null, "non-nvidia providers never allocate a semaphore key"); +}); + +test("the concurrency gate is a no-op without a connectionId", async () => { + const release = await acquireNvidiaConcurrencySlot("nvidia", null); + assert.equal(release, null, "no connectionId to scope the gate to"); +}); + +test("getProviderConcurrencyCap resolves override -> static default -> fallback", () => { + setProviderQuotaOverrides(null); + assert.equal( + getProviderConcurrencyCap("nvidia", 99), + 6, + "nvidia's registered static default is 6 (mid-point of the issue's 4-8 range)" + ); + assert.equal( + getProviderConcurrencyCap("some-unregistered-provider", 99), + 99, + "unregistered providers fall back to the caller-supplied default" + ); + setProviderQuotaOverrides({ nvidia: { concurrency: 3 } }); + try { + assert.equal(getProviderConcurrencyCap("nvidia", 99), 3, "override wins over the static default"); + } finally { + setProviderQuotaOverrides(null); + } +}); + +test("semaphore.getStats reflects nvidia's per-connection gate key", async () => { + setProviderQuotaOverrides({ nvidia: { concurrency: 1 } }); + try { + const release = await acquireNvidiaConcurrencySlot("nvidia", "conn-stats"); + assert.ok(release); + const stats = semaphore.getStats(); + assert.ok( + Object.prototype.hasOwnProperty.call(stats, "nvidia:conn-stats"), + "gate key is scoped as `${provider}:${connectionId}`" + ); + release?.(); + } finally { + setProviderQuotaOverrides(null); + } +}); + +// ── Failure-mode separation regression guard ──────────────────────────────── + +test("nvidia 429 does not trip the provider circuit breaker (unchanged 408/500/502/503/504-only classification)", () => { + // This PR does not touch src/sse/handlers/chat.ts or the circuit breaker — this + // is a documentation-alignment guard proving Phase 1 didn't accidentally widen + // PROVIDER_BREAKER_FAILURE_STATUSES to include 429 (which would collapse the + // per-model lockout this PR adds into a whole-connection/provider outage). + const chatHandlerPath = path.join( + process.cwd(), + "src", + "sse", + "handlers", + "chat.ts" + ); + const source = fs.readFileSync(chatHandlerPath, "utf8"); + const match = source.match(/PROVIDER_BREAKER_FAILURE_STATUSES\s*=\s*new Set\(\[([^\]]+)\]\)/); + assert.ok(match, "PROVIDER_BREAKER_FAILURE_STATUSES declaration found"); + const statuses = match![1].split(",").map((s) => Number(s.trim())); + assert.deepEqual( + statuses.sort((a, b) => a - b), + [408, 500, 502, 503, 504] + ); + assert.ok(!statuses.includes(429), "429 must never trip the provider circuit breaker"); +}); diff --git a/tests/unit/resilience-settings-normalize-split.test.ts b/tests/unit/resilience-settings-normalize-split.test.ts index 5e1ab767c8..218f8c1cdc 100644 --- a/tests/unit/resilience-settings-normalize-split.test.ts +++ b/tests/unit/resilience-settings-normalize-split.test.ts @@ -81,6 +81,7 @@ describe("resilience/settings normalize split-guard", () => { "connectionCooldown", "providerBreaker", "providerCooldown", + "providerQuotaOverrides", "quotaPreflight", "quotaShareConcurrencyLimit", "requestQueue",