From 2d49f1c743c429c3aa06fe2de02fe2e6b762bb05 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sun, 9 Aug 2026 09:47:17 -0300 Subject: [PATCH] maint: follow-up cherry-pick fix-in-place #9833 (conflict-resolved fallback) (#9899) * fix(nvidia): keep 410 failures model-scoped * test: register NVIDIA 410 regression for mutation coverage * chore: preserve Stryker config formatting * fix(auth): reduce file size Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: Zartharas <1402357+Zartharas@users.noreply.github.com> --- src/sse/services/auth.ts | 69 +------- stryker.conf.json | 1 + tests/unit/nvidia-410-model-scope.test.ts | 196 ++++++++++++++++++++++ 3 files changed, 205 insertions(+), 61 deletions(-) create mode 100644 tests/unit/nvidia-410-model-scope.test.ts diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index b20d3d5d2e..3243a0316a 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -83,7 +83,6 @@ import { getResource404Bypass } from "./requestResourceHealth"; import * as log from "../utils/logger"; import { fisherYatesShuffle, getNextFromDeckSync } from "@/shared/utils/shuffleDeck"; import { readHeaderValue, type AuthRequestHeaders } from "./headerReader.ts"; - type JsonRecord = Record; interface RecoverableConnectionState { connectionId: string; @@ -94,7 +93,6 @@ interface RecoverableConnectionState { lastErrorType?: string | null; lastErrorSource?: string | null; } - interface CredentialSelectionOptions { allowSuppressedConnections?: boolean; allowRateLimitedConnections?: boolean; @@ -104,14 +102,12 @@ interface CredentialSelectionOptions { sessionKey?: string | null; sessionAffinityTtlMs?: number | null; } - interface CooldownInspectionState { connection: ProviderConnectionView; connectionCooldownMs: number | null; codexScopeCooldownMs: number | null; retryableModelCooldownMs: number | null; } - const MIN_QUOTA_THRESHOLD_PERCENT = 1; const MAX_QUOTA_THRESHOLD_PERCENT = 100; const NON_RETRYABLE_MODEL_LOCKOUT_REASONS = new Set(["not_found", "not_found_local"]); @@ -119,25 +115,20 @@ const NON_RETRYABLE_MODEL_LOCKOUT_REASONS = new Set(["not_found", "not_found_loc // this base. Real upstream Retry-After hints still win — they flow through // `exactCooldownMs` (usedUpstreamRetryHint), not this base. (#5222) const ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS = 30_000; - function asRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } - function toStringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } - function toNullableNumber(value: unknown): number | null { if (value === null || value === undefined) return null; const parsed = toNumber(value, Number.NaN); return Number.isFinite(parsed) ? parsed : null; } - function toBooleanOrDefault(value: unknown, fallback: boolean): boolean { return typeof value === "boolean" ? value : fallback; } - function normalizeSessionKey(value: unknown, prefix: string): string | null { if (typeof value !== "string" || value.trim().length === 0) return null; const trimmed = value.trim(); @@ -146,7 +137,6 @@ function normalizeSessionKey(value: unknown, prefix: string): string | null { } return `${prefix}:sha256:${createHash("sha256").update(trimmed).digest("hex")}`; } - function extractTextForSessionHash(value: unknown): string | null { if (typeof value === "string") return value; if (Array.isArray(value)) { @@ -164,7 +154,6 @@ function extractTextForSessionHash(value: unknown): string | null { if (value && typeof value === "object") return JSON.stringify(value); return null; } - function getFirstInputText(body: unknown): string | null { const record = asRecord(body); if (record.input !== undefined) { @@ -189,7 +178,6 @@ function getFirstInputText(body: unknown): string | null { return null; } - export function extractSessionAffinityKey( body: unknown, headers?: Headers | { get?: (name: string) => string | null } | null @@ -216,7 +204,6 @@ export function extractSessionAffinityKey( if (!inputText || inputText.trim().length === 0) return null; return `input:sha256:${createHash("sha256").update(inputText.slice(0, 4096)).digest("hex")}`; } - function getCodexLimitPolicy(providerSpecificData: JsonRecord): { use5h: boolean; useWeekly: boolean; @@ -227,13 +214,11 @@ function getCodexLimitPolicy(providerSpecificData: JsonRecord): { useWeekly: toBooleanOrDefault(policy.useWeekly, true), }; } - interface QuotaLimitPolicy { enabled: boolean; thresholdPercent: number; windows: string[]; } - interface QuotaCacheView { quotas?: Record< string, @@ -243,7 +228,6 @@ interface QuotaCacheView { } >; } - function normalizeQuotaThreshold( value: unknown, fallback = DEFAULT_QUOTA_THRESHOLD_PERCENT @@ -251,17 +235,14 @@ function normalizeQuotaThreshold( const parsed = toNumber(value, fallback); return Math.min(MAX_QUOTA_THRESHOLD_PERCENT, Math.max(MIN_QUOTA_THRESHOLD_PERCENT, parsed)); } - function normalizeWindowName(windowName: unknown): string | null { if (typeof windowName !== "string") return null; const normalized = windowName.trim().toLowerCase(); return normalized.length > 0 ? normalized : null; } - function uniqueWindows(windows: string[]): string[] { return [...new Set(windows)]; } - function normalizeCodexWindowName(windowName: unknown): string | null { if (typeof windowName !== "string") return null; const normalized = windowName.trim().toLowerCase(); @@ -273,7 +254,6 @@ function normalizeCodexWindowName(windowName: unknown): string | null { } return toCodexBaseQuotaWindowName(normalized); } - function applyCodexWindowPolicy(rawWindows: string[], providerSpecificData: JsonRecord): string[] { const codexPolicy = getCodexLimitPolicy(providerSpecificData); const normalizedRaw = rawWindows.map(normalizeCodexWindowName).filter(Boolean) as string[]; @@ -291,7 +271,6 @@ function applyCodexWindowPolicy(rawWindows: string[], providerSpecificData: Json return uniqueWindows(windows); } - function getCodexScopeRateLimitedUntil( providerSpecificData: JsonRecord, model: string | null @@ -302,7 +281,6 @@ function getCodexScopeRateLimitedUntil( const value = scopeMap[scope]; return typeof value === "string" && value.trim().length > 0 ? value : null; } - function isCodexScopeUnavailable( connection: ProviderConnectionView, model: string | null @@ -311,7 +289,6 @@ function isCodexScopeUnavailable( if (!until) return false; return new Date(until).getTime() > Date.now(); } - function getEarliestCodexScopeRateLimitedUntil( connections: ProviderConnectionView[], model: string | null @@ -332,11 +309,9 @@ function getEarliestCodexScopeRateLimitedUntil( return earliest; } - function normalizeStatus(value: string | null): string { return (value || "").trim().toLowerCase(); } - function isTerminalConnectionStatus(connection: ProviderConnectionView): boolean { const status = normalizeStatus(connection.testStatus); return status === "credits_exhausted" || status === "banned" || status === "expired"; @@ -354,7 +329,6 @@ function isRecoverableCookieAuth401( resolveProviderId(provider) in WEB_COOKIE_PROVIDERS ); } - function resolveTerminalConnectionStatus( status: number, result: { permanent?: boolean; creditsExhausted?: boolean }, @@ -381,7 +355,6 @@ function resolveTerminalConnectionStatus( } return null; } - export function resolveQuotaLimitPolicy( provider: string, providerSpecificData: JsonRecord @@ -407,7 +380,6 @@ export function resolveQuotaLimitPolicy( windows, }; } - export function evaluateQuotaLimitPolicy( provider: string, connection: ProviderConnectionView, @@ -440,7 +412,6 @@ export function evaluateQuotaLimitPolicy( resetAt: getEarliestFutureDate(resetCandidates), }; } - function parseFutureDateMs(value: string | null): number | null { if (!value) return null; // Tolerate numeric-epoch strings (e.g. "1781696905131.0") as well as ISO @@ -449,7 +420,6 @@ function parseFutureDateMs(value: string | null): number | null { if (!Number.isFinite(ms) || ms <= Date.now()) return null; return ms; } - function getEarliestFutureDate(candidates: Array): string | null { return ( candidates @@ -461,31 +431,26 @@ function getEarliestFutureDate(candidates: Array): string | null .sort((a, b) => (a.ms as number) - (b.ms as number))[0]?.raw || null ); } - function getCachedQuotaResetAt(connectionId: string): string | null { const entry = getQuotaCache(connectionId); if (!entry?.quotas) return null; return getEarliestFutureDate(Object.values(entry.quotas).map((quota) => quota.resetAt)); } - function isRetryableModelLockoutReason(reason: unknown): boolean { return typeof reason === "string" && reason.length > 0 ? !NON_RETRYABLE_MODEL_LOCKOUT_REASONS.has(reason) : false; } - function pushClampedPercentage(percentages: number[], value: number): void { if (Number.isFinite(value)) { percentages.push(Math.max(0, Math.min(100, value))); } } - function isResetAtInPast(resetAt: string | null): boolean { if (!resetAt) return false; const resetMs = new Date(resetAt).getTime(); return Number.isFinite(resetMs) && resetMs <= Date.now(); } - function collectPolicyQuotaHeadroomPercentages( provider: string, connection: ProviderConnectionView, @@ -508,7 +473,6 @@ function collectPolicyQuotaHeadroomPercentages( return percentages; } - function collectCachedQuotaHeadroomPercentages( provider: string, connection: ProviderConnectionView, @@ -528,7 +492,6 @@ function collectCachedQuotaHeadroomPercentages( return percentages; } - function getConnectionQuotaHeadroomPercent( provider: string, connection: ProviderConnectionView, @@ -548,7 +511,6 @@ function getConnectionQuotaHeadroomPercent( return percentages.length > 0 ? Math.min(...percentages) : null; } - function getConnectionErrorPenalty(connection: ProviderConnectionView): number { const errorType = normalizeStatus(connection.lastErrorType); const errorSource = normalizeStatus(connection.lastErrorSource); @@ -572,7 +534,6 @@ function getConnectionErrorPenalty(connection: ProviderConnectionView): number { return penalty; } - function getConnectionRecencyPenalty(connection: ProviderConnectionView): number { if (!connection.lastUsedAt) return 0; const ageMs = Date.now() - new Date(connection.lastUsedAt).getTime(); @@ -582,7 +543,6 @@ function getConnectionRecencyPenalty(connection: ProviderConnectionView): number if (ageMs < 5 * 60_000) return 1; return 0; } - function getP2CConnectionScore( provider: string, connection: ProviderConnectionView, @@ -628,7 +588,6 @@ function getP2CConnectionScore( return { score, quotaHeadroomPercent }; } - function compareP2CConnections( provider: string, a: ProviderConnectionView, @@ -662,12 +621,10 @@ function compareP2CConnections( * exclude it (#3061), otherwise it gets re-selected forever. */ const SYNTHETIC_NOAUTH_CONNECTION_ID = "noauth"; - type AnonymousFallbackProviderDefinition = { anonymousFallback?: boolean; noAuth?: boolean; }; - function buildSyntheticNoAuthCredentials(providerSpecificData: JsonRecord = {}): { apiKey: null; accessToken: null; @@ -756,7 +713,6 @@ async function loadNoAuthProviderSpecificData(providerId: string): Promise, @@ -790,7 +745,6 @@ async function maybeSyntheticNoAuthFallback( const providerSpecificData = await loadNoAuthProviderSpecificData(providerId); return buildSyntheticNoAuthCredentials(providerSpecificData); } - function normalizeExcludedConnectionIds( excludeConnectionId: string | null, extraExcludedConnectionIds: string[] | null | undefined @@ -811,7 +765,6 @@ function normalizeExcludedConnectionIds( return normalized; } - function formatConnectionPrefixesForLog(ids: Iterable, max = 6): string { const prefixes = Array.from(ids) .filter((id) => typeof id === "string" && id.length > 0) @@ -819,7 +772,6 @@ function formatConnectionPrefixesForLog(ids: Iterable, max = 6): string .map((id) => `${id.slice(0, 8)}...`); return prefixes.length > 0 ? prefixes.join(",") : "none"; } - function buildQuotaPreflightRateLimitedResult( provider: string, blockedByPreflight: Array<{ @@ -850,12 +802,10 @@ function buildQuotaPreflightRateLimitedResult( lastErrorCode: 429, }; } - function quotaPreflightUnavailableUntil(resetAt?: string | null): string { const resetMs = parseFutureDateMs(resetAt ?? null); return new Date(resetMs ?? Date.now() + 5 * 60 * 1000).toISOString(); } - async function markQuotaPreflightAccountUnavailable( provider: string, connectionId: string, @@ -884,14 +834,12 @@ async function markQuotaPreflightAccountUnavailable( // Provider-scoped mutexes prevent race conditions during account selection without // serializing unrelated providers behind a single global lock. const selectionMutexes = new Map>(); - function getSelectionMutexKey(provider: string, options: CredentialSelectionOptions): string { return [ resolveProviderId(provider) || provider, options.forcedConnectionId ? `forced:${options.forcedConnectionId}` : "pool", ].join(":"); } - function createSelectionLock(key: string) { const currentMutex = selectionMutexes.get(key) ?? Promise.resolve(); let resolveMutex: (() => void) | undefined; @@ -923,7 +871,6 @@ export { fisherYatesShuffle, getNextFromDeckSync as getNextFromDeck }; // Re-export readHeaderValue and AuthRequestHeaders from headerReader.ts for // backwards compat with existing imports (e.g. googApiKeyAuth.ts). export { readHeaderValue, type AuthRequestHeaders } from "./headerReader.ts"; - const PROVIDER_SEARCH_PAIRS: string[][] = [ ["nvidia", "nvidia_nim"], ["kimi-coding", "kimi-coding-apikey"], @@ -1703,7 +1650,6 @@ export async function getProviderCredentials( selectionLock.release(); } } - export async function getProviderCredentialsWithQuotaPreflight( provider: string, excludeConnectionId: string | null = null, @@ -2005,16 +1951,17 @@ export async function markAccountUnavailable( const disableCooling = connProviderSpecificData.disableCooling === true; const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels); + const isNvidiaModelGone = provider === "nvidia" && status === 410; const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs }; if ( isPerModelQuotaProvider && provider && provider !== "codex" && model && - (status === 404 || status === 429 || status >= 500) + (status === 404 || isNvidiaModelGone || status === 429 || status >= 500) ) { const reason = - status === 404 + status === 404 || isNvidiaModelGone ? "not_found" : status === 429 && fallbackResult.reason === RateLimitReason.QUOTA_EXHAUSTED ? "quota_exhausted" @@ -2046,7 +1993,10 @@ export async function markAccountUnavailable( ? "model" : getQuotaScopeLabelForProvider(provider, model); const antigravityFamilyInferredBaseCooldownMs = - !usesExactAntigravityLock && provider === "antigravity" && quotaScope === "family" && status === 429 + !usesExactAntigravityLock && + provider === "antigravity" && + quotaScope === "family" && + status === 429 ? ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS : null; const lockout = recordModelLockoutFailure( @@ -2055,7 +2005,7 @@ export async function markAccountUnavailable( model, reason, status, - status === 404 + status === 404 || isNvidiaModelGone ? (effectiveProviderProfile?.baseCooldownMs ?? COOLDOWN_MS.notFoundLocal) : (antigravityFamilyInferredBaseCooldownMs ?? fallbackResult.baseCooldownMs ?? @@ -2352,7 +2302,6 @@ export interface RecoveredStateExpectation { lastErrorAt: string | null; rateLimitedUntil: string | null; } - export async function clearRecoveredProviderState( credentials: Partial | null, expectedState?: RecoveredStateExpectation @@ -2373,12 +2322,10 @@ export async function clearRecoveredProviderState( await clearAccountError(credentials.connectionId, credentials); return { applied: true }; } - type AuthRequestLike = { headers?: AuthRequestHeaders | null; url?: string | null; }; - function readNonEmptyUrlToken(request: AuthRequestLike): string | null { if (typeof request?.url !== "string" || request.url.trim().length === 0) return null; diff --git a/stryker.conf.json b/stryker.conf.json index 0b177926cd..7c52c54726 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -247,6 +247,7 @@ "tests/unit/no-memory-header.test.ts", "tests/unit/noauth-autocombo-lockout-7623.test.ts", "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts", + "tests/unit/nvidia-410-model-scope.test.ts", "tests/unit/nvidia-passthrough-models-6773.test.ts", "tests/unit/nvidia-quota-phase1.test.ts", "tests/unit/oauth-providers-config.test.ts", diff --git a/tests/unit/nvidia-410-model-scope.test.ts b/tests/unit/nvidia-410-model-scope.test.ts new file mode 100644 index 0000000000..13fbaa6ef6 --- /dev/null +++ b/tests/unit/nvidia-410-model-scope.test.ts @@ -0,0 +1,196 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-nvidia-410-model-scope-")); + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "nvidia-410-model-scope-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); +const fallback = await import("../../open-sse/services/accountFallback.ts"); + +const DEAD_MODEL = "deepseek-ai/deepseek-v4-pro"; +const HEALTHY_MODEL = "z-ai/glm-5.2"; + +const GONE_BODY = JSON.stringify({ + type: "about:blank", + title: "Gone", + status: 410, + detail: + "The model 'deepseek-ai/deepseek-v4-pro' has reached its end of life " + + "and is no longer available.", +}); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedNvidiaConnection() { + return providersDb.createProviderConnection({ + provider: "nvidia", + authType: "apikey", + name: "nvidia-410-model-scope", + apiKey: "sk-nvidia-410-model-scope", + isActive: true, + testStatus: "active", + providerSpecificData: {}, + }); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(async () => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("NVIDIA 410 Gone stays model-scoped and leaves the connection usable", async () => { + const connection = await seedNvidiaConnection(); + + assert.equal( + fallback.hasPerModelQuota("nvidia", DEAD_MODEL), + true, + "NVIDIA must use per-model failure scoping" + ); + + const result = await auth.markAccountUnavailable( + connection.id, + 410, + GONE_BODY, + "nvidia", + DEAD_MODEL + ); + + assert.equal(result.shouldFallback, true); + + const after = await providersDb.getProviderConnectionById(connection.id); + + assert.equal( + after?.rateLimitedUntil ?? null, + null, + "410 for one retired NVIDIA model must not apply a connection-wide cooldown" + ); + + assert.equal( + after?.testStatus, + "active", + "410 for one retired NVIDIA model must leave the NVIDIA connection active" + ); + + assert.equal( + fallback.isModelLocked("nvidia", connection.id, DEAD_MODEL), + true, + "the retired model itself should be locked" + ); + + assert.equal( + fallback.isModelLocked("nvidia", connection.id, HEALTHY_MODEL), + false, + "a healthy sibling NVIDIA model must remain unlocked" + ); + + const healthyCredentials = await auth.getProviderCredentials("nvidia", null, null, HEALTHY_MODEL); + + assert.equal( + healthyCredentials?.connectionId, + connection.id, + "the same NVIDIA connection must remain selectable for healthy sibling models" + ); +}); + +test("non-per-model provider keeps 410 connection-scoped", async () => { + assert.equal( + fallback.hasPerModelQuota("openai", DEAD_MODEL), + false, + "plain OpenAI API-key connections are not per-model quota providers" + ); + + const connection = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "openai-410-connection-scope", + apiKey: "sk-openai-410-connection-scope", + isActive: true, + testStatus: "active", + providerSpecificData: {}, + }); + + const result = await auth.markAccountUnavailable( + connection.id, + 410, + "Gone", + "openai", + DEAD_MODEL + ); + + assert.equal(result.shouldFallback, true); + + const after = await providersDb.getProviderConnectionById(connection.id); + + assert.ok( + after?.rateLimitedUntil, + "non-per-model providers should retain the existing connection-level 410 behavior" + ); + + assert.equal( + after?.testStatus, + "unavailable", + "410 model scoping must not be applied globally to every provider" + ); +}); + +test("other per-model providers retain existing 410 connection scope", async () => { + assert.equal( + fallback.hasPerModelQuota("gemini", DEAD_MODEL), + true, + "Gemini provides a non-NVIDIA per-model control case" + ); + + const connection = await providersDb.createProviderConnection({ + provider: "gemini", + authType: "apikey", + name: "gemini-410-control", + apiKey: "sk-gemini-410-control", + isActive: true, + testStatus: "active", + providerSpecificData: {}, + }); + + const result = await auth.markAccountUnavailable( + connection.id, + 410, + "Gone", + "gemini", + DEAD_MODEL + ); + + assert.equal(result.shouldFallback, true); + + const after = await providersDb.getProviderConnectionById(connection.id); + + assert.ok( + after?.rateLimitedUntil, + "410 must remain connection-scoped for per-model providers without an explicit 410 contract" + ); + + assert.equal( + after?.testStatus, + "unavailable", + "the NVIDIA-specific 410 fix must not change other provider semantics" + ); + + assert.equal( + fallback.isModelLocked("gemini", connection.id, DEAD_MODEL), + false, + "a generic per-model provider must not inherit NVIDIA's 410 model lock" + ); +});