fix(auth): reduce file size

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

src/sse/services/auth.ts

@@ -83,7 +83,6 @@ import { getResource404Bypass } from "./requestResourceHealth";
 import * as log from "../utils/logger";
 import { fisherYatesShuffle, getNextFromDeckSync } from "@/shared/utils/shuffleDeck";
 import { readHeaderValue, type AuthRequestHeaders } from "./headerReader.ts";
-
 type JsonRecord = Record<string, unknown>;
 interface RecoverableConnectionState {
   connectionId: string;
@@ -94,7 +93,6 @@ interface RecoverableConnectionState {
   lastErrorType?: string | null;
   lastErrorSource?: string | null;
 }
-
 interface CredentialSelectionOptions {
   allowSuppressedConnections?: boolean;
   allowRateLimitedConnections?: boolean;
@@ -104,14 +102,12 @@ interface CredentialSelectionOptions {
   sessionKey?: string | null;
   sessionAffinityTtlMs?: number | null;
 }
-
 interface CooldownInspectionState {
   connection: ProviderConnectionView;
   connectionCooldownMs: number | null;
   codexScopeCooldownMs: number | null;
   retryableModelCooldownMs: number | null;
 }
-
 const MIN_QUOTA_THRESHOLD_PERCENT = 1;
 const MAX_QUOTA_THRESHOLD_PERCENT = 100;
 const NON_RETRYABLE_MODEL_LOCKOUT_REASONS = new Set(["not_found", "not_found_local"]);
@@ -119,25 +115,20 @@ const NON_RETRYABLE_MODEL_LOCKOUT_REASONS = new Set(["not_found", "not_found_loc
 // this base. Real upstream Retry-After hints still win — they flow through
 // `exactCooldownMs` (usedUpstreamRetryHint), not this base. (#5222)
 const ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS = 30_000;
-
 function asRecord(value: unknown): JsonRecord {
   return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
 }
-
 function toStringOrNull(value: unknown): string | null {
   return typeof value === "string" && value.trim().length > 0 ? value : null;
 }
-
 function toNullableNumber(value: unknown): number | null {
   if (value === null || value === undefined) return null;
   const parsed = toNumber(value, Number.NaN);
   return Number.isFinite(parsed) ? parsed : null;
 }
-
 function toBooleanOrDefault(value: unknown, fallback: boolean): boolean {
   return typeof value === "boolean" ? value : fallback;
 }
-
 function normalizeSessionKey(value: unknown, prefix: string): string | null {
   if (typeof value !== "string" || value.trim().length === 0) return null;
   const trimmed = value.trim();
@@ -146,7 +137,6 @@ function normalizeSessionKey(value: unknown, prefix: string): string | null {
   }
   return `${prefix}:sha256:${createHash("sha256").update(trimmed).digest("hex")}`;
 }
-
 function extractTextForSessionHash(value: unknown): string | null {
   if (typeof value === "string") return value;
   if (Array.isArray(value)) {
@@ -164,7 +154,6 @@ function extractTextForSessionHash(value: unknown): string | null {
   if (value && typeof value === "object") return JSON.stringify(value);
   return null;
 }
-
 function getFirstInputText(body: unknown): string | null {
   const record = asRecord(body);
   if (record.input !== undefined) {
@@ -189,7 +178,6 @@ function getFirstInputText(body: unknown): string | null {
 
   return null;
 }
-
 export function extractSessionAffinityKey(
   body: unknown,
   headers?: Headers | { get?: (name: string) => string | null } | null
@@ -216,7 +204,6 @@ export function extractSessionAffinityKey(
   if (!inputText || inputText.trim().length === 0) return null;
   return `input:sha256:${createHash("sha256").update(inputText.slice(0, 4096)).digest("hex")}`;
 }
-
 function getCodexLimitPolicy(providerSpecificData: JsonRecord): {
   use5h: boolean;
   useWeekly: boolean;
@@ -227,13 +214,11 @@ function getCodexLimitPolicy(providerSpecificData: JsonRecord): {
     useWeekly: toBooleanOrDefault(policy.useWeekly, true),
   };
 }
-
 interface QuotaLimitPolicy {
   enabled: boolean;
   thresholdPercent: number;
   windows: string[];
 }
-
 interface QuotaCacheView {
   quotas?: Record<
     string,
@@ -243,7 +228,6 @@ interface QuotaCacheView {
     }
   >;
 }
-
 function normalizeQuotaThreshold(
   value: unknown,
   fallback = DEFAULT_QUOTA_THRESHOLD_PERCENT
@@ -251,17 +235,14 @@ function normalizeQuotaThreshold(
   const parsed = toNumber(value, fallback);
   return Math.min(MAX_QUOTA_THRESHOLD_PERCENT, Math.max(MIN_QUOTA_THRESHOLD_PERCENT, parsed));
 }
-
 function normalizeWindowName(windowName: unknown): string | null {
   if (typeof windowName !== "string") return null;
   const normalized = windowName.trim().toLowerCase();
   return normalized.length > 0 ? normalized : null;
 }
-
 function uniqueWindows(windows: string[]): string[] {
   return [...new Set(windows)];
 }
-
 function normalizeCodexWindowName(windowName: unknown): string | null {
   if (typeof windowName !== "string") return null;
   const normalized = windowName.trim().toLowerCase();
@@ -273,7 +254,6 @@ function normalizeCodexWindowName(windowName: unknown): string | null {
   }
   return toCodexBaseQuotaWindowName(normalized);
 }
-
 function applyCodexWindowPolicy(rawWindows: string[], providerSpecificData: JsonRecord): string[] {
   const codexPolicy = getCodexLimitPolicy(providerSpecificData);
   const normalizedRaw = rawWindows.map(normalizeCodexWindowName).filter(Boolean) as string[];
@@ -291,7 +271,6 @@ function applyCodexWindowPolicy(rawWindows: string[], providerSpecificData: Json
 
   return uniqueWindows(windows);
 }
-
 function getCodexScopeRateLimitedUntil(
   providerSpecificData: JsonRecord,
   model: string | null
@@ -302,7 +281,6 @@ function getCodexScopeRateLimitedUntil(
   const value = scopeMap[scope];
   return typeof value === "string" && value.trim().length > 0 ? value : null;
 }
-
 function isCodexScopeUnavailable(
   connection: ProviderConnectionView,
   model: string | null
@@ -311,7 +289,6 @@ function isCodexScopeUnavailable(
   if (!until) return false;
   return new Date(until).getTime() > Date.now();
 }
-
 function getEarliestCodexScopeRateLimitedUntil(
   connections: ProviderConnectionView[],
   model: string | null
@@ -332,11 +309,9 @@ function getEarliestCodexScopeRateLimitedUntil(
 
   return earliest;
 }
-
 function normalizeStatus(value: string | null): string {
   return (value || "").trim().toLowerCase();
 }
-
 function isTerminalConnectionStatus(connection: ProviderConnectionView): boolean {
   const status = normalizeStatus(connection.testStatus);
   return status === "credits_exhausted" || status === "banned" || status === "expired";
@@ -354,7 +329,6 @@ function isRecoverableCookieAuth401(
     resolveProviderId(provider) in WEB_COOKIE_PROVIDERS
   );
 }
-
 function resolveTerminalConnectionStatus(
   status: number,
   result: { permanent?: boolean; creditsExhausted?: boolean },
@@ -381,7 +355,6 @@ function resolveTerminalConnectionStatus(
   }
   return null;
 }
-
 export function resolveQuotaLimitPolicy(
   provider: string,
   providerSpecificData: JsonRecord
@@ -407,7 +380,6 @@ export function resolveQuotaLimitPolicy(
     windows,
   };
 }
-
 export function evaluateQuotaLimitPolicy(
   provider: string,
   connection: ProviderConnectionView,
@@ -440,7 +412,6 @@ export function evaluateQuotaLimitPolicy(
     resetAt: getEarliestFutureDate(resetCandidates),
   };
 }
-
 function parseFutureDateMs(value: string | null): number | null {
   if (!value) return null;
   // Tolerate numeric-epoch strings (e.g. "1781696905131.0") as well as ISO
@@ -449,7 +420,6 @@ function parseFutureDateMs(value: string | null): number | null {
   if (!Number.isFinite(ms) || ms <= Date.now()) return null;
   return ms;
 }
-
 function getEarliestFutureDate(candidates: Array<string | null>): string | null {
   return (
     candidates
@@ -461,31 +431,26 @@ function getEarliestFutureDate(candidates: Array<string | null>): string | null
       .sort((a, b) => (a.ms as number) - (b.ms as number))[0]?.raw || null
   );
 }
-
 function getCachedQuotaResetAt(connectionId: string): string | null {
   const entry = getQuotaCache(connectionId);
   if (!entry?.quotas) return null;
   return getEarliestFutureDate(Object.values(entry.quotas).map((quota) => quota.resetAt));
 }
-
 function isRetryableModelLockoutReason(reason: unknown): boolean {
   return typeof reason === "string" && reason.length > 0
     ? !NON_RETRYABLE_MODEL_LOCKOUT_REASONS.has(reason)
     : false;
 }
-
 function pushClampedPercentage(percentages: number[], value: number): void {
   if (Number.isFinite(value)) {
     percentages.push(Math.max(0, Math.min(100, value)));
   }
 }
-
 function isResetAtInPast(resetAt: string | null): boolean {
   if (!resetAt) return false;
   const resetMs = new Date(resetAt).getTime();
   return Number.isFinite(resetMs) && resetMs <= Date.now();
 }
-
 function collectPolicyQuotaHeadroomPercentages(
   provider: string,
   connection: ProviderConnectionView,
@@ -508,7 +473,6 @@ function collectPolicyQuotaHeadroomPercentages(
 
   return percentages;
 }
-
 function collectCachedQuotaHeadroomPercentages(
   provider: string,
   connection: ProviderConnectionView,
@@ -528,7 +492,6 @@ function collectCachedQuotaHeadroomPercentages(
 
   return percentages;
 }
-
 function getConnectionQuotaHeadroomPercent(
   provider: string,
   connection: ProviderConnectionView,
@@ -548,7 +511,6 @@ function getConnectionQuotaHeadroomPercent(
 
   return percentages.length > 0 ? Math.min(...percentages) : null;
 }
-
 function getConnectionErrorPenalty(connection: ProviderConnectionView): number {
   const errorType = normalizeStatus(connection.lastErrorType);
   const errorSource = normalizeStatus(connection.lastErrorSource);
@@ -572,7 +534,6 @@ function getConnectionErrorPenalty(connection: ProviderConnectionView): number {
 
   return penalty;
 }
-
 function getConnectionRecencyPenalty(connection: ProviderConnectionView): number {
   if (!connection.lastUsedAt) return 0;
   const ageMs = Date.now() - new Date(connection.lastUsedAt).getTime();
@@ -582,7 +543,6 @@ function getConnectionRecencyPenalty(connection: ProviderConnectionView): number
   if (ageMs < 5 * 60_000) return 1;
   return 0;
 }
-
 function getP2CConnectionScore(
   provider: string,
   connection: ProviderConnectionView,
@@ -628,7 +588,6 @@ function getP2CConnectionScore(
 
   return { score, quotaHeadroomPercent };
 }
-
 function compareP2CConnections(
   provider: string,
   a: ProviderConnectionView,
@@ -662,12 +621,10 @@ function compareP2CConnections(
  * exclude it (#3061), otherwise it gets re-selected forever.
  */
 const SYNTHETIC_NOAUTH_CONNECTION_ID = "noauth";
-
 type AnonymousFallbackProviderDefinition = {
   anonymousFallback?: boolean;
   noAuth?: boolean;
 };
-
 function buildSyntheticNoAuthCredentials(providerSpecificData: JsonRecord = {}): {
   apiKey: null;
   accessToken: null;
@@ -756,7 +713,6 @@ async function loadNoAuthProviderSpecificData(providerId: string): Promise<JsonR
     return {};
   }
 }
-
 function providerCanUseSyntheticNoAuthFallback(providerId: string): boolean {
   const providerDef = getProviderById(providerId) as
     AnonymousFallbackProviderDefinition | undefined;
@@ -772,7 +728,6 @@ function providerCanUseSyntheticNoAuthFallback(providerId: string): boolean {
     webCookieProviderDef?.noAuth === true
   );
 }
-
 async function maybeSyntheticNoAuthFallback(
   providerId: string,
   excludedConnectionIds: Set<string>,
@@ -790,7 +745,6 @@ async function maybeSyntheticNoAuthFallback(
   const providerSpecificData = await loadNoAuthProviderSpecificData(providerId);
   return buildSyntheticNoAuthCredentials(providerSpecificData);
 }
-
 function normalizeExcludedConnectionIds(
   excludeConnectionId: string | null,
   extraExcludedConnectionIds: string[] | null | undefined
@@ -811,7 +765,6 @@ function normalizeExcludedConnectionIds(
 
   return normalized;
 }
-
 function formatConnectionPrefixesForLog(ids: Iterable<string>, max = 6): string {
   const prefixes = Array.from(ids)
     .filter((id) => typeof id === "string" && id.length > 0)
@@ -819,7 +772,6 @@ function formatConnectionPrefixesForLog(ids: Iterable<string>, max = 6): string
     .map((id) => `${id.slice(0, 8)}...`);
   return prefixes.length > 0 ? prefixes.join(",") : "none";
 }
-
 function buildQuotaPreflightRateLimitedResult(
   provider: string,
   blockedByPreflight: Array<{
@@ -850,12 +802,10 @@ function buildQuotaPreflightRateLimitedResult(
     lastErrorCode: 429,
   };
 }
-
 function quotaPreflightUnavailableUntil(resetAt?: string | null): string {
   const resetMs = parseFutureDateMs(resetAt ?? null);
   return new Date(resetMs ?? Date.now() + 5 * 60 * 1000).toISOString();
 }
-
 async function markQuotaPreflightAccountUnavailable(
   provider: string,
   connectionId: string,
@@ -884,14 +834,12 @@ async function markQuotaPreflightAccountUnavailable(
 // Provider-scoped mutexes prevent race conditions during account selection without
 // serializing unrelated providers behind a single global lock.
 const selectionMutexes = new Map<string, Promise<void>>();
-
 function getSelectionMutexKey(provider: string, options: CredentialSelectionOptions): string {
   return [
     resolveProviderId(provider) || provider,
     options.forcedConnectionId ? `forced:${options.forcedConnectionId}` : "pool",
   ].join(":");
 }
-
 function createSelectionLock(key: string) {
   const currentMutex = selectionMutexes.get(key) ?? Promise.resolve();
   let resolveMutex: (() => void) | undefined;
@@ -923,7 +871,6 @@ export { fisherYatesShuffle, getNextFromDeckSync as getNextFromDeck };
 // Re-export readHeaderValue and AuthRequestHeaders from headerReader.ts for
 // backwards compat with existing imports (e.g. googApiKeyAuth.ts).
 export { readHeaderValue, type AuthRequestHeaders } from "./headerReader.ts";
-
 const PROVIDER_SEARCH_PAIRS: string[][] = [
   ["nvidia", "nvidia_nim"],
   ["kimi-coding", "kimi-coding-apikey"],
@@ -1703,7 +1650,6 @@ export async function getProviderCredentials(
     selectionLock.release();
   }
 }
-
 export async function getProviderCredentialsWithQuotaPreflight(
   provider: string,
   excludeConnectionId: string | null = null,
@@ -2005,16 +1951,17 @@ export async function markAccountUnavailable(
     const disableCooling = connProviderSpecificData.disableCooling === true;
 
     const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
+    const isNvidiaModelGone = provider === "nvidia" && status === 410;
     const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs };
     if (
       isPerModelQuotaProvider &&
       provider &&
       provider !== "codex" &&
       model &&
-      (status === 404 || status === 429 || status >= 500)
+      (status === 404 || isNvidiaModelGone || status === 429 || status >= 500)
     ) {
       const reason =
-        status === 404
+        status === 404 || isNvidiaModelGone
           ? "not_found"
           : status === 429 && fallbackResult.reason === RateLimitReason.QUOTA_EXHAUSTED
             ? "quota_exhausted"
@@ -2046,7 +1993,10 @@ export async function markAccountUnavailable(
         ? "model"
         : getQuotaScopeLabelForProvider(provider, model);
       const antigravityFamilyInferredBaseCooldownMs =
-        !usesExactAntigravityLock && provider === "antigravity" && quotaScope === "family" && status === 429
+        !usesExactAntigravityLock &&
+        provider === "antigravity" &&
+        quotaScope === "family" &&
+        status === 429
           ? ANTIGRAVITY_FAMILY_INFERRED_BASE_COOLDOWN_MS
           : null;
       const lockout = recordModelLockoutFailure(
@@ -2055,7 +2005,7 @@ export async function markAccountUnavailable(
         model,
         reason,
         status,
-        status === 404
+        status === 404 || isNvidiaModelGone
           ? (effectiveProviderProfile?.baseCooldownMs ?? COOLDOWN_MS.notFoundLocal)
           : (antigravityFamilyInferredBaseCooldownMs ??
               fallbackResult.baseCooldownMs ??
@@ -2352,7 +2302,6 @@ export interface RecoveredStateExpectation {
   lastErrorAt: string | null;
   rateLimitedUntil: string | null;
 }
-
 export async function clearRecoveredProviderState(
   credentials: Partial<RecoverableConnectionState> | null,
   expectedState?: RecoveredStateExpectation
@@ -2373,12 +2322,10 @@ export async function clearRecoveredProviderState(
   await clearAccountError(credentials.connectionId, credentials);
   return { applied: true };
 }
-
 type AuthRequestLike = {
   headers?: AuthRequestHeaders | null;
   url?: string | null;
 };
-
 function readNonEmptyUrlToken(request: AuthRequestLike): string | null {
   if (typeof request?.url !== "string" || request.url.trim().length === 0) return null;
 

stryker.conf.json

@@ -247,6 +247,7 @@
       "tests/unit/no-memory-header.test.ts",
       "tests/unit/noauth-autocombo-lockout-7623.test.ts",
       "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts",
+      "tests/unit/nvidia-410-model-scope.test.ts",
       "tests/unit/nvidia-passthrough-models-6773.test.ts",
       "tests/unit/nvidia-quota-phase1.test.ts",
       "tests/unit/oauth-providers-config.test.ts",

tests/unit/nvidia-410-model-scope.test.ts

@@ -0,0 +1,196 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-nvidia-410-model-scope-"));
+
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "nvidia-410-model-scope-test-secret";
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const auth = await import("../../src/sse/services/auth.ts");
+const fallback = await import("../../open-sse/services/accountFallback.ts");
+
+const DEAD_MODEL = "deepseek-ai/deepseek-v4-pro";
+const HEALTHY_MODEL = "z-ai/glm-5.2";
+
+const GONE_BODY = JSON.stringify({
+  type: "about:blank",
+  title: "Gone",
+  status: 410,
+  detail:
+    "The model 'deepseek-ai/deepseek-v4-pro' has reached its end of life " +
+    "and is no longer available.",
+});
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+async function seedNvidiaConnection() {
+  return providersDb.createProviderConnection({
+    provider: "nvidia",
+    authType: "apikey",
+    name: "nvidia-410-model-scope",
+    apiKey: "sk-nvidia-410-model-scope",
+    isActive: true,
+    testStatus: "active",
+    providerSpecificData: {},
+  });
+}
+
+test.beforeEach(async () => {
+  await resetStorage();
+});
+
+test.after(async () => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+test("NVIDIA 410 Gone stays model-scoped and leaves the connection usable", async () => {
+  const connection = await seedNvidiaConnection();
+
+  assert.equal(
+    fallback.hasPerModelQuota("nvidia", DEAD_MODEL),
+    true,
+    "NVIDIA must use per-model failure scoping"
+  );
+
+  const result = await auth.markAccountUnavailable(
+    connection.id,
+    410,
+    GONE_BODY,
+    "nvidia",
+    DEAD_MODEL
+  );
+
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(connection.id);
+
+  assert.equal(
+    after?.rateLimitedUntil ?? null,
+    null,
+    "410 for one retired NVIDIA model must not apply a connection-wide cooldown"
+  );
+
+  assert.equal(
+    after?.testStatus,
+    "active",
+    "410 for one retired NVIDIA model must leave the NVIDIA connection active"
+  );
+
+  assert.equal(
+    fallback.isModelLocked("nvidia", connection.id, DEAD_MODEL),
+    true,
+    "the retired model itself should be locked"
+  );
+
+  assert.equal(
+    fallback.isModelLocked("nvidia", connection.id, HEALTHY_MODEL),
+    false,
+    "a healthy sibling NVIDIA model must remain unlocked"
+  );
+
+  const healthyCredentials = await auth.getProviderCredentials("nvidia", null, null, HEALTHY_MODEL);
+
+  assert.equal(
+    healthyCredentials?.connectionId,
+    connection.id,
+    "the same NVIDIA connection must remain selectable for healthy sibling models"
+  );
+});
+
+test("non-per-model provider keeps 410 connection-scoped", async () => {
+  assert.equal(
+    fallback.hasPerModelQuota("openai", DEAD_MODEL),
+    false,
+    "plain OpenAI API-key connections are not per-model quota providers"
+  );
+
+  const connection = await providersDb.createProviderConnection({
+    provider: "openai",
+    authType: "apikey",
+    name: "openai-410-connection-scope",
+    apiKey: "sk-openai-410-connection-scope",
+    isActive: true,
+    testStatus: "active",
+    providerSpecificData: {},
+  });
+
+  const result = await auth.markAccountUnavailable(
+    connection.id,
+    410,
+    "Gone",
+    "openai",
+    DEAD_MODEL
+  );
+
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(connection.id);
+
+  assert.ok(
+    after?.rateLimitedUntil,
+    "non-per-model providers should retain the existing connection-level 410 behavior"
+  );
+
+  assert.equal(
+    after?.testStatus,
+    "unavailable",
+    "410 model scoping must not be applied globally to every provider"
+  );
+});
+
+test("other per-model providers retain existing 410 connection scope", async () => {
+  assert.equal(
+    fallback.hasPerModelQuota("gemini", DEAD_MODEL),
+    true,
+    "Gemini provides a non-NVIDIA per-model control case"
+  );
+
+  const connection = await providersDb.createProviderConnection({
+    provider: "gemini",
+    authType: "apikey",
+    name: "gemini-410-control",
+    apiKey: "sk-gemini-410-control",
+    isActive: true,
+    testStatus: "active",
+    providerSpecificData: {},
+  });
+
+  const result = await auth.markAccountUnavailable(
+    connection.id,
+    410,
+    "Gone",
+    "gemini",
+    DEAD_MODEL
+  );
+
+  assert.equal(result.shouldFallback, true);
+
+  const after = await providersDb.getProviderConnectionById(connection.id);
+
+  assert.ok(
+    after?.rateLimitedUntil,
+    "410 must remain connection-scoped for per-model providers without an explicit 410 contract"
+  );
+
+  assert.equal(
+    after?.testStatus,
+    "unavailable",
+    "the NVIDIA-specific 410 fix must not change other provider semantics"
+  );
+
+  assert.equal(
+    fallback.isModelLocked("gemini", connection.id, DEAD_MODEL),
+    false,
+    "a generic per-model provider must not inherit NVIDIA's 410 model lock"
+  );
+});