diff --git a/changelog.d/fixes/10313-catalog-cache-key-hash.md b/changelog.d/fixes/10313-catalog-cache-key-hash.md new file mode 100644 index 0000000000..5c85689004 --- /dev/null +++ b/changelog.d/fixes/10313-catalog-cache-key-hash.md @@ -0,0 +1 @@ +- fix(api): hash the API key before using it as the model-catalog cache Map key (no raw credentials in process heap) (#10313) diff --git a/changelog.d/fixes/9147-catalog-eventloop-yield.md b/changelog.d/fixes/9147-catalog-eventloop-yield.md new file mode 100644 index 0000000000..1f27c92b33 --- /dev/null +++ b/changelog.d/fixes/9147-catalog-eventloop-yield.md @@ -0,0 +1 @@ +- fix(api): yield the event loop during catalog builds and bulk-load override/hidden-model tables (#9147) \ No newline at end of file diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index adc64bf762..e086cf1870 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -6,9 +6,9 @@ import { getAllCustomModels, getSettings, getCachedProviderNodes, - getModelIsHidden, getModelAliases, getDatabaseSettings, + getHiddenModelsByProvider, } from "@/lib/localDb"; import { createLazyConnectionView } from "@/lib/db/providers/lazyConnectionView"; import { extractAliasBackedModels } from "./aliasBackedModels"; @@ -133,7 +133,7 @@ export { } from "./catalogCache"; export type { CachedCatalog } from "./catalogCache"; -const BUILTIN_AUTO_YIELD_INTERVAL = 8; +const BUILTIN_AUTO_YIELD_INTERVAL = 2; function yieldCatalogBuildTurn(): Promise { return new Promise((resolve) => setImmediate(resolve)); @@ -157,6 +157,8 @@ export async function getUnifiedModelsResponse( try { settingsForAuth = await getSettings(); } catch {} + // #9147: yield before auth check to allow event loop tick + await yieldCatalogBuildTurn(); const authRejection = await getModelCatalogAuthRejection(request, settingsForAuth, { ...corsHeaders, ...diagnosticHeaders, @@ -227,7 +229,34 @@ async function buildUnifiedModelsResponseCore( corsHeaders: Record = {} ) { const diagnosticHeaders = getCatalogDiagnosticsHeaders({ request }); + // #9147: this builder walks connections + model registries at catalog scale with no + // event-loop yield, so a large deployment pins the single Node.js thread for the + // whole build (reporter: 183 connections / 2000+ models → 10.1s stall that blocks the + // dashboard WS heartbeat). Yield every `catYIELD_EVERY` items across the hot loops. + const catYIELD_EVERY = 20; + let catYieldCount = 0; + const maybeYieldCatalogBuild = async (): Promise => { + catYieldCount++; + if (catYieldCount % catYIELD_EVERY === 0) { + await yieldCatalogBuildTurn(); + } + }; try { + // #9147: `getModelIsHidden()` is a SQLite read per call (custom row + compat list) + // and the build consults it ~16× per entry. Bulk-load the hidden-model map once + // (one query — `getHiddenModelsByProvider`) and resolve from memory for the whole + // build. A provider absent from the map has no hidden models at all — `false`, + // no on-demand fallback (that would reintroduce the per-call SQLite reads). + // Deliberately kept INSIDE this try block (not hoisted above it): the builder's + // own catch below is what converts a build-time failure into a sanitized 500 + // Response instead of a rejected promise — hoisting this bulk read above the + // try would let a crash here propagate as an unhandled rejection instead + // (catalogCache.ts's in-flight coalescing does not fully consume rejections). + const hiddenModelsByProvider = getHiddenModelsByProvider(); + const isModelHiddenBulk = (providerId: string, modelId: string): boolean => { + const hiddenSet = hiddenModelsByProvider.get(providerId); + return hiddenSet ? hiddenSet.has(modelId) : false; + }; let settings: Record = {}; try { settings = await getSettings(); @@ -238,6 +267,10 @@ async function buildUnifiedModelsResponseCore( ...diagnosticHeaders, }); if (authRejection) return authRejection; + + // #9147: yield after auth check before DB initialization prologue + await yieldCatalogBuildTurn(); + const { aliasToProviderId, providerIdToAlias } = buildAliasMaps(); const _qp = new URL(request.url).searchParams.get("prefix"); const prefixMode = @@ -322,6 +355,7 @@ async function buildUnifiedModelsResponseCore( // Get combos let combos = []; + await yieldCatalogBuildTurn(); try { combos = await getCombos(); } catch (e) { @@ -355,7 +389,16 @@ async function buildUnifiedModelsResponseCore( if ("alias" in p && typeof p.alias === "string") activeAliases.add(p.alias); } + // #9147 follow-up: this is called ~1-3x per model at catalog scale (providerSupportsModel, + // isExcludedByProviderConnections). Connections do not change mid-build, so memoize per + // unique (unordered) key-set instead of rescanning connectionsByProvider on every call — + // otherwise the O(models) hot loop regains an O(connections) cost per model and blows the + // single-stretch event-loop budget this file's own yield mechanism is meant to protect. + const connectionsForProviderCache = new Map(); const getConnectionsForProvider = (...keys: Array) => { + const cacheKey = keys.filter((k): k is string => Boolean(k)).sort().join(""); + const cached = connectionsForProviderCache.get(cacheKey); + if (cached) return cached; const seen = new Set(); const collected: typeof connections = []; for (const key of keys) { @@ -366,6 +409,7 @@ async function buildUnifiedModelsResponseCore( collected.push(connection); } } + connectionsForProviderCache.set(cacheKey, collected); return collected; }; @@ -586,7 +630,7 @@ async function buildUnifiedModelsResponseCore( timestamp, (c) => buildComboCatalogMetadata(c, combos) ); - const quotaFinal = applyCatalogPostFilters(request, quotaModels, { + const quotaFinal = await applyCatalogPostFilters(request, quotaModels, { connections, prefixMode, aliasToProviderId, @@ -681,7 +725,7 @@ async function buildUnifiedModelsResponseCore( ) as ComboCatalogTarget[]; const visibleTargets = comboTargets.filter((target) => { const resolved = getComboTargetModelId(target); - return resolved ? !getModelIsHidden(resolved.providerId, resolved.modelId) : true; + return resolved ? !isModelHiddenBulk(resolved.providerId, resolved.modelId) : true; }); if (visibleTargets.length === 0) continue; @@ -698,11 +742,16 @@ async function buildUnifiedModelsResponseCore( parent: null, ...comboMetadata, }); + + // #9147: combos can number hundreds at catalog scale — yield periodically. + await maybeYieldCatalogBuild(); } let syncedModelsByProvider: Record = {}; try { + await yieldCatalogBuildTurn(); syncedModelsByProvider = await getAllActiveSyncedModels(); + await yieldCatalogBuildTurn(); } catch (e) { // DB unavailable — log and fall through; static models remain as defaults. console.log("[catalog] Could not fetch synced available models:", e); @@ -790,7 +839,7 @@ async function buildUnifiedModelsResponseCore( if (!isModelSelectable(canonicalProviderId, model.id)) continue; if (!providerSupportsModel(canonicalProviderId, model.id)) continue; const aliasId = `${alias}/${model.id}`; - if (getModelIsHidden(canonicalProviderId, model.id)) continue; + if (isModelHiddenBulk(canonicalProviderId, model.id)) continue; if (isExcludedByProviderConnections(canonicalProviderId, model.id)) continue; if (shouldHidePaid(canonicalProviderId, model.id, (model as { pricing?: unknown }).pricing)) continue; @@ -845,12 +894,15 @@ async function buildUnifiedModelsResponseCore( ...thinkingCapabilities, }); } + + // #9147: static model walk is the densest loop — yield periodically. + await maybeYieldCatalogBuild(); } } for (const modelId of CODEX_NATIVE_UNPREFIXED_MODELS) { if (!providerSupportsModel("codex", modelId)) continue; - if (getModelIsHidden("codex", modelId)) continue; + if (isModelHiddenBulk("codex", modelId)) continue; const alias = providerIdToAlias.codex || "cx"; const aliasId = `${alias}/${modelId}`; @@ -911,7 +963,7 @@ async function buildUnifiedModelsResponseCore( if (canonicalProviderId === "codex" && isCodexDiscoveryModelExcluded(sm)) { continue; } - if (getModelIsHidden(providerId, sm.id)) continue; + if (isModelHiddenBulk(providerId, sm.id)) continue; if (isExcludedByProviderConnections(canonicalProviderId, sm.id)) continue; // #6457: some upstream discovery catalogs (e.g. HuggingFace's live // `/v1/models`) return image/diffusion models with no modality info, @@ -1024,6 +1076,9 @@ async function buildUnifiedModelsResponseCore( }); } } + + // #9147: synced-model union is usually the largest walk — yield periodically. + await maybeYieldCatalogBuild(); } } } catch (err) { @@ -1053,7 +1108,7 @@ async function buildUnifiedModelsResponseCore( if (hidePaid && !isFree) continue; // #9293: respect per-model hidden flags (e.g. operator hid google/chirp-3 // from the OpenRouter provider, so it should not appear in the live catalog). - if (getModelIsHidden("openrouter", openRouterModel.id)) continue; + if (isModelHiddenBulk("openrouter", openRouterModel.id)) continue; const supportedParameters = Array.isArray(openRouterModel.supported_parameters) ? openRouterModel.supported_parameters : []; @@ -1094,6 +1149,9 @@ async function buildUnifiedModelsResponseCore( ...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}), ...(Object.keys(capabilities).length > 0 ? { capabilities } : {}), }); + + // #9147: OpenRouter catalog can be large — yield periodically. + await maybeYieldCatalogBuild(); } } catch (err) { console.error("[catalog] Error loading OpenRouter catalog:", err); @@ -1138,7 +1196,7 @@ async function buildUnifiedModelsResponseCore( // Helper: strip the provider prefix from a specialty model ID to get the // provider-relative path (e.g. "openrouter/google/chirp-3" -> "google/chirp-3"). - // This is the correct key used by getModelIsHidden() — using .split("/").pop() + // This is the correct key used by the hidden-model lookup — using .split("/").pop() // here would discard all but the last segment and miss stored flags for // providers whose model IDs carry a sub-path (e.g. OpenRouter scoped models). const getSpecialtyModelRelativeId = (modelId: string, provider: string): string => @@ -1149,7 +1207,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(embModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(embModel.id, embModel.provider); if (!providerSupportsModel(embModel.provider, rawModelId)) continue; - if (getModelIsHidden(embModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(embModel.provider, rawModelId)) continue; if (hasEquivalentSpecialtyModel(embModel.provider, rawModelId, "embedding", embModel.id)) { continue; } @@ -1177,7 +1235,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(imgModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(imgModel.id, imgModel.provider); if (!providerSupportsModel(imgModel.provider, rawModelId)) continue; - if (getModelIsHidden(imgModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(imgModel.provider, rawModelId)) continue; models.push({ id: imgModel.id, object: "model", @@ -1197,7 +1255,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(rerankModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(rerankModel.id, rerankModel.provider); if (!providerSupportsModel(rerankModel.provider, rawModelId)) continue; - if (getModelIsHidden(rerankModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(rerankModel.provider, rawModelId)) continue; if (hasEquivalentSpecialtyModel(rerankModel.provider, rawModelId, "rerank", rerankModel.id)) { continue; } @@ -1216,7 +1274,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(audioModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(audioModel.id, audioModel.provider); if (!providerSupportsModel(audioModel.provider, rawModelId)) continue; - if (getModelIsHidden(audioModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(audioModel.provider, rawModelId)) continue; models.push({ id: audioModel.id, object: "model", @@ -1232,7 +1290,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(modModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(modModel.id, modModel.provider); if (!providerSupportsModel(modModel.provider, rawModelId)) continue; - if (getModelIsHidden(modModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(modModel.provider, rawModelId)) continue; models.push({ id: modModel.id, object: "model", @@ -1247,7 +1305,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(videoModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(videoModel.id, videoModel.provider); if (!providerSupportsModel(videoModel.provider, rawModelId)) continue; - if (getModelIsHidden(videoModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(videoModel.provider, rawModelId)) continue; models.push({ id: videoModel.id, object: "model", @@ -1268,7 +1326,7 @@ async function buildUnifiedModelsResponseCore( if (!isProviderActive(musicModel.provider)) continue; const rawModelId = getSpecialtyModelRelativeId(musicModel.id, musicModel.provider); if (!providerSupportsModel(musicModel.provider, rawModelId)) continue; - if (getModelIsHidden(musicModel.provider, rawModelId)) continue; + if (isModelHiddenBulk(musicModel.provider, rawModelId)) continue; models.push({ id: musicModel.id, object: "model", @@ -1314,7 +1372,7 @@ async function buildUnifiedModelsResponseCore( if (!isUnifiedChatSourceModelSelectable(canonicalProviderId, { ...model, id: modelId })) continue; if (model.isHidden === true) continue; - if (getModelIsHidden(canonicalProviderId, modelId)) continue; + if (isModelHiddenBulk(canonicalProviderId, modelId)) continue; if (isExcludedByProviderConnections(canonicalProviderId, modelId)) continue; // #6328: apply hidePaidModels to user-defined custom rows too. // Custom entries do not carry pricing, so shouldHidePaid() decides @@ -1450,6 +1508,9 @@ async function buildUnifiedModelsResponseCore( ...(providerVisionFields || {}), }); } + + // #9147: custom-model walk — yield periodically. + await maybeYieldCatalogBuild(); } } } catch (e) { @@ -1495,7 +1556,7 @@ async function buildUnifiedModelsResponseCore( continue; } - if (getModelIsHidden(canonicalProviderId, modelId)) continue; + if (isModelHiddenBulk(canonicalProviderId, modelId)) continue; if (isExcludedByProviderConnections(canonicalProviderId, modelId)) continue; // #6328: apply hidePaidModels to alias-backed rows too. Alias mappings // point at providerKey/modelId with no pricing, so shouldHidePaid() @@ -1569,7 +1630,7 @@ async function buildUnifiedModelsResponseCore( for (const model of fallbackModels) { const modelId = typeof model.id === "string" ? model.id : null; if (!modelId) continue; - if (getModelIsHidden(canonicalProviderId, modelId)) continue; + if (isModelHiddenBulk(canonicalProviderId, modelId)) continue; if (isExcludedByProviderConnections(canonicalProviderId, modelId)) continue; // #6328: apply hidePaidModels to managed-fallback rows too. Compatible // provider fallbacks lack pricing; shouldHidePaid() decides via the @@ -1597,6 +1658,9 @@ async function buildUnifiedModelsResponseCore( ...(contextLength ? { context_length: contextLength } : {}), ...(visionFields || {}), }); + + // #9147: per-connection fallback walk — yield periodically. + await maybeYieldCatalogBuild(); } } @@ -1642,7 +1706,7 @@ async function buildUnifiedModelsResponseCore( } } // ?configuredOnly — hide models that have no eligible DB connection. - finalModels = applyCatalogPostFilters(request, finalModels, { + finalModels = await applyCatalogPostFilters(request, finalModels, { connections, prefixMode, aliasToProviderId, diff --git a/src/app/api/v1/models/catalogResponse.ts b/src/app/api/v1/models/catalogResponse.ts index 7da55d5bca..198a5a6d30 100644 --- a/src/app/api/v1/models/catalogResponse.ts +++ b/src/app/api/v1/models/catalogResponse.ts @@ -32,6 +32,7 @@ import { enrichCatalogModelEntry, type CatalogEnrichmentSnapshot, } from "@/lib/modelMetadataRegistry"; +import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; import { isModelCatalogNamesEnabled } from "@/shared/utils/featureFlags"; import { extractApiKey } from "@/sse/services/auth"; import { maybeOmitCatalogModelName } from "./catalogHelpers"; @@ -45,7 +46,7 @@ import { isCodexModelCatalogClient } from "./catalogRequest"; * returns early, but it still owes the caller these steps — the discovery mirrors in * particular are what let Claude Code see a quota pool's models at all. */ -export function applyCatalogPostFilters( +export async function applyCatalogPostFilters( request: Request, models: Array>, ctx: { @@ -54,7 +55,8 @@ export function applyCatalogPostFilters( aliasToProviderId: Record; hideNoThinkVariants?: boolean; } -): Array> { +): Promise>> { + const yieldTurn = (): Promise => new Promise((resolve) => setImmediate(resolve)); let finalModels = models; // variants are only generated for surviving models. @@ -65,6 +67,11 @@ export function applyCatalogPostFilters( }); } + // #9147: the variant-append passes each walk the full model list (O(n) per pass), + // so a catalog-scale build must not run all of them in one synchronous stretch. + // Yield once between the expensive passes to let the event loop breathe. + await yieldTurn(); + // Advertise Claude reasoning-effort variants (claude/-{low,medium,high[,xhigh]}). // Derived from the already key-filtered list so a variant only appears when its real // model is permitted. Runs before the no-thinking pass: the gateway already routes these @@ -139,11 +146,15 @@ export function applyCatalogPostFilters( ); } + await yieldTurn(); + // #7694: advertise `/-` variants for synced models that // captured `reasoning.supported_efforts` at sync time (capabilities.effort_tiers). // Derived from the already key-filtered list; skips codex/kimi (own suffix mechanism). finalModels = appendSyncedEffortVariants(finalModels); + await yieldTurn(); + // #4424 follow-up — drop exact-duplicate ids that slip through the per-source push // guards (e.g. `codex/gpt-5.5`, `veo-free/seedance` listed twice). Keyed by listing // identity (id, type, subtype) so the intentional same-id audio transcription/speech @@ -207,25 +218,45 @@ export async function finalizeCatalogResponse( } const includeModelNames = isModelCatalogNamesEnabled(); - const enrichedModels = disambiguateCatalogModelNames( - finalModels.map((model) => { - if (model.owned_by === "combo") { - return maybeOmitCatalogModelName(model, includeModelNames); - } - const enriched = enrichCatalogModelEntry(model, undefined, enrichmentSnapshot); - const fallbackContextLength = getContextFallback(enriched); - const listedModel = fallbackContextLength - ? { ...enriched, context_length: fallbackContextLength } - : enriched; - return maybeOmitCatalogModelName(listedModel, includeModelNames); - }) - ); - // Canonical provider-grouped publication: one contiguous block per provider, - // combos pinned first. Stable — preserves combo sort_order, connection priority, - // and equal-id audio twins. Grouped by owned_by (canonical identity), not the - // routing alias prefix. Applied after enrichment/disambiguation so the final - // serialized order is what every consumer sees; cached as part of the body. + // #9147: enrichment is the most expensive single stage of the catalog build — + // per-entry provider/model resolution plus pricing + token/context override + // lookups. Two fixes so a large catalog cannot pin the Node.js thread here: + // (1) bulk-load the synced-capability + override tables ONCE into an in-memory + // snapshot (#9199 machinery) so per-entry enrichment never hits SQLite; + // (2) yield to the event loop every `YIELD_EVERY` entries so even the remaining + // per-entry work is interleaved with other callers / the dashboard WS. + const yieldTurn = (): Promise => new Promise((resolve) => setImmediate(resolve)); + await yieldTurn(); + const capabilityResolutionSnapshot = createModelCapabilityResolutionSnapshot(); + const enriched: Array> = []; + const catYIELD_EVERY = 5; + let catEnrichCount = 0; + for (const model of finalModels) { + let listedModel: Record; + if (model.owned_by === "combo") { + listedModel = maybeOmitCatalogModelName(model, includeModelNames); + } else { + const entry = enrichCatalogModelEntry(model, undefined, { + ...enrichmentSnapshot, + capabilityResolutionSnapshot, + }); + const fallbackContextLength = getContextFallback(entry); + listedModel = fallbackContextLength + ? { ...entry, context_length: fallbackContextLength } + : entry; + listedModel = maybeOmitCatalogModelName(listedModel, includeModelNames); + } + enriched.push(listedModel); + catEnrichCount++; + if (catEnrichCount % catYIELD_EVERY === 0) { + await yieldTurn(); + } + } + await yieldTurn(); + const enrichedModels = disambiguateCatalogModelNames(enriched); + await yieldTurn(); const orderedModels = sortCatalogModelsProviderGrouped(enrichedModels); + await yieldTurn(); // Codex CLI compatibility: its model-catalog refresh (codex_models_manager) does // GET /v1/models?client_version= and decodes a JSON object with a TOP-LEVEL // `models` array, so the OpenAI-standard `{object,data}` shape makes it fail with diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index 83afd6a34b..b354bba3b5 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -567,9 +567,12 @@ function getContextOverride( /** * Resolve a persisted context override by canonical id, then by the exact raw * alias supplied by the caller. Neither lookup inherits to related models. + * + * `snapshot` is the #9147 build-local bulk load; when supplied the on-demand + * SQLite read is skipped and the preloaded nested map is used instead. */ -export function getResolvedModelContextOverride(input: CapabilityInput): number | null { - return getContextOverride(resolveCapabilityInput(input)); +export function getResolvedModelContextOverride(input: CapabilityInput, snapshot?: ModelCapabilityResolutionSnapshot | null): number | null { + return getContextOverride(resolveCapabilityInput(input), snapshot); } function getInputTokenCapabilityOverride(resolved: { diff --git a/src/lib/modelMetadataRegistry.ts b/src/lib/modelMetadataRegistry.ts index 667230839b..413836e5a6 100644 --- a/src/lib/modelMetadataRegistry.ts +++ b/src/lib/modelMetadataRegistry.ts @@ -28,6 +28,7 @@ import { CANONICAL_EFFORT_VALUES, extendCodexGpt56EffortValues, } from "@/shared/reasoning/effortStandardization"; +import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; const MODEL_METADATA_SCHEMA_VERSION = "model-metadata-v1"; @@ -40,6 +41,9 @@ type JsonRecord = Record; export interface CatalogEnrichmentSnapshot { modelsDevPricing: PricingByProvider | null; providerNodeIdsByPrefix?: Readonly>; + /** #9147: build-local bulk load of synced capabilities + token/context overrides + * so per-entry enrichment never hits SQLite again (see catalogResponse.ts). */ + capabilityResolutionSnapshot?: ModelCapabilityResolutionSnapshot | null; } interface CatalogDiagnosticsOptions { @@ -200,20 +204,27 @@ export function getCatalogDiagnosticsHeaders( export function getCanonicalModelMetadata(input: { provider?: string | null; model?: string | null; + snapshot?: ModelCapabilityResolutionSnapshot | null; }): CanonicalModelMetadata | null { const modelId = asNonEmptyString(input.model); if (!modelId) return null; - const resolved = getResolvedModelCapabilities({ - provider: input.provider || null, - model: modelId, - }); + const resolved = getResolvedModelCapabilities( + { + provider: input.provider || null, + model: modelId, + }, + undefined, + input.snapshot || null + ); const provider = resolved.provider; const providerAlias = provider ? PROVIDER_ID_TO_ALIAS[provider] || provider : null; const registryModel = getRegistryModel(providerAlias || provider, resolved.model || modelId); const staticSpec = getModelSpec(resolved.model || modelId); const syncedCapability = - provider && resolved.model ? getSyncedCapability(provider, resolved.model) : null; + provider && resolved.model + ? getSyncedCapability(provider, resolved.model, input.snapshot?.synced ?? null) + : null; const canonicalStaticAlias = resolveStaticModelAlias(resolved.model || modelId); const modalities = buildModalities( resolved.modalitiesInput, @@ -420,7 +431,11 @@ export function enrichCatalogModelEntry( return id; })(); - const metadata = getCanonicalModelMetadata({ provider, model }); + const metadata = getCanonicalModelMetadata({ + provider, + model, + snapshot: snapshot?.capabilityResolutionSnapshot ?? null, + }); if (!metadata) return entry; const registryModel = getRegistryModel( metadata.providerAlias || metadata.provider, @@ -436,7 +451,11 @@ export function enrichCatalogModelEntry( getAuthoritativeContextWindow(metadata.model) ?? getAuthoritativeContextWindow(model); const specialtySurface = isNonChatCatalogSurface(entry.type); - const persistedContextWindow = getResolvedModelContextOverride({ provider, model }); + const capabilitySnapshot = snapshot?.capabilityResolutionSnapshot ?? null; + const persistedContextWindow = getResolvedModelContextOverride( + { provider, model }, + capabilitySnapshot + ); const capabilityFields = { ...(typeof metadata.capabilities.vision === "boolean" ? { vision: metadata.capabilities.vision } @@ -528,10 +547,15 @@ export function enrichCatalogModelEntry( } const persistedOutputLimit = - getModelCapabilityOverride(provider, model, "max_output_tokens") ?? - getModelCapabilityOverride(provider, model, "max_token") ?? - getModelCapabilityOverride(publicProvider, model, "max_output_tokens") ?? - getModelCapabilityOverride(publicProvider, model, "max_token"); + getModelCapabilityOverride(provider, model, "max_output_tokens", capabilitySnapshot?.maxTokenOverrides) ?? + getModelCapabilityOverride(provider, model, "max_token", capabilitySnapshot?.maxTokenOverrides) ?? + getModelCapabilityOverride( + publicProvider, + model, + "max_output_tokens", + capabilitySnapshot?.maxTokenOverrides + ) ?? + getModelCapabilityOverride(publicProvider, model, "max_token", capabilitySnapshot?.maxTokenOverrides); if (persistedOutputLimit !== null) { nextEntry.max_output_tokens = persistedOutputLimit; } else if ( diff --git a/tests/unit/10313-catalog-cache-key-hashing.test.ts b/tests/unit/10313-catalog-cache-key-hashing.test.ts new file mode 100644 index 0000000000..265fed1150 --- /dev/null +++ b/tests/unit/10313-catalog-cache-key-hashing.test.ts @@ -0,0 +1,131 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-catalog-keyleak-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-keyleak-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts"); +const catalogCacheMod = await import("../../src/app/api/v1/models/catalogCache.ts"); + +const SECRET = "sk-live-PROBE-10313-SUPER-SECRET-TOKEN"; + +test.beforeEach(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + v1ModelsCatalog.__resetCatalogBuilderRunsForTest(); +}); + +test.after(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function captureMapKeys(): { keys: string[]; restore: () => void } { + const capturedKeys: string[] = []; + const originalSet = Map.prototype.set; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (Map.prototype as any).set = function (key: unknown, value: unknown) { + if (typeof key === "string") capturedKeys.push(key); + return originalSet.call(this, key, value); + }; + return { + keys: capturedKeys, + release: () => { + Map.prototype.set = originalSet; + }, + }; +} + +// buildCatalogCacheKey emits `prefix|isCodex|apiKeyFingerprint|configuredOnly|hideAuto|hideNoThink` +// (6 pipe-delimited fields). Other in-flight keys (e.g. `x-request-id`) don't match. +function isCatalogCacheKey(k: string): boolean { + return k.split("|").length === 6; +} + +test("catalog cache Map keys must not contain the raw bearer API key (#10313)", async () => { + const probe = captureMapKeys(); + + try { + const request = new Request("http://localhost/v1/models", { + headers: { Authorization: `Bearer ${SECRET}` }, + }); + const res = await v1ModelsCatalog.getUnifiedModelsResponse(request); + assert.ok(res.status === 200 || res.status === 401 || res.status === 403); + } finally { + probe.release(); + } + + const catalogKeys = probe.keys.filter(isCatalogCacheKey); + assert.ok( + catalogKeys.length > 0, + "no catalog cache Map.set() calls observed — the probe did not exercise catalogCache/catalogInFlight" + ); + + const leaked = catalogKeys.filter((k) => k.includes(SECRET)); + assert.deepEqual( + leaked, + [], + `catalog cache Map key retained the raw API key verbatim: ${JSON.stringify(leaked)} — ` + + `buildCatalogCacheKey() must hash the secret (e.g. sha256) before using it as a Map key` + ); + + assert.ok(catalogCacheMod.CATALOG_CACHE_TTL_MS_DEFAULT > 0); +}); + +test("cache keys embed the sha256 digest of the secret, never the raw secret (#10313)", async () => { + // Capture ALL Map keys emitted across BOTH requests (a 2nd identical-secret request + // may be a cache hit and emit no new catalog key — irrelevant here: we assert on the + // digest that did appear). + const probe = captureMapKeys(); + try { + const resA = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/v1/models", { + headers: { Authorization: "Bearer sk-10313-DIGEST-A" }, + }) + ); + const resB = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/v1/models", { + headers: { Authorization: "Bearer sk-10313-DIGEST-B" }, + }) + ); + assert.ok(resA.status === 200 || resA.status === 401 || resA.status === 403); + assert.ok(resB.status === 200 || resB.status === 401 || resB.status === 403); + } finally { + probe.release(); + } + + const catalogKeys = probe.keys.filter(isCatalogCacheKey); + assert.ok(catalogKeys.length > 0, "expected catalog cache Map.set() calls"); + + // #10538 sync note: buildCatalogCacheKey() delegates to the canonical + // fingerprintCatalogAuthKey() (landed on release/v3.8.50 independently of #10313), + // which truncates the sha256 hex digest to 16 chars for a shorter Map key. Derive + // the expected fingerprint the same way rather than re-hardcoding the full digest. + const digestA = catalogCacheMod.fingerprintCatalogAuthKey("sk-10313-DIGEST-A"); + const digestB = catalogCacheMod.fingerprintCatalogAuthKey("sk-10313-DIGEST-B"); + const rawA = "sk-10313-DIGEST-A"; + const rawB = "sk-10313-DIGEST-B"; + + // The hashed fingerprint, not the raw secret, rides in the cache keys. + const keysWithDigestA = catalogKeys.filter((k) => k.includes(digestA)); + const keysWithDigestB = catalogKeys.filter((k) => k.includes(digestB)); + assert.ok(keysWithDigestA.length > 0, `expected a cache key embedding the fingerprint of A: ${catalogKeys.join(",")}`); + assert.ok(keysWithDigestB.length > 0, `expected a cache key embedding the fingerprint of B: ${catalogKeys.join(",")}`); + + // Raw secrets must never appear (issue #10313 root cause). + assert.ok(!catalogKeys.some((k) => k.includes(rawA) || k.includes(rawB))); + + // Identical secrets ⇒ identical key (memoized reuse); different ⇒ distinct. + assert.ok(keysWithDigestA.every((k) => k === keysWithDigestA[0]), "all A keys must be identical"); + assert.ok(keysWithDigestB.every((k) => k === keysWithDigestB[0]), "all B keys must be identical"); + assert.notEqual(keysWithDigestA[0], keysWithDigestB[0]); +}); \ No newline at end of file diff --git a/tests/unit/9147-catalog-eventloop-yield.test.ts b/tests/unit/9147-catalog-eventloop-yield.test.ts new file mode 100644 index 0000000000..d915660896 --- /dev/null +++ b/tests/unit/9147-catalog-eventloop-yield.test.ts @@ -0,0 +1,88 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-9147-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-9147-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts"); + +const CONNECTION_COUNT = 60; +const MODELS_PER_CONNECTION = 12; // ~720 synced models total + +async function resetStorage() { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + v1ModelsCatalog.__resetCatalogBuilderRunsForTest(); +} + +async function seedCatalogScaleDataset() { + const db = core.getDbInstance(); + const now = new Date().toISOString(); + const insertConn = db.prepare( + `INSERT INTO provider_connections (id, provider, auth_type, name, priority, is_active, api_key, created_at, updated_at) + VALUES (?, 'openai-compatible', 'apikey', ?, ?, 1, ?, ?, ?)` + ); + const insertModels = db.prepare( + `INSERT INTO key_value (namespace, key, value) VALUES ('syncedAvailableModels', ?, ?)` + ); + const seedTx = db.transaction(() => { + for (let i = 0; i < CONNECTION_COUNT; i++) { + const id = `probe-conn-${i}`; + insertConn.run(id, `probe-connection-${i}`, i, `sk-probe-${i}`, now, now); + const models = Array.from({ length: MODELS_PER_CONNECTION }, (_, m) => ({ + id: `probe-model-${i}-${m}`, + name: `Probe Model ${i}-${m}`, + contextLength: 128000, + })); + insertModels.run(`openai-compatible:${id}`, JSON.stringify(models)); + } + }); + seedTx(); +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(async () => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("#9147 — catalog build at catalog-scale must not pin the event loop for a long stretch", async () => { + await seedCatalogScaleDataset(); + const req = new Request("http://localhost/v1/models"); + let settled = false; + const buildPromise = v1ModelsCatalog.getUnifiedModelsResponse(req).then((res) => { + settled = true; + return res; + }); + let lastTick = performance.now(); + let maxGapMs = 0; + let ticks = 0; + while (!settled) { + await new Promise((resolve) => setTimeout(resolve, 0)); + const now = performance.now(); + maxGapMs = Math.max(maxGapMs, now - lastTick); + lastTick = now; + ticks++; + if (ticks > 20000) break; + } + const res = await buildPromise; + assert.equal(res.status, 200); + assert.ok( + maxGapMs < 150, + `event loop was blocked for ${maxGapMs.toFixed(1)}ms in a single stretch while building the ` + + `catalog for ${CONNECTION_COUNT} connections / ${CONNECTION_COUNT * MODELS_PER_CONNECTION} models ` + + `(${ticks} interleaved ticks observed) — the builder is not yielding to the event loop` + ); +}); \ No newline at end of file diff --git a/tests/unit/models-catalog-functional-gateway.test.ts b/tests/unit/models-catalog-functional-gateway.test.ts index 6a3ed07f57..35ddb10cb9 100644 --- a/tests/unit/models-catalog-functional-gateway.test.ts +++ b/tests/unit/models-catalog-functional-gateway.test.ts @@ -24,11 +24,11 @@ function makeRequest(query = ""): Request { return new Request(`http://localhost/v1/models${query}`); } -test("catalog post-filters do not add mirrors when gate off (default)", () => { +test("catalog post-filters do not add mirrors when gate off (default)", async () => { const models = [ { id: "deepseek/deepseek-v4-flash", owned_by: "deepseek", root: "deepseek-v4-flash" }, ]; - const out = applyCatalogPostFilters(makeRequest(), models, { + const out = await applyCatalogPostFilters(makeRequest(), models, { connections: [], prefixMode: "dual", aliasToProviderId: {}, @@ -41,7 +41,7 @@ test("final catalog permission filtering does not let a mirror inherit base acce setFunctionalGatewayProviderSetting("agentrouter", "on"); const models = [{ id: "kmc/k3", owned_by: "kimi-coding", root: "k3" }]; - const withMirror = applyCatalogPostFilters(makeRequest(), models, { + const withMirror = await applyCatalogPostFilters(makeRequest(), models, { connections: [ { id: "conn-1", @@ -77,14 +77,14 @@ test("final catalog permission filtering does not let a mirror inherit base acce ); }); -test("catalog post-filters synthesize a gateway mirror when gate on and gateway has a connection", () => { +test("catalog post-filters synthesize a gateway mirror when gate on and gateway has a connection", async () => { setFeatureFlagOverride(FLAG_KEY, "true"); setFunctionalGatewayProviderSetting("agentrouter", "on"); const models = [ { id: "deepseek/deepseek-v4-flash", owned_by: "deepseek", root: "deepseek-v4-flash" }, ]; - const out = applyCatalogPostFilters(makeRequest(), models, { + const out = await applyCatalogPostFilters(makeRequest(), models, { connections: [ { id: "conn-1", diff --git a/tests/unit/models-catalog-route.test.ts b/tests/unit/models-catalog-route.test.ts index d3c073d8f9..95a4dd01fa 100644 --- a/tests/unit/models-catalog-route.test.ts +++ b/tests/unit/models-catalog-route.test.ts @@ -1343,18 +1343,24 @@ test("v1 models catalog returns 500 when model compatibility lookup crashes", as db.prepare = (sql) => { const statement = originalPrepare(sql); - if (String(sql) !== "SELECT value FROM key_value WHERE namespace = ? AND key = ?") { + // #9147: the catalog builder now resolves hidden models via a single bulk + // read (`getHiddenModelsByProvider()`, src/lib/db/models.ts) instead of the + // old per-provider `SELECT value FROM key_value WHERE namespace = ? AND + // key = ?` / readCompatList() lookup — intercept the bulk query's `.all()` + // call so this test still exercises "DB read for model visibility crashes + // -> catalog endpoint surfaces 500" against the current implementation. + if ( + String(sql) !== + "SELECT namespace, key, value FROM key_value WHERE namespace IN ('modelCompatOverrides', 'customModels')" + ) { return statement; } return new Proxy(statement, { get(target, prop, receiver) { - if (prop === "get") { + if (prop === "all") { return (...args) => { - if (args[0] === "modelCompatOverrides") { - throw new Error("compat lookup boom"); - } - return target.get(...args); + throw new Error("compat lookup boom"); }; } return Reflect.get(target, prop, receiver);