pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-11 17:52:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

[v3.8.50] fix(models): keep model catalogs responsive (#9199)

Browse Source 
* fix(models): preserve catalog on affinity bookkeeping

Related to #8697.

Focused follow-up to #8728; this does not replace or supersede that contribution.

* docs(changelog): record model catalog affinity fix

* fix(models): keep cold catalog builds responsive

* docs(changelog): record catalog responsiveness fix

* fix(models): snapshot auto candidate capabilities

* fix(models): invalidate capability catalog snapshots

* test(models): register catalog invalidation coverage

* fix(models): bulk-load catalog capability snapshots

Resolve synced capabilities and persisted overrides from one build-local view instead of repeating per-target SQLite reads. Keep ordinary runtime lookups on demand and preserve catalog generation invalidation.

Refs: #9199

* fix(models): snapshot catalog pricing once per build

Production profiling showed per-model models.dev pricing reads and JSON parsing dominated cold catalog builds. Reuse one build-local pricing snapshot during enrichment and yield before publication so queued health checks can run, while preserving fresh reads for ordinary callers.

* docs(changelog): record catalog pricing snapshot
This commit is contained in:
Xiangzhe
2026-08-11 20:59:45 +08:00
committed by GitHub
parent 2b2d947faf
commit 36959c37d3
22 changed files with 1958 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
changelog.d/fixes/9199-model-catalog-affinity.md Normal file
View File

@@ -0,0 +1,4 @@
- **fix(models):** Preserve published model catalogs during session-affinity bookkeeping so routine affinity updates do not force unnecessary cold rebuilds ([#9199](https://github.com/diegosouzapw/OmniRoute/pull/9199)) — thanks @xz-dev
- **fix(models):** Reuse one build-local virtual-auto candidate snapshot across built-in catalog entries and cooperatively yield during cold catalog generation, while detaching invalidated in-flight generations so policy changes cannot publish stale results ([#9199](https://github.com/diegosouzapw/OmniRoute/pull/9199)) — thanks @xz-dev
- **fix(models):** Resolve token limits and model capabilities once per unique candidate in that build-local snapshot, eliminating repeated SQLite lookups across the 38 built-in auto entries while preserving fresh runtime preparation and hard invalidation ([#9199](https://github.com/diegosouzapw/OmniRoute/pull/9199)) — thanks @xz-dev
- **fix(models):** Read and parse models.dev pricing once per cold catalog build, then yield before final enrichment so queued health checks are not starved while every model in that build shares one coherent pricing snapshot ([#9199](https://github.com/diegosouzapw/OmniRoute/pull/9199)) — thanks @xz-dev

33
open-sse/services/autoCombo/builtinCatalog.ts
View File

@@ -1,5 +1,6 @@
import type { AutoVariant } from "./autoPrefix";
import { VALID_VARIANTS } from "./autoPrefix";
import type { PreparedVirtualAutoComboInputs } from "./virtualFactory";
import { parseAutoSuffix, type AutoCategory, type AutoTier } from "./suffixComposition";
import { isValidModelFamily, AUTO_FAMILY_IDS } from "./modelFamily";
@@ -158,13 +159,31 @@ export function resolveBuiltinAutoSpec(modelStr: string, suffix: string): Builti
return { variant: undefined };
}
export async function createBuiltinAutoCombo(modelStr: string, suffix: string) {
const { createVirtualAutoCombo } = await import("./virtualFactory.ts");
export async function prepareBuiltinAutoComboInputs(): Promise<PreparedVirtualAutoComboInputs> {
const { prepareVirtualAutoComboInputs } = await import("./virtualFactory.ts");
return prepareVirtualAutoComboInputs({ includeResolvedCapabilities: true });
}
export async function createBuiltinAutoCombo(
modelStr: string,
suffix: string,
prepared?: PreparedVirtualAutoComboInputs
) {
const { createVirtualAutoCombo, createVirtualAutoComboFromPrepared } =
await import("./virtualFactory.ts");
const materialize = (
variant: AutoVariant | undefined,
spec?: Parameters<typeof createVirtualAutoCombo>[1]
) =>
prepared
? createVirtualAutoComboFromPrepared(prepared, variant, spec)
: createVirtualAutoCombo(variant, spec);
const spec = resolveBuiltinAutoSpec(modelStr, suffix);
if ("category" in spec) {
// #4235 Phase B category/tier path (incl. vision ids like auto/best-vision).
const virtualCombo = await createVirtualAutoCombo(undefined, {
const virtualCombo = await materialize(undefined, {
category: spec.category,
...(spec.tier ? { tier: spec.tier } : {}),
});
@@ -174,7 +193,7 @@ export async function createBuiltinAutoCombo(modelStr: string, suffix: string) {
}
if ("variant" in spec && spec.variant !== undefined) {
const virtualCombo = await createVirtualAutoCombo(spec.variant, {
const virtualCombo = await materialize(spec.variant, {
...(modelStr === "auto/best-free" ? { tier: "free" as const } : {}),
});
virtualCombo.name = modelStr;
@@ -186,7 +205,7 @@ export async function createBuiltinAutoCombo(modelStr: string, suffix: string) {
// auto/best-chat, auto/pro-chat) still materialize via the default
// (unconstrained) virtual combo rather than throwing "Unknown built-in".
if (Object.prototype.hasOwnProperty.call(AUTO_TEMPLATE_VARIANTS, modelStr)) {
const virtualCombo = await createVirtualAutoCombo(undefined);
const virtualCombo = await materialize(undefined);
virtualCombo.name = modelStr;
virtualCombo.id = modelStr;
return virtualCombo;
@@ -195,7 +214,7 @@ export async function createBuiltinAutoCombo(modelStr: string, suffix: string) {
// #4235 Phase B: `auto/<category>[:<tier>]` (e.g. auto/coding:fast, auto/vision).
const parsed = parseAutoSuffix(suffix);
if (parsed.valid) {
const virtualCombo = await createVirtualAutoCombo(undefined, {
const virtualCombo = await materialize(undefined, {
category: parsed.category,
tier: parsed.tier,
});
@@ -208,7 +227,7 @@ export async function createBuiltinAutoCombo(modelStr: string, suffix: string) {
// auto/gemma, auto/llama, auto/gemini) — spans whatever installed backends
// currently expose that model family, degrading gracefully as backends rotate.
if (isValidModelFamily(suffix)) {
const virtualCombo = await createVirtualAutoCombo(undefined, { family: suffix });
const virtualCombo = await materialize(undefined, { family: suffix });
virtualCombo.name = modelStr;
virtualCombo.id = modelStr;
return virtualCombo;

9
open-sse/services/autoCombo/suffixComposition.ts
View File

@@ -95,6 +95,9 @@ export function tierToWeightVariant(tier?: AutoTier): AutoVariant | "reliability
interface PoolCandidate {
provider: string;
model: string;
resolvedSupportsVision?: boolean;
resolvedReasoning?: boolean;
resolvedSupportsThinking?: boolean;
}
/**
@@ -110,6 +113,9 @@ export function buildAutoCandidateFilter(
if (category === "vision" || category === "multimodal") {
checks.push((c) => {
if (c.resolvedSupportsVision !== undefined) {
return c.resolvedSupportsVision || isVisionModelId(c.model);
}
try {
const caps = getResolvedModelCapabilities({ provider: c.provider, model: c.model });
const capable =
@@ -127,6 +133,9 @@ export function buildAutoCandidateFilter(
}
if (category === "reasoning") {
checks.push((c) => {
if (c.resolvedReasoning !== undefined && c.resolvedSupportsThinking !== undefined) {
return c.resolvedReasoning || c.resolvedSupportsThinking;
}
try {
const caps = getResolvedModelCapabilities({ provider: c.provider, model: c.model });
return caps.reasoning === true || caps.supportsThinking === true;

243
open-sse/services/autoCombo/virtualFactory.ts
View File

@@ -9,7 +9,11 @@ import { NOAUTH_PROVIDERS } from "@/shared/constants/providers";
import { hasUsableWebSessionCredential } from "@/shared/providers/webSessionCredentials";
import { defaultLogger as log } from "@omniroute/open-sse/utils/logger";
import { getTokenLimit } from "../contextManager";
import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
import {
createModelCapabilityResolutionSnapshot,
getResolvedModelCapabilities,
type ModelCapabilityResolutionSnapshot,
} from "@/lib/modelCapabilities";
import {
buildAutoCandidateFilter,
tierToWeightVariant,
@@ -65,6 +69,12 @@ export interface VirtualAutoComboCandidate {
model: string;
modelStr: string; // e.g., 'openai/gpt-4o'
costPer1MTokens: number; // from providerRegistry
/** Build-local capability snapshot. Runtime calls rebuild it; catalog entries reuse it. */
resolvedContextLength?: number | null;
resolvedMaxOutputTokens?: number | null;
resolvedSupportsVision?: boolean;
resolvedReasoning?: boolean;
resolvedSupportsThinking?: boolean;
}
type VirtualAutoCombo = AutoComboConfig & {
@@ -106,6 +116,15 @@ type VirtualAutoCombo = AutoComboConfig & {
};
};
/**
* Build-local candidate snapshots shared by the built-in entries in one model-catalog build.
* Runtime routing does not retain or reuse this object across requests.
*/
export interface PreparedVirtualAutoComboInputs {
readonly regularCandidates: readonly VirtualAutoComboCandidate[];
readonly familyCandidates: readonly VirtualAutoComboCandidate[];
}
function toExpiryMs(value: unknown): number | null {
if (value === null || value === undefined || value === "") return null;
@@ -289,7 +308,14 @@ function getNoAuthCandidates(
*/
const DEFAULT_ADVERTISED_MAX_OUTPUT_TOKENS = 8192;
export function computeAdvertisedLimits(candidates: Array<{ provider: string; model: string }>): {
type AdvertisedLimitCandidate = {
provider: string;
model: string;
resolvedContextLength?: number | null;
resolvedMaxOutputTokens?: number | null;
};
export function computeAdvertisedLimits(candidates: AdvertisedLimitCandidate[]): {
contextLength: number | null;
maxOutputTokens: number | null;
} {
@@ -300,14 +326,20 @@ export function computeAdvertisedLimits(candidates: Array<{ provider: string; mo
let contextLength: number | null = null;
let maxOutputTokens: number | null = null;
for (const candidate of candidates) {
const limit = getTokenLimit(candidate.provider, candidate.model);
if (Number.isFinite(limit) && limit > 0) {
const limit =
candidate.resolvedContextLength !== undefined
? candidate.resolvedContextLength
: getTokenLimit(candidate.provider, candidate.model);
if (typeof limit === "number" && Number.isFinite(limit) && limit > 0) {
contextLength = contextLength === null ? limit : Math.max(contextLength, limit);
}
const output = getResolvedModelCapabilities({
provider: candidate.provider,
model: candidate.model,
}).maxOutputTokens;
const output =
candidate.resolvedMaxOutputTokens !== undefined
? candidate.resolvedMaxOutputTokens
: getResolvedModelCapabilities({
provider: candidate.provider,
model: candidate.model,
}).maxOutputTokens;
if (typeof output === "number" && Number.isFinite(output) && output > 0) {
maxOutputTokens = maxOutputTokens === null ? output : Math.max(maxOutputTokens, output);
}
@@ -318,12 +350,82 @@ export function computeAdvertisedLimits(candidates: Array<{ provider: string; mo
return { contextLength, maxOutputTokens };
}
export async function createVirtualAutoCombo(
variant: AutoVariant | undefined,
spec?: AutoComboSpec,
apiKeyId?: string,
autoChannel?: string
): Promise<VirtualAutoCombo> {
const PREPARED_CAPABILITY_YIELD_INTERVAL = 16;
type PreparedCapabilityValues = {
resolvedContextLength: number | null;
resolvedMaxOutputTokens: number | null;
resolvedSupportsVision: boolean;
resolvedReasoning: boolean;
resolvedSupportsThinking: boolean;
};
type PreparedCapabilityState = {
/** Nested provider → model memo; collision-free for arbitrary model ids. */
byTarget: Map<string, Map<string, PreparedCapabilityValues>>;
resolvedSinceYield: number;
/** Build-local bulk maps; one per catalog prepare, never retained at runtime. */
resolutionSnapshot: ModelCapabilityResolutionSnapshot;
};
function yieldVirtualAutoPreparationTurn(): Promise<void> {
return new Promise((resolve) => setImmediate(resolve));
}
async function attachPreparedCapabilityValues(
candidates: readonly VirtualAutoComboCandidate[],
state: PreparedCapabilityState
): Promise<VirtualAutoComboCandidate[]> {
const prepared: VirtualAutoComboCandidate[] = [];
for (const candidate of candidates) {
let byModel = state.byTarget.get(candidate.provider);
if (!byModel) {
byModel = new Map();
state.byTarget.set(candidate.provider, byModel);
}
let values = byModel.get(candidate.model);
if (!values) {
const contextLength = getTokenLimit(
candidate.provider,
candidate.model,
state.resolutionSnapshot
);
const capabilities = getResolvedModelCapabilities(
{
provider: candidate.provider,
model: candidate.model,
},
state.resolutionSnapshot
);
const maxOutputTokens = capabilities.maxOutputTokens;
values = {
resolvedContextLength:
Number.isFinite(contextLength) && contextLength > 0 ? contextLength : null,
resolvedMaxOutputTokens:
typeof maxOutputTokens === "number" &&
Number.isFinite(maxOutputTokens) &&
maxOutputTokens > 0
? maxOutputTokens
: null,
resolvedSupportsVision: capabilities.supportsVision === true,
resolvedReasoning: capabilities.reasoning === true,
resolvedSupportsThinking: capabilities.supportsThinking === true,
};
byModel.set(candidate.model, values);
state.resolvedSinceYield++;
if (state.resolvedSinceYield >= PREPARED_CAPABILITY_YIELD_INTERVAL) {
state.resolvedSinceYield = 0;
await yieldVirtualAutoPreparationTurn();
}
}
prepared.push({ ...candidate, ...values });
}
return prepared;
}
export async function prepareVirtualAutoComboInputs(
options: { includeResolvedCapabilities?: boolean } = {}
): Promise<PreparedVirtualAutoComboInputs> {
const [connections, disabledNoAuthConnections, settings] = await Promise.all([
getCachedProviderConnections({ isActive: true }) as Promise<VirtualFactoryConn[]>,
// #6557: no-auth providers (opencode/mimocode/etc.) don't get an isActive
@@ -405,50 +507,79 @@ export async function createVirtualAutoCombo(
}
}
candidatePool.push(
...getNoAuthCandidates(
new Set(validConnections.map((conn) => conn.provider)),
blockedProviders,
disabledNoAuthProviders,
noAuthProviderSpecificData,
hiddenModelsMap,
// #6453/#8183 (operator decision 2026-07-24): auto/<family> combos are an
// identity selector, not a reliability-curated pool — bypass the no-auth
// allowlist gate so any backend that genuinely serves the family (e.g.
// auggie for auto/glm) is admitted. Category/tier and flat-variant pools
// (spec.family unset) keep the allowlist gate intact.
Boolean(spec?.family)
)
);
// #7623: honor existing model lockouts + connection cooldown/terminal state so
// auto/* never advertises models the dispatch path would immediately skip.
const connectionsById = new Map<string, ConnectionResilienceView>();
for (const conn of [...connections, ...disabledNoAuthConnections]) {
connectionsById.set(conn.id, conn);
}
const resilienceFilteredPool = filterResilienceBlockedCandidates(
candidatePool,
connectionsById
);
if (resilienceFilteredPool !== candidatePool) {
candidatePool.length = 0;
candidatePool.push(...resilienceFilteredPool);
const connectedProviders = new Set(validConnections.map((conn) => conn.provider));
const buildPreparedPool = (bypassNoAuthAllowlist: boolean) => {
let pool = [
...candidatePool,
...getNoAuthCandidates(
connectedProviders,
blockedProviders,
disabledNoAuthProviders,
noAuthProviderSpecificData,
hiddenModelsMap,
bypassNoAuthAllowlist
),
];
const resilienceFilteredPool = filterResilienceBlockedCandidates(pool, connectionsById);
if (resilienceFilteredPool !== pool) pool = resilienceFilteredPool;
// #6512 (follow-up to #6328/#6495): when the operator opts into `hidePaidModels`,
// exclude paid-only backends from EVERY `auto/*` candidate pool.
const paidFilteredPool = filterPaidOnlyCandidates(pool, settings.hidePaidModels === true);
if (paidFilteredPool !== pool) pool = paidFilteredPool;
return pool;
};
const regularCandidates = buildPreparedPool(false);
// #6453/#8183: family selectors bypass the reliability-curated no-auth allowlist.
const familyCandidates = buildPreparedPool(true);
if (!options.includeResolvedCapabilities) {
return { regularCandidates, familyCandidates };
}
// #6512 (follow-up to #6328/#6495): when the operator opts into `hidePaidModels`,
// exclude paid-only backends from EVERY `auto/*` candidate pool — not just the
// `/v1/models` listing — so auto-routing never picks a model that will 402/403.
// If this empties the pool the existing graceful empty-pool path below handles it
// (consistent with the opt-in intent). Default OFF → pool unchanged.
const paidFilteredPool = filterPaidOnlyCandidates(
candidatePool,
settings.hidePaidModels === true
// One uninterrupted bulk read of all three capability tables for this prepare only.
// Do not yield between the three loads; later cooperative yields remain fine because
// catalog generation guards already prevent publishing across intervening writes.
const capabilityState: PreparedCapabilityState = {
byTarget: new Map(),
resolvedSinceYield: 0,
resolutionSnapshot: createModelCapabilityResolutionSnapshot(),
};
return {
regularCandidates: await attachPreparedCapabilityValues(regularCandidates, capabilityState),
familyCandidates: await attachPreparedCapabilityValues(familyCandidates, capabilityState),
};
}
function clonePreparedCandidates(
candidates: readonly VirtualAutoComboCandidate[]
): VirtualAutoComboCandidate[] {
return candidates.map((candidate) => ({
...candidate,
...(candidate.allowedConnectionIds
? { allowedConnectionIds: [...candidate.allowedConnectionIds] }
: {}),
}));
}
export async function createVirtualAutoComboFromPrepared(
prepared: PreparedVirtualAutoComboInputs,
variant: AutoVariant | undefined,
spec?: AutoComboSpec,
apiKeyId?: string,
autoChannel?: string
): Promise<VirtualAutoCombo> {
let candidatePool = clonePreparedCandidates(
spec?.family ? prepared.familyCandidates : prepared.regularCandidates
);
if (paidFilteredPool !== candidatePool) {
candidatePool.length = 0;
candidatePool.push(...paidFilteredPool);
}
// #7819 (Level 2): per-API-key candidate exclusions. Fail-open — an absent
// apiKeyId/autoChannel (every caller before #7819) or a DB lookup failure
@@ -513,9 +644,7 @@ export async function createVirtualAutoCombo(
? buildAutoCandidateFilter(spec.category, spec.tier)
: null;
if (candidateFilter) {
const narrowed = candidatePool.filter((c) =>
candidateFilter({ provider: c.provider, model: c.model })
);
const narrowed = candidatePool.filter((candidate) => candidateFilter(candidate));
const label = spec?.family
? `auto/${spec.family}`
: `auto/${spec?.category ?? ""}${spec?.tier ? `:${spec.tier}` : ""}`;
@@ -683,3 +812,13 @@ export async function createVirtualAutoCombo(
advertisedMaxOutputTokens: advertisedLimits.maxOutputTokens,
};
}
export async function createVirtualAutoCombo(
variant: AutoVariant | undefined,
spec?: AutoComboSpec,
apiKeyId?: string,
autoChannel?: string
): Promise<VirtualAutoCombo> {
const prepared = await prepareVirtualAutoComboInputs();
return createVirtualAutoComboFromPrepared(prepared, variant, spec, apiKeyId, autoChannel);
}

18
open-sse/services/contextManager.ts
View File

@@ -6,7 +6,10 @@
*/
import { REGISTRY } from "../config/providerRegistry.ts";
import { getModelContextLimit } from "../../src/lib/modelCapabilities.ts";
import {
getModelContextLimit,
type ModelCapabilityResolutionSnapshot,
} from "../../src/lib/modelCapabilities.ts";
import { parseModel } from "./model.ts";
import { jsonLength } from "../utils/jsonSize.ts";
@@ -270,8 +273,12 @@ export function estimateTokens(text: unknown): number {
* Get token limit for a provider/model combination
* Priority: Env override > models.dev DB > Registry defaultContextLength > DEFAULT_LIMITS
*/
export function getTokenLimit(provider: string, model: string | null = null): number {
return resolveTokenLimit(provider, model).limit;
export function getTokenLimit(
provider: string,
model: string | null = null,
snapshot?: ModelCapabilityResolutionSnapshot | null
): number {
return resolveTokenLimit(provider, model, snapshot).limit;
}
/**
@@ -310,7 +317,8 @@ export function getComboTargetTokenLimit(options: {
*/
function resolveTokenLimit(
provider: string,
model: string | null = null
model: string | null = null,
snapshot?: ModelCapabilityResolutionSnapshot | null
): { limit: number; specific: boolean } {
// 1. Check environment variable override first
const envOverride = getEnvOverride(provider);
@@ -320,7 +328,7 @@ function resolveTokenLimit(
// 2. Check models.dev synced DB for per-model context limit
if (model) {
const dbLimit = getModelContextLimit(provider, model);
const dbLimit = getModelContextLimit(provider, model, snapshot);
if (dbLimit && dbLimit > 0) return { limit: dbLimit, specific: true };
}

139
src/app/api/v1/models/catalog.ts
View File

@@ -35,6 +35,7 @@ import {
AUTO_SUFFIX_VARIANTS,
AUTO_FAMILY_IDS,
createBuiltinAutoCombo,
prepareBuiltinAutoComboInputs,
isPaidTierAutoId,
} from "@omniroute/open-sse/services/autoCombo/builtinCatalog";
import type { SyncedAvailableModel } from "@/lib/db/models";
@@ -52,8 +53,9 @@ import {
INTERNAL_PROXY_ERROR,
getCanonicalModelMetadata,
getCatalogDiagnosticsHeaders,
type CatalogEnrichmentSnapshot,
} from "@/lib/modelMetadataRegistry";
import { getSyncedCapability } from "@/lib/modelsDevSync";
import { getModelsDevPricing, getSyncedCapability } from "@/lib/modelsDevSync";
import { getModelSpec } from "@/shared/constants/modelSpecs";
import { getModelsCatalogPrefixMode } from "@/shared/utils/featureFlags";
import { applyCatalogPostFilters, finalizeCatalogResponse } from "./catalogResponse";
@@ -134,6 +136,12 @@ export {
} from "./catalogCache";
export type { CachedCatalog, CatalogCachePolicy } from "./catalogCache";
const BUILTIN_AUTO_YIELD_INTERVAL = 8;
function yieldCatalogBuildTurn(): Promise<void> {
return new Promise((resolve) => setImmediate(resolve));
}
/**
* Build unified OpenAI-compatible model catalog response.
* Reused by `/api/v1/models` and `/api/v1` to avoid semantic drift (T09).
@@ -608,51 +616,63 @@ async function buildUnifiedModelsResponseCore(
// #4164 entry is emitted instead, so the id is never dropped.
// #4235 Phase B: also advertise the curated `auto/<category>[:<tier>]` combos.
// #6453: also advertise the `auto/<family>` combos (auto/glm, auto/minimax, ...).
// #9418: skip the entire loop when hideAutoCombos is on — the ids are still
// routable when sent explicitly, just not advertised in the catalog.
if (!hideAuto) {
for (const autoId of [
...Object.keys(AUTO_TEMPLATE_VARIANTS),
...AUTO_SUFFIX_VARIANTS,
...AUTO_FAMILY_IDS,
]) {
if (blockedProviders.has("auto") || listedIds.has(autoId)) continue; // #5192
// #6328 (follow-up to #6495 / #6512): REMOVE — not just hide — paid-tier
// auto/* ids (auto/pro-* + auto/*:pro) from the advertised catalog when the
// operator opts into hidePaidModels. The candidate-pool filter in
// virtualFactory (#6512) still gates request-time routing for the rest.
if (hidePaid && isPaidTierAutoId(autoId)) continue;
listedIds.add(autoId);
const baseAutoEntry = {
id: autoId,
object: "model",
created: timestamp,
owned_by: "combo",
permission: [],
root: autoId,
parent: null,
};
try {
const suffix = autoId.replace(/^auto\/?/, "");
const virtualCombo = await createBuiltinAutoCombo(autoId, suffix);
const contextLength = virtualCombo.advertisedContextLength || 128000;
const maxOutputTokens = virtualCombo.advertisedMaxOutputTokens || 8192;
models.push({
...baseAutoEntry,
context_length: contextLength,
max_input_tokens: contextLength,
max_output_tokens: maxOutputTokens,
capabilities: {
tool_calling: true,
reasoning: true,
thinking: true,
temperature: true,
},
});
} catch (err) {
console.log(`[catalog] Could not materialize built-in auto model ${autoId}:`, err);
models.push(baseAutoEntry);
// #9199: prepare the shared connection/settings/registry candidate snapshot once for this
// catalog build. Runtime auto routing still prepares fresh request-scoped inputs.
let preparedAutoInputs: Awaited<ReturnType<typeof prepareBuiltinAutoComboInputs>> | undefined;
let materializedAutoCount = 0;
for (const autoId of [
...Object.keys(AUTO_TEMPLATE_VARIANTS),
...AUTO_SUFFIX_VARIANTS,
...AUTO_FAMILY_IDS,
]) {
// #9418: skip the entire loop when hideAutoCombos is on — the ids are still
// routable when sent explicitly, just not advertised in the catalog.
if (hideAuto) break;
if (blockedProviders.has("auto") || listedIds.has(autoId)) continue; // #5192
// #6328 (follow-up to #6495 / #6512): REMOVE — not just hide — paid-tier
// auto/* ids (auto/pro-* + auto/*:pro) from the advertised catalog when the
// operator opts into hidePaidModels. The candidate-pool filter in
// virtualFactory (#6512) still gates request-time routing for the rest.
if (hidePaid && isPaidTierAutoId(autoId)) continue;
listedIds.add(autoId);
const baseAutoEntry = {
id: autoId,
object: "model",
created: timestamp,
owned_by: "combo",
permission: [],
root: autoId,
parent: null,
};
try {
const suffix = autoId.replace(/^auto\/?/, "");
if (!preparedAutoInputs) {
preparedAutoInputs = await prepareBuiltinAutoComboInputs();
await yieldCatalogBuildTurn();
}
const virtualCombo = await createBuiltinAutoCombo(autoId, suffix, preparedAutoInputs);
const contextLength = virtualCombo.advertisedContextLength || 128000;
const maxOutputTokens = virtualCombo.advertisedMaxOutputTokens || 8192;
models.push({
...baseAutoEntry,
context_length: contextLength,
max_input_tokens: contextLength,
max_output_tokens: maxOutputTokens,
capabilities: {
tool_calling: true,
reasoning: true,
thinking: true,
temperature: true,
},
});
} catch (err) {
console.log(`[catalog] Could not materialize built-in auto model ${autoId}:`, err);
models.push(baseAutoEntry);
}
materializedAutoCount++;
if (materializedAutoCount % BUILTIN_AUTO_YIELD_INTERVAL === 0) {
await yieldCatalogBuildTurn();
}
}
@@ -1618,10 +1638,31 @@ async function buildUnifiedModelsResponseCore(
return modelId ? getTokenLimit(canonicalId, modelId) : getTokenLimit(canonicalId);
};
return finalizeCatalogResponse(request, finalModels, getDefaultContextFallback, {
...corsHeaders,
...diagnosticHeaders,
});
let enrichmentSnapshot: CatalogEnrichmentSnapshot | undefined;
if (finalModels.some((model) => model.owned_by !== "combo")) {
let modelsDevPricing: ReturnType<typeof getModelsDevPricing> | null = null;
try {
modelsDevPricing = getModelsDevPricing();
} catch {
// Pricing lookup is optional; hardcoded defaults still enrich the response.
}
enrichmentSnapshot = { modelsDevPricing };
// The production profile identified pricing snapshot construction as the last
// dominant synchronous stage. Let already-queued health checks run before the
// remaining in-memory enrichment and JSON serialization.
await yieldCatalogBuildTurn();
}
return finalizeCatalogResponse(
request,
finalModels,
getDefaultContextFallback,
{
...corsHeaders,
...diagnosticHeaders,
},
enrichmentSnapshot
);
} catch (error) {
console.log("Error fetching models:", error);
// Hard rule #12 — this is the realistically reachable 500 for the endpoint

393
src/app/api/v1/models/catalogCache.ts
View File

@@ -5,16 +5,15 @@
* builder walks 8 registries and hits SQLite for connections, combos, custom
* models and aliases; under Next.js's single-threaded App Router request
* handling, N concurrent calls execute back-to-back and the Nth completes at
* N × single-request latency. Identical concurrent requests are therefore
* coalesced onto one in-flight promise and successful serialized bodies are
* memoized for a short fresh window.
* N × single-request latency. So identical concurrent requests are coalesced
* onto one in-flight promise and the serialized body is memoized for a short
* window.
*
* Auth rejection is NOT handled here and must stay in the caller: it depends on
* live per-request state (dashboard cookie, API key) and must never be cached.
*/
import { getModelCatalogCacheVersion } from "@/lib/db/readCache";
import { extractApiKey } from "@/sse/services/auth";
import { after } from "next/server";
import { isCodexModelCatalogClient } from "./catalogRequest";
@@ -25,7 +24,7 @@ export type CachedCatalog = {
expiresAt: number;
};
/** Payload shape returned by the shared builder primitive the caller injects. */
/** Payload shape returned by the builder the caller injects. */
export type CatalogPayload = {
body: string;
headers: Record<string, string>;
@@ -33,65 +32,59 @@ export type CatalogPayload = {
cacheTTL: number;
};
export type CatalogRefreshTask = () => Promise<void>;
export type CatalogRefreshScheduler = (task: CatalogRefreshTask) => void;
/**
* Per-call cache policy. Request-context routes inject Next.js `after()` as the
* scheduler; unit tests and direct non-framework callers can inject a deterministic
* scheduler without making the cache branch on runner-specific environment variables.
* A client with a short discovery timeout (Claude Code allows 3 s) must never
* wait on a full rebuild. Once a cached 200 expires it is still served
* immediately for up to this long while a background refresh repopulates it.
* Bounded so a refresh that keeps failing cannot pin an old catalog forever —
* past this window callers fall back to waiting, same as a cold cache.
*/
export type CatalogCachePolicy = {
getStaleWhileRevalidateMs?: () => number;
scheduleBackgroundRefresh?: CatalogRefreshScheduler;
};
/**
* Production stale-while-revalidate window.
*
* A successful snapshot remains eligible indefinitely after the 60-second fresh TTL.
* TTL expiry requests return that last success and schedule one refresh. Database state
* changes are different: the version signal below hard-invalidates every snapshot and
* makes the next request await a current-generation build.
*/
export const CATALOG_STALE_WHILE_REVALIDATE_MS = Number.POSITIVE_INFINITY;
export const CATALOG_STALE_WHILE_REVALIDATE_MS = 30_000;
/**
* Fallback memoization window; overridden by `settings.cache.modelCatalogCacheTtlMs`.
*
* This is only the fresh window. Ordinary expiry serves the last successful snapshot
* while refreshing. `modelCatalogCacheVersion` changes bypass stale serving entirely.
* This does NOT govern post-write freshness — `invalidateDbCache()` bumps
* `modelCatalogCacheVersion` on every settings/connections/combos/pricing write and
* `dropCatalogCacheIfStateChanged()` drops the whole cache the moment it moves, so a
* write is reflected on the very next read regardless of this value. What it governs is
* the "nothing was written" case, where replaying a body built seconds ago is precisely
* the point of the cache.
*
* It was 1500 ms, which was shorter than a single build: measured 2026-07-28 on the
* production VPS, the builder takes ~49 s for a 1.3 MB / 2645-model catalog. Any two
* requests more than 1.5 s apart therefore both missed the fresh window, and the second
* fell into stale-while-revalidate — which rebuilds via `setTimeout(…, 0)` and, because
* the builder is overwhelmingly synchronous under the single-threaded App Router, pins
* the event loop so even the "served immediately" stale body only reaches the client
* once the rebuild finishes. Net effect: ~50 s on essentially every call.
*
* Held at 60 s to match the ceiling the settings schema already allows for the override
* (`settingsSchemas.ts`, `.max(60000)`), so the default can never exceed what an
* operator is permitted to configure.
*/
export const CATALOG_CACHE_TTL_MS_DEFAULT = 60_000;
type CatalogInFlight = {
generation: number;
promise: Promise<CatalogPayload>;
version: number;
promise: Promise<CachedCatalog>;
};
const catalogCache = new Map<string, CachedCatalog>();
const catalogInFlight = new Map<string, CatalogInFlight>();
let catalogGeneration = 0;
let lastSeenCatalogCacheVersion = getModelCatalogCacheVersion();
let staleWhileRevalidateMsAccessor = () => CATALOG_STALE_WHILE_REVALIDATE_MS;
/**
* An in-flight build is bound to the catalog-state generation it started from
* (`getModelCatalogCacheVersion()` at launch). After a write invalidates the
* catalog, the generation moves on: a stale in-flight build must neither be
* joined by new requests nor repopulate the now-current cache when it finishes.
* It still resolves to its own original caller (that request legitimately waits
* on it), just without being persisted.
*/
type InFlightBuild = { generation: number; promise: Promise<CachedCatalog> };
const catalogInFlight = new Map<string, InFlightBuild>();
let _catalogBuilderRuns = 0;
function defaultBackgroundRefreshScheduler(task: CatalogRefreshTask): void {
// All production routes run in Next.js request context, including callers that transform the
// shared response. Direct test/startup callers have no request store and need a safe fallback.
try {
after(task);
} catch {
setImmediate(() => void task());
}
}
/** Current SWR policy value; production defaults to unbounded stale serving. */
export function getCatalogStaleWhileRevalidateMs(): number {
return staleWhileRevalidateMsAccessor();
}
function buildCatalogCacheKey(
request: Request,
catalogSettings?: { hideAutoCombos?: boolean; hideNoThinkVariants?: boolean }
@@ -106,28 +99,32 @@ function buildCatalogCacheKey(
return `${prefix}|${isCodex}|${apiKey}|${configuredOnly}|${hideAuto}|${hideNoThink}`;
}
/**
* Observe the DB-side invalidation signal.
*
* Every observed version transition is hard invalidation: snapshots are cleared,
* the local generation advances, and old work is detached. Completion guards also
* call this function, so a version change that occurs while a builder is running
* prevents that builder from writing even before another request arrives.
*/
function synchronizeCatalogGeneration(): void {
// Tracks the model-catalog cache version (src/lib/db/readCache.ts) as of the last
// cache access. invalidateDbCache() bumps that version on every settings/connections/
// combos/pricing write; when it moves on, every memoized entry here was built from
// state that no longer holds, so drop them all rather than keying by version (which
// would leak one Map entry per version forever instead of ever pruning old ones).
let lastSeenCatalogCacheVersion = getModelCatalogCacheVersion();
function dropCatalogCacheIfStateChanged(): void {
const currentVersion = getModelCatalogCacheVersion();
if (currentVersion === lastSeenCatalogCacheVersion) return;
lastSeenCatalogCacheVersion = currentVersion;
catalogGeneration++;
catalogCache.clear();
catalogInFlight.clear();
// Deliberately NOT clearing catalogInFlight: an in-flight build bound to the
// previous generation is left to finish for its original caller, but the
// generation check in the join path (below) keeps new requests from joining
// it, and the generation check in storePayload keeps it from repopulating
// the now-current cache. Clearing it here would just detach the entry while
// the build still ran — wasted work with no correctness gain.
}
// Header sources mix Title-Case keys (diagnostic/cors headers built by app code) with
// lower-case ones (payload headers captured via the Fetch `Headers` iterator). Merge
// through a real Headers so the caller's per-request diagnostics overwrite cached values
// case-insensitively.
// lower-case ones (payload headers captured via the Fetch `Headers` iterator). A plain
// object spread keeps both casings as distinct keys, and the `Response` constructor
// then *appends* rather than overwrites them, producing comma-joined duplicates (e.g.
// request-id echoing "foo, foo"). Merge through a real `Headers` so `.set()` overwrites
// case-insensitively. Earlier sources are the base; the caller passes diagnostics last
// so per-request fields reflect the current request, not whichever one filled the cache.
export function mergeCatalogHeaders(
...sources: Array<Record<string, string> | undefined>
): Headers {
@@ -141,26 +138,82 @@ export function mergeCatalogHeaders(
return merged;
}
function isSuccessfulPayload(payload: CatalogPayload): boolean {
return payload.status >= 200 && payload.status < 300;
}
function storeSuccessfulPayload(
/**
* Persist a freshly built payload — but only when the build still belongs to the
* current catalog-state generation. A build that started before a write
* invalidation (its `buildGeneration` is older than `getModelCatalogCacheVersion()`)
* returns its entry to its original caller but must NOT repopulate the cache: the
* payload reflects pre-write state and caching it would serve stale data.
*/
function storePayload(
cacheKey: string,
payload: CatalogPayload,
inFlight: CatalogInFlight
): void {
synchronizeCatalogGeneration();
if (!isSuccessfulPayload(payload)) return;
if (inFlight.generation !== catalogGeneration) return;
if (catalogInFlight.get(cacheKey) !== inFlight) return;
catalogCache.set(cacheKey, {
buildGeneration: number
): CachedCatalog {
const entry: CachedCatalog = {
body: payload.body,
headers: payload.headers,
status: payload.status,
expiresAt: Date.now() + payload.cacheTTL,
};
if (buildGeneration === getModelCatalogCacheVersion()) {
catalogCache.set(cacheKey, entry);
}
return entry;
}
/**
* Kick off a background rebuild so an expired-but-stale-eligible entry can be
* refreshed without the current request waiting on it. Reuses catalogInFlight —
* no second coalescing mechanism — so a concurrent cold/stale request for the
* same key joins this refresh instead of starting another.
*
* The builder runs one macrotask later so the stale response that triggered this
* call is handed back before the builder's synchronous prologue runs; the whole
* point of this path is that the caller does not pay for the rebuild.
*
* The tracked promise **rejects** on failure. catalogInFlight is shared with the
* cold path: a caller whose entry aged past the stale window skips the stale
* branch and awaits whatever promise it finds here, and resolving with the stale
* entry would hand it a body it was no longer entitled to while disguising a
* build failure as a 200. The rejection is pre-handled so this path can never
* raise an unhandledRejection; a failed refresh simply never overwrites the entry.
*/
function scheduleBackgroundRefresh(
cacheKey: string,
request: Request,
buildPayload: (request: Request) => Promise<CatalogPayload>
): void {
if (catalogInFlight.has(cacheKey)) return; // a refresh for this key is already running
const generation = getModelCatalogCacheVersion();
const refreshPromise: Promise<CachedCatalog> = new Promise((resolve, reject) => {
setTimeout(() => {
runBuilder(buildPayload, request)
.then((payload) => resolve(storePayload(cacheKey, payload, generation)))
.catch((err) => {
console.error(
`[catalog] Background stale-while-revalidate refresh failed for key "${cacheKey}":`,
err
);
reject(err);
});
}, 0);
});
inFlight = { version: lastSeenCatalogCacheVersion, promise };
// Nobody on the stale path awaits this, so pre-handle the rejection; a cold-path
// caller that joins it via catalogInFlight attaches its own handler and still
// observes the failure.
promise.catch(() => {});
catalogInFlight.set(cacheKey, { generation, promise: refreshPromise });
refreshPromise
.catch(() => {})
.finally(() => {
if (catalogInFlight.get(cacheKey)?.promise === refreshPromise)
catalogInFlight.delete(cacheKey);
});
}
function runBuilder(
@@ -168,105 +221,24 @@ function runBuilder(
request: Request
): Promise<CatalogPayload> {
_catalogBuilderRuns++;
try {
return Promise.resolve(buildPayload(request));
} catch (error) {
return Promise.reject(error);
}
}
function cleanInFlight(cacheKey: string, inFlight: CatalogInFlight): void {
if (catalogInFlight.get(cacheKey) === inFlight) {
catalogInFlight.delete(cacheKey);
}
}
function startSynchronousBuild(
cacheKey: string,
request: Request,
buildPayload: (request: Request) => Promise<CatalogPayload>
): CatalogInFlight {
const generation = catalogGeneration;
let inFlight!: CatalogInFlight;
const promise = runBuilder(buildPayload, request).then((payload) => {
storeSuccessfulPayload(cacheKey, payload, inFlight);
return payload;
});
inFlight = { generation, promise };
catalogInFlight.set(cacheKey, inFlight);
inFlight.promise.then(
() => cleanInFlight(cacheKey, inFlight),
() => cleanInFlight(cacheKey, inFlight)
);
return inFlight;
}
function scheduleBackgroundRefresh(
cacheKey: string,
request: Request,
buildPayload: (request: Request) => Promise<CatalogPayload>,
schedule: CatalogRefreshScheduler
): void {
if (catalogInFlight.has(cacheKey)) return;
let resolveRefresh!: (payload: CatalogPayload) => void;
let rejectRefresh!: (error: unknown) => void;
const inFlight: CatalogInFlight = {
generation: catalogGeneration,
promise: new Promise<CatalogPayload>((resolve, reject) => {
resolveRefresh = resolve;
rejectRefresh = reject;
}),
};
// Reserve the key before handing the task to the scheduler. Multiple stale reads in
// the same request turn therefore cannot enqueue duplicate refreshes.
catalogInFlight.set(cacheKey, inFlight);
void inFlight.promise.catch(() => {}); // background failures are always handled
const task: CatalogRefreshTask = async () => {
synchronizeCatalogGeneration();
if (inFlight.generation !== catalogGeneration || catalogInFlight.get(cacheKey) !== inFlight) {
// Hard invalidation or a deterministic test reset detached this scheduled task
// before it started. Resolve its private bookkeeping promise without rebuilding.
resolveRefresh({ body: "", headers: {}, status: 204, cacheTTL: 0 });
return;
}
try {
const payload = await runBuilder(buildPayload, request);
storeSuccessfulPayload(cacheKey, payload, inFlight);
resolveRefresh(payload);
} catch (error) {
console.error("[catalog] Background stale-while-revalidate refresh failed:", error);
rejectRefresh(error);
} finally {
cleanInFlight(cacheKey, inFlight);
}
};
try {
schedule(task);
} catch (error) {
cleanInFlight(cacheKey, inFlight);
rejectRefresh(error);
console.error("[catalog] Failed to schedule background refresh:", error);
}
return buildPayload(request);
}
/**
* Resolve the cached catalog response for `request`, building it through the shared
* `buildPayload` primitive when there is no current snapshot.
* Resolve the cached catalog response for `request`, building it through
* `buildPayload` when there is nothing fresh to serve.
*
* Returns `null` when the caller must build and handle errors itself — i.e. the
* in-flight build rejected — so the error-response shape stays in the caller.
*/
export async function resolveCachedCatalogResponse(
request: Request,
headerSources: { corsHeaders: Record<string, string>; diagnosticHeaders: Record<string, string> },
buildPayload: (request: Request) => Promise<CatalogPayload>,
policy: CatalogCachePolicy = {},
catalogSettings?: { hideAutoCombos?: boolean; hideNoThinkVariants?: boolean }
): Promise<Response> {
const { corsHeaders, diagnosticHeaders } = headerSources;
synchronizeCatalogGeneration();
dropCatalogCacheIfStateChanged();
const cacheKey = buildCatalogCacheKey(request, catalogSettings);
const now = Date.now();
@@ -279,32 +251,41 @@ export async function resolveCachedCatalogResponse(
});
}
const staleWhileRevalidateMs =
policy.getStaleWhileRevalidateMs?.() ?? getCatalogStaleWhileRevalidateMs();
// Stale-while-revalidate: an expired entry is still served immediately as long as
// (a) it was a successful build — a cached error replayed as "stale" would mask an
// intermittent failure behind a fake success forever — and (b) it is within the
// staleness window, so a refresh that keeps failing eventually falls through to the
// cold-path wait instead of pinning ancient data.
if (
cached &&
cached.status >= 200 &&
cached.status < 300 &&
now - cached.expiresAt <= staleWhileRevalidateMs
cached.status === 200 &&
now - cached.expiresAt <= CATALOG_STALE_WHILE_REVALIDATE_MS
) {
scheduleBackgroundRefresh(
cacheKey,
request,
buildPayload,
policy.scheduleBackgroundRefresh ?? defaultBackgroundRefreshScheduler
);
scheduleBackgroundRefresh(cacheKey, request, buildPayload);
return new Response(cached.body, {
status: cached.status,
headers: mergeCatalogHeaders(corsHeaders, cached.headers, diagnosticHeaders),
});
}
let inFlight = catalogInFlight.get(cacheKey);
if (!inFlight) {
inFlight = startSynchronousBuild(cacheKey, request, buildPayload);
const currentGeneration = getModelCatalogCacheVersion();
let inflight = catalogInFlight.get(cacheKey);
// Only join an in-flight build from the CURRENT generation. A build bound to an
// older (pre-write) generation reflects stale state, so a new request starts a
// fresh build instead of joining it.
if (!inflight || inflight.generation !== currentGeneration) {
const generation = currentGeneration;
const promise = runBuilder(buildPayload, request).then((payload) =>
storePayload(cacheKey, payload, generation)
);
inflight = { generation, promise };
catalogInFlight.set(cacheKey, inflight);
promise.finally(() => {
if (catalogInFlight.get(cacheKey)?.promise === promise) catalogInFlight.delete(cacheKey);
});
}
const payload = await inFlight.promise;
const payload = await inflight.promise;
return new Response(payload.body, {
status: payload.status,
headers: mergeCatalogHeaders(corsHeaders, payload.headers, diagnosticHeaders),
@@ -312,26 +293,14 @@ export async function resolveCachedCatalogResponse(
}
// ── Test hooks ───────────────────────────────────────────────────────────────
// Not part of the public application API.
// Not part of the public API; do not read from app code.
/** Deterministically resets counters, policy, snapshots, generations, and old work. */
/** Resets the builder counter and every cached/in-flight entry. */
export function __resetCatalogBuilderRunsForTest(): void {
_catalogBuilderRuns = 0;
catalogGeneration++;
catalogCache.clear();
catalogInFlight.clear();
lastSeenCatalogCacheVersion = getModelCatalogCacheVersion();
staleWhileRevalidateMsAccessor = () => CATALOG_STALE_WHILE_REVALIDATE_MS;
}
/** Injects the SWR policy accessor without environment-dependent behavior. */
export function __setCatalogStaleWhileRevalidateAccessorForTest(accessor: () => number): void {
staleWhileRevalidateMsAccessor = accessor;
}
/** Backward-compatible scalar policy hook retained for focused tests. */
export function __setCatalogStaleWhileRevalidateMsForTest(ms: number): void {
staleWhileRevalidateMsAccessor = () => ms;
}
/** Counts full builder executions — proves concurrent requests share one run (#6408). */
@@ -339,7 +308,11 @@ export function __getCatalogBuilderRunsForTest(): number {
return _catalogBuilderRuns;
}
/** Marks every successful snapshot expired without sleeping out the real TTL. */
/**
* Marks every cached entry as expired `msAgo` milliseconds ago instead of sleeping
* out the real TTL. Pass more than CATALOG_STALE_WHILE_REVALIDATE_MS to simulate an
* entry that has aged past the stale-serving window.
*/
export function __expireCatalogCacheForTest(msAgo = 1): void {
const expiresAt = Date.now() - msAgo;
for (const [key, entry] of catalogCache.entries()) {
@@ -347,22 +320,38 @@ export function __expireCatalogCacheForTest(msAgo = 1): void {
}
}
/** Seeds a request-keyed snapshot for status/staleness compatibility tests. */
/**
* Seeds the entry a given request would read, for status/staleness combinations the
* intentionally exception-resistant builder cannot be made to produce (e.g. a cached
* non-200). Takes the Request so the cache-key format stays private to this module.
*/
export function __setCatalogCacheEntryForTest(request: Request, entry: CachedCatalog): void {
catalogCache.set(buildCatalogCacheKey(request), entry);
}
/** Awaits any currently running or scheduled refresh without real-time sleeps. */
/** Awaits any background refresh in flight, instead of guessing at a real-time sleep. */
export async function __flushCatalogBackgroundRefreshForTest(): Promise<void> {
await Promise.all([...catalogInFlight.values()].map(({ promise }) => promise.catch(() => {})));
await Promise.all([...catalogInFlight.values()].map((entry) => entry.promise.catch(() => {})));
}
/** Injects a handled in-flight rejection for the catalog error-shape regression test. */
/**
* Injects a synthetic in-flight rejection so the caller's catch branch (sanitized
* error body) can be exercised deterministically — the builder core try/catches every
* registry and DB read individually, so it is not a practical error-injection point.
*
* Deliberately does not self-clean the way production entries do: this promise is
* already rejected at creation, so a cleanup callback would delete the map entry
* within a microtask or two — before the caller's several-await auth check finishes —
* silently swapping in a fresh cold build instead of the intended failure. The next
* __resetCatalogBuilderRunsForTest() clears it.
*/
export function __forceCatalogInFlightRejectionForTest(request: Request, error: unknown): void {
const promise: Promise<CatalogPayload> = Promise.reject(error);
void promise.catch(() => {});
const rejected: Promise<CachedCatalog> = Promise.reject(error);
rejected.catch(() => {}); // mark as handled — avoids an unhandledRejection warning
// Bind to the current generation so the cold path still joins it (a stale
// generation would be skipped as pre-write state and never awaited).
catalogInFlight.set(buildCatalogCacheKey(request), {
generation: catalogGeneration,
promise,
generation: getModelCatalogCacheVersion(),
promise: rejected,
});
}

6
src/app/api/v1/models/catalogResponse.ts
View File

@@ -30,6 +30,7 @@ import { sortCatalogModelsProviderGrouped } from "./catalogOrder";
import {
disambiguateCatalogModelNames,
enrichCatalogModelEntry,
type CatalogEnrichmentSnapshot,
} from "@/lib/modelMetadataRegistry";
import { isModelCatalogNamesEnabled } from "@/shared/utils/featureFlags";
import { extractApiKey } from "@/sse/services/auth";
@@ -189,7 +190,8 @@ export async function finalizeCatalogResponse(
request: Request,
finalModels: Array<Record<string, unknown>>,
getContextFallback: (model: Record<string, unknown>) => number | undefined,
headers: Record<string, string>
headers: Record<string, string>,
enrichmentSnapshot?: CatalogEnrichmentSnapshot
): Promise<Response> {
const apiKey = extractApiKey(request);
if (apiKey) {
@@ -210,7 +212,7 @@ export async function finalizeCatalogResponse(
if (model.owned_by === "combo") {
return maybeOmitCatalogModelName(model, includeModelNames);
}
const enriched = enrichCatalogModelEntry(model);
const enriched = enrichCatalogModelEntry(model, undefined, enrichmentSnapshot);
const fallbackContextLength = getContextFallback(enriched);
const listedModel = fallbackContextLength
? { ...enriched, context_length: fallbackContextLength }

11
src/lib/db/modelCapabilityOverrides.ts
View File

@@ -63,14 +63,23 @@ function toOverride(row: OverrideRow): ModelCapabilityOverride | null {
};
}
/** Nested provider → model → max_token map used by build-local snapshots. */
export type NestedMaxTokenOverrideMap = ReadonlyMap<string, ReadonlyMap<string, number>>;
export function getModelCapabilityOverride(
provider: string | null | undefined,
modelId: string | null | undefined,
key: ModelCapabilityOverrideKey
key: ModelCapabilityOverrideKey,
bulkMaxTokenOverrides?: NestedMaxTokenOverrideMap | null
): number | null {
const target = parseModelOverrideTarget(`${provider || ""}/${modelId || ""}`);
if (!target || !isSupportedKey(key)) return null;
if (bulkMaxTokenOverrides) {
if (key !== "max_token") return null;
return bulkMaxTokenOverrides.get(target.provider)?.get(target.modelId) ?? null;
}
try {
const row = getDbInstance()
.prepare(

11
src/lib/db/modelContextOverrides.ts
View File

@@ -78,11 +78,20 @@ export function getModelContextOverrideRecord(
}
}
/** Nested provider → model → context map used by build-local snapshots. */
export type NestedContextOverrideMap = ReadonlyMap<string, ReadonlyMap<string, number>>;
/** The overridden context window (tokens) for (provider, modelId), or null. Never throws. */
export function getModelContextOverride(
provider: string | null | undefined,
modelId: string | null | undefined
modelId: string | null | undefined,
bulkContextOverrides?: NestedContextOverrideMap | null
): number | null {
if (bulkContextOverrides) {
const key = normalizeKey(provider, modelId);
if (!key) return null;
return bulkContextOverrides.get(key.provider)?.get(key.modelId) ?? null;
}
const record = getModelContextOverrideRecord(provider, modelId);
return record ? record.realContext : null;
}

2
src/lib/db/readCache.ts
View File

@@ -267,7 +267,7 @@ export function invalidateModelCatalogCache(): void {
/**
* Invalidate caches (call after writes to any of: settings, pricing,
* connections, combos, nodes).
* connections, combos, nodes, model capability/context metadata).
*
* When scope is `"connections"` and an `id` is provided, only that
* connection's by-ID cache entry is invalidated (the filter-keyed raw

75
src/lib/modelCapabilities.ts
View File

@@ -16,6 +16,10 @@ import { getModelContextOverride } from "@/lib/db/modelContextOverrides";
import { getModelCapabilityOverride } from "@/lib/db/modelCapabilityOverrides";
import { getDbInstance } from "@/lib/db/core";
import { getKeyValue } from "@/lib/db/models/shared";
import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
export type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
export { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
import { isVisionModelId } from "@/shared/constants/visionModels";
import { getUnsupportedParams } from "@omniroute/open-sse/config/providerRegistry.ts";
import {
@@ -371,7 +375,8 @@ function reverseModelsDevProviders(provider: string): readonly string[] {
function getSyncedCapabilityForResolved(
provider: string | null,
model: string | null,
rawModel: string | null
rawModel: string | null,
snapshot?: ModelCapabilityResolutionSnapshot | null
): SyncedCapabilities {
if (!provider || !model) return null;
@@ -401,9 +406,10 @@ function getSyncedCapabilityForResolved(
new Set([provider, ...reverseModelsDevProviders(provider), "vercel"])
);
const bulk = snapshot?.synced ?? null;
for (const prov of providerCandidates) {
for (const mid of modelCandidates) {
const found = getSyncedCapability(prov, mid);
const found = getSyncedCapability(prov, mid, bulk);
if (found) return found;
}
}
@@ -627,15 +633,43 @@ function getOutputTokenCapabilityOverride(resolved: {
return getCapabilityOverride(resolved, "max_output_tokens");
}
export function getExplicitModelOutputCap(input: CapabilityInput): number | null {
/**
* Bulk-load friendly max_token override lookup (#9199). When a snapshot is
* supplied its preloaded map is used; otherwise falls back to the on-demand
* read (same precedence as getOutputTokenCapabilityOverride).
*/
function getMaxTokenCapabilityOverride(
resolved: {
provider: string | null;
model: string | null;
rawModel: string | null;
},
snapshot?: ModelCapabilityResolutionSnapshot | null
): number | null {
const bulk = snapshot?.maxTokenOverrides ?? null;
return (
getModelCapabilityOverride(resolved.provider, resolved.model, "max_token", bulk) ??
(resolved.rawModel && resolved.rawModel !== resolved.model
? getModelCapabilityOverride(resolved.provider, resolved.rawModel, "max_token", bulk)
: null)
);
}
export function getExplicitModelOutputCap(
input: CapabilityInput,
snapshot?: ModelCapabilityResolutionSnapshot | null
): number | null {
const resolved = resolveCapabilityInput(input);
const maxTokenOverride = getOutputTokenCapabilityOverride(resolved);
const maxTokenOverride = snapshot
? getMaxTokenCapabilityOverride(resolved, snapshot)
: getOutputTokenCapabilityOverride(resolved);
if (maxTokenOverride !== null) return maxTokenOverride;
const synced = getSyncedCapabilityForResolved(
resolved.provider,
resolved.model,
resolved.rawModel
resolved.rawModel,
snapshot
);
if (synced && typeof synced.limit_output === "number") return synced.limit_output;
@@ -648,7 +682,8 @@ export function getExplicitModelOutputCap(input: CapabilityInput): number | null
export function getResolvedModelCapabilities(
input: CapabilityInput,
options?: ResolveModelCapabilitiesOptions
options?: ResolveModelCapabilitiesOptions,
snapshot?: ModelCapabilityResolutionSnapshot | null
): ResolvedModelCapabilities {
// Reconciliation / auto-discovery needs the override-free catalog view so a
// persisted override never feeds back into the comparison that (re)writes it.
@@ -659,7 +694,8 @@ export function getResolvedModelCapabilities(
const synced = getSyncedCapabilityForResolved(
resolved.provider,
resolved.model,
resolved.rawModel
resolved.rawModel,
snapshot
);
const modalitiesInput = parseModalities(synced?.modalities_input);
@@ -717,9 +753,11 @@ export function getResolvedModelCapabilities(
null;
const maxInputOverride = usePersistedOverrides ? getInputTokenCapabilityOverride(resolved) : null;
const maxTokenOverride = usePersistedOverrides
? getOutputTokenCapabilityOverride(resolved)
: null;
const maxTokenOverride = snapshot
? getMaxTokenCapabilityOverride(resolved, snapshot)
: usePersistedOverrides
? getOutputTokenCapabilityOverride(resolved)
: null;
// Vision consults leaf static metadata for path-shaped ids; other capability
// fields keep using the non-leaf `spec` from getStaticSpec() above.
@@ -919,11 +957,20 @@ export function capThinkingBudget(input: CapabilityInput, budget: number): numbe
export function getModelContextLimit(
providerOrInput: CapabilityInput,
modelId?: string
modelId?: string,
snapshot?: ModelCapabilityResolutionSnapshot | null
): number | null {
const resolved =
typeof providerOrInput === "string" && modelId !== undefined
? getResolvedModelCapabilities({ provider: providerOrInput, model: modelId })
: getResolvedModelCapabilities(providerOrInput);
return resolved.contextWindow;
? getResolvedModelCapabilities({ provider: providerOrInput, model: modelId }, undefined, snapshot)
: getResolvedModelCapabilities(providerOrInput, undefined, snapshot);
// Feature 5004: a persisted override (operator-set or auto-discovered) wins over the
// static catalog / models.dev sync. `getResolvedModelCapabilities` stays override-free
// so the reconciler can compare the catalog value against provider-declared windows.
const override = getModelContextOverride(
resolved.provider,
resolved.model,
snapshot?.contextOverrides ?? null
);
return override ?? resolved.contextWindow;
}

65
src/lib/modelCapabilityResolutionSnapshot.ts Normal file
View File

@@ -0,0 +1,65 @@
/**
* Build-local capability/context/override resolution snapshot (#9199).
*
* Catalog preparation bulk-loads the three capability tables once into a
* build-local view for pure in-memory resolution. This must not flip models.dev's
* module-global all-row cache, and ordinary runtime callers keep on-demand DB reads.
*
* Override maps are nested by provider then model so provider/model pairs cannot
* collide via delimiter composition.
*/
import { listModelCapabilityOverrides } from "@/lib/db/modelCapabilityOverrides";
import { listModelContextOverrides } from "@/lib/db/modelContextOverrides";
import {
loadAllSyncedCapabilitiesUncached,
type CapabilitiesByProvider,
} from "@/lib/modelsDevSync";
/** Nested provider → model → numeric override map (collision-free). */
export type NestedOverrideMap = ReadonlyMap<string, ReadonlyMap<string, number>>;
export interface ModelCapabilityResolutionSnapshot {
readonly synced: CapabilitiesByProvider;
readonly maxTokenOverrides: NestedOverrideMap;
readonly contextOverrides: NestedOverrideMap;
}
function setNestedOverride(
map: Map<string, Map<string, number>>,
provider: string,
modelId: string,
value: number
): void {
let byModel = map.get(provider);
if (!byModel) {
byModel = new Map();
map.set(provider, byModel);
}
byModel.set(modelId, value);
}
/**
* Load all three capability tables in one uninterrupted JS turn.
* Callers must not yield between the bulk reads if they need a coherent view;
* existing catalog generation guards remain authoritative across later yields.
*/
export function createModelCapabilityResolutionSnapshot(): ModelCapabilityResolutionSnapshot {
const synced = loadAllSyncedCapabilitiesUncached();
const maxTokenOverrides = new Map<string, Map<string, number>>();
for (const entry of listModelCapabilityOverrides()) {
if (entry.key !== "max_token") continue;
setNestedOverride(maxTokenOverrides, entry.provider, entry.modelId, entry.value);
}
const contextOverrides = new Map<string, Map<string, number>>();
for (const entry of listModelContextOverrides()) {
setNestedOverride(contextOverrides, entry.provider, entry.modelId, entry.realContext);
}
return {
synced,
maxTokenOverrides,
contextOverrides,
};
}

26
src/lib/modelMetadataRegistry.ts
View File

@@ -15,7 +15,12 @@ import {
} from "@/shared/constants/modelSpecs";
import { AI_PROVIDERS } from "@/shared/constants/providers";
import { PROVIDER_ID_TO_ALIAS, PROVIDER_MODELS } from "@/shared/constants/models";
import { getSyncStatus, getSyncedCapability, getModelsDevPricing } from "@/lib/modelsDevSync";
import {
getSyncStatus,
getSyncedCapability,
getModelsDevPricing,
type PricingByProvider,
} from "@/lib/modelsDevSync";
import { getSyncedPricing } from "@/lib/pricingSync";
import { getPricingForModel as getDefaultPricingForModel } from "@/shared/constants/pricing";
import {
@@ -31,6 +36,10 @@ export const INTERNAL_PROXY_ERROR = "INTERNAL_PROXY_ERROR";
type JsonRecord = Record<string, unknown>;
export interface CatalogEnrichmentSnapshot {
modelsDevPricing: PricingByProvider | null;
}
interface CatalogDiagnosticsOptions {
request?: Request | null;
requestId?: string | null;
@@ -300,16 +309,16 @@ function findInsensitive<T>(obj: Record<string, T> | null | undefined, key: stri
function resolveCatalogPricing(
provider: string | null,
model: string | null
model: string | null,
snapshot?: CatalogEnrichmentSnapshot
): Record<string, number> | null {
if (!provider || !model) return null;
// Prefer models.dev synced pricing when present; fall back to hardcoded defaults.
try {
const modelsDev = getModelsDevPricing() as Record<
string,
Record<string, Record<string, number>>
>;
const modelsDev = (
snapshot ? snapshot.modelsDevPricing || {} : getModelsDevPricing()
) as Record<string, Record<string, Record<string, number>>>;
const providerPricing =
findInsensitive(modelsDev, provider) ||
findInsensitive(modelsDev, provider.replace(/-cn$/, ""));
@@ -391,7 +400,8 @@ function resolveCatalogPricing(
export function enrichCatalogModelEntry<T extends JsonRecord>(
entry: T,
input?: { provider?: string | null; model?: string | null }
input?: { provider?: string | null; model?: string | null },
snapshot?: CatalogEnrichmentSnapshot
): T {
const provider =
input?.provider ||
@@ -537,7 +547,7 @@ export function enrichCatalogModelEntry<T extends JsonRecord>(
}
if (nextEntry.pricing == null) {
const pricing = resolveCatalogPricing(provider, model);
const pricing = resolveCatalogPricing(provider, model, snapshot);
if (pricing) nextEntry.pricing = pricing;
}

127
src/lib/modelsDevSync.ts
View File

@@ -305,6 +305,49 @@ export function ensureCapabilitiesTable(): void {
`);
}
function defineEnumerableDataProperty<T extends object>(
target: T,
key: string,
value: unknown
): void {
Object.defineProperty(target, key, {
value,
writable: true,
enumerable: true,
configurable: true,
});
}
function capabilitiesFromRows(rows: unknown[]): CapabilitiesByProvider {
const result: CapabilitiesByProvider = {};
for (const row of rows) {
const record = toRecord(row);
const prov = typeof record.provider === "string" ? record.provider : null;
const mid = typeof record.model_id === "string" ? record.model_id : null;
if (!prov || !mid) continue;
if (!Object.hasOwn(result, prov)) {
defineEnumerableDataProperty(result, prov, {});
}
defineEnumerableDataProperty(result[prov], mid, mapCapabilityRecord(record));
}
return result;
}
/**
* Uncached full-table models.dev capability read for build-local snapshots.
* Shares mapping with the ordinary all-row API but never mutates the module-global
* `cachedCapabilities` / `cachedCapabilitiesLoadedAll` runtime cache.
*/
export function loadAllSyncedCapabilitiesUncached(): CapabilitiesByProvider {
const db = getDbInstance();
ensureCapabilitiesTable();
const rows = db.prepare("SELECT * FROM model_capabilities").all();
return capabilitiesFromRows(rows);
}
/**
* Read synced capabilities from `model_capabilities` table.
*/
@@ -337,18 +380,7 @@ export function getSyncedCapabilities(provider?: string, modelId?: string): Capa
}
}
const rows = db.prepare(query).all(...params);
const result: CapabilitiesByProvider = {};
for (const row of rows) {
const record = toRecord(row);
const prov = typeof record.provider === "string" ? record.provider : null;
const mid = typeof record.model_id === "string" ? record.model_id : null;
if (!prov || !mid) continue;
if (!result[prov]) result[prov] = {};
result[prov][mid] = mapCapabilityRecord(record);
}
const result = capabilitiesFromRows(db.prepare(query).all(...params));
if (!provider && !modelId) {
cachedCapabilities = result;
@@ -372,35 +404,62 @@ const SYNCED_CAPABILITY_FALLBACK_ALIASES: Record<string, string[]> = {
"opencode-go": ["opencode-zen"],
};
export function getSyncedCapability(
function lookupSyncedCapabilityWithFallbacks(
provider: string,
modelId: string
modelId: string,
lookup: (provider: string) => ModelCapabilityEntry | null
): ModelCapabilityEntry | null {
if (!provider || !modelId) return null;
const direct = lookup(provider);
if (direct) return direct;
// #8697-adjacent: this used to hit SQLite with a per-model SELECT on every cold
// call, relying on some other caller (getSyncedCapabilities() with no args) to have
// already warmed the whole-table cache first — no such caller sits in the /v1/models
// catalog build path, so a cold rebuild ran one SQLite round-trip per model per call
// site instead of one bulk read for the whole rebuild. Self-warm here instead of
// depending on an external caller.
if (!cachedCapabilitiesLoadedAll) {
getSyncedCapabilities();
}
const lookupCached = (p: string) => cachedCapabilities?.[p]?.[modelId] ?? null;
const directCached = lookupCached(provider);
if (directCached) return directCached;
const fallbacks = SYNCED_CAPABILITY_FALLBACK_ALIASES[provider];
if (fallbacks) {
for (const alt of fallbacks) {
const found = lookupCached(alt);
const found = lookup(alt);
if (found) return found;
}
}
return null;
}
export function getSyncedCapability(
provider: string,
modelId: string,
bulk?: CapabilitiesByProvider | null
): ModelCapabilityEntry | null {
if (!provider || !modelId) return null;
if (bulk) {
return lookupSyncedCapabilityWithFallbacks(
provider,
modelId,
(p) => bulk[p]?.[modelId] ?? null
);
}
// Fast path: every provider is in the in-memory cache, skip SQLite entirely.
if (cachedCapabilitiesLoadedAll) {
return lookupSyncedCapabilityWithFallbacks(
provider,
modelId,
(p) => cachedCapabilities?.[p]?.[modelId] ?? null
);
}
// Cold path: hit SQLite. Prepare the statement once, reuse for every alias.
const db = getDbInstance();
ensureCapabilitiesTable();
const stmt = db.prepare(
"SELECT * FROM model_capabilities WHERE provider = ? AND model_id = ? LIMIT 1"
);
return lookupSyncedCapabilityWithFallbacks(provider, modelId, (p) => {
const row = stmt.get(p, modelId);
if (!row) return null;
return mapCapabilityRecord(toRecord(row));
});
}
/**
* Save synced capabilities to `model_capabilities` table (full replace).
*/
@@ -419,11 +478,12 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
`);
const now = new Date().toISOString();
let changed = false;
const tx = db.transaction(() => {
del.run();
if (del.run().changes > 0) changed = true;
for (const [provider, models] of Object.entries(data)) {
for (const [modelId, cap] of Object.entries(models)) {
insert.run(
const info = insert.run(
provider,
modelId,
cap.tool_call === null ? null : cap.tool_call ? 1 : 0,
@@ -445,6 +505,7 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
cap.interleaved_field,
now
);
if (info.changes > 0) changed = true;
}
}
});
@@ -452,6 +513,7 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
backupDbFile("pre-write");
cachedCapabilities = data;
cachedCapabilitiesLoadedAll = true;
if (changed) invalidateDbCache("model-capabilities");
}
/**
@@ -460,10 +522,11 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
export function clearModelsDevCapabilities(): void {
const db = getDbInstance();
ensureCapabilitiesTable();
db.prepare("DELETE FROM model_capabilities").run();
const info = db.prepare("DELETE FROM model_capabilities").run();
backupDbFile("pre-write");
cachedCapabilities = {};
cachedCapabilitiesLoadedAll = true;
if (info.changes > 0) invalidateDbCache("model-capabilities");
}
// ─── Main sync function ──────────────────────────────────

2
src/sse/services/auth.ts
View File

@@ -86,6 +86,7 @@ import {
resolveForcedConnectionForCredentialPool,
resolveSessionAffinityTtlMs,
selectSessionAffinityConnection,
syncSessionAffinityRuntimeFields,
} from "./sessionAffinityPin";
import {
isAnonymousFallbackDisabledBySettings,
@@ -1537,6 +1538,7 @@ export async function getProviderCredentials(
);
if (affinityConnection) {
connection = affinityConnection;
syncSessionAffinityRuntimeFields(connectionsRaw, connection);
} else if (options.sessionKey) {
log.info(
"AUTH",

27
src/sse/services/sessionAffinityPin.ts
View File

@@ -32,7 +32,7 @@ import {
touchSessionAccountAffinity,
deleteSessionAccountAffinity,
} from "@/lib/db/sessionAccountAffinity";
import { updateProviderConnection } from "@/lib/db/providers";
import { touchConnectionLastUsed } from "@/lib/db/providers";
import { isModelExcludedByConnection } from "@/domain/connectionModelRules";
import { isAccountQuotaExhausted } from "@/domain/quotaCache";
import {
@@ -57,6 +57,16 @@ export interface SessionAffinityConnection {
priority?: number | null;
}
export function syncSessionAffinityRuntimeFields(
connections: SessionAffinityConnection[],
selected: SessionAffinityConnection
): void {
const cached = connections.find((connection) => connection.id === selected.id);
if (!cached) return;
cached.lastUsedAt = selected.lastUsedAt;
cached.consecutiveUseCount = selected.consecutiveUseCount;
}
export function formatSessionKeyForLog(sessionKey: string): string {
return `${sessionKey.slice(0, 18)}...`;
}
@@ -92,10 +102,10 @@ export async function selectSessionAffinityConnection<T extends SessionAffinityC
const connection = connections.find((candidate) => candidate.id === existing.connectionId);
if (connection) {
touchSessionAccountAffinity(sessionKey, provider, Date.now(), ttlMs);
await updateProviderConnection(connection.id, {
lastUsedAt: new Date().toISOString(),
consecutiveUseCount: (connection.consecutiveUseCount || 0) + 1,
});
const nextCount = (connection.consecutiveUseCount || 0) + 1;
await touchConnectionLastUsed(connection.id, nextCount);
connection.lastUsedAt = new Date().toISOString();
connection.consecutiveUseCount = nextCount;
log.info(
"AUTH",
`session_key=${formatSessionKeyForLog(sessionKey)} -> connection ${connection.id.slice(
@@ -117,10 +127,9 @@ export async function selectSessionAffinityConnection<T extends SessionAffinityC
if (!connection) return null;
upsertSessionAccountAffinity(sessionKey, provider, connection.id, Date.now(), ttlMs);
await updateProviderConnection(connection.id, {
lastUsedAt: new Date().toISOString(),
consecutiveUseCount: 1,
});
await touchConnectionLastUsed(connection.id, 1);
connection.lastUsedAt = new Date().toISOString();
connection.consecutiveUseCount = 1;
log.info(
"AUTH",
`new affinity created for session_key=${formatSessionKeyForLog(

1
stryker.conf.json
View File

@@ -244,6 +244,7 @@
"tests/unit/microsoft-designer-web-6672.test.ts",
"tests/unit/middleware-header-strip-5849.test.ts",
"tests/unit/middleware-hooks-error-sanitization.test.ts",
"tests/unit/model-catalog-runtime-invalidation.test.ts",
"tests/unit/model-cooldowns-route-auth.test.ts",
"tests/unit/model-cooldowns-route.test.ts",
"tests/unit/model-lockout-decay.test.ts",

179
tests/integration/model-catalog-responsiveness-9199.test.ts Normal file
View File

@@ -0,0 +1,179 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import test from "node:test";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-catalog-9199-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-9199-test-secret";
const core = await import("../../src/lib/db/core.ts");
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");
const modelsCatalog = await import("../../src/app/api/v1/models/catalog.ts");
const modelsRoute = await import("../../src/app/api/v1/models/route.ts");
const healthRoute = await import("../../src/app/api/health/ping/route.ts");
async function resetStorage() {
core.resetDbInstance();
apiKeysDb.resetApiKeyState();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
modelsCatalog.__resetCatalogBuilderRunsForTest();
}
test.beforeEach(async () => {
await resetStorage();
});
test.after(async () => {
await resetStorage();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
});
test(
"#9199 authenticated cold model catalog yields to an unrelated health request before publication",
{ timeout: 30_000 },
async () => {
await settingsDb.updateSettings({
requireLogin: true,
requireAuthForModels: true,
password: "",
});
const apiKey = await apiKeysDb.createApiKey("catalog responsiveness", "machine-9199");
let catalogSettled = false;
const catalogPromise = modelsRoute
.GET(
new Request("http://localhost/v1/models?prefix=alias", {
headers: { Authorization: `Bearer ${apiKey.key}` },
})
)
.then((response) => {
catalogSettled = true;
return response;
});
const heartbeat = await new Promise<Response>((resolve, reject) => {
setImmediate(() => {
healthRoute.GET().then(resolve, reject);
});
});
assert.equal(heartbeat.status, 200);
assert.equal(
catalogSettled,
false,
"the catalog monopolized the event loop until publication; unrelated requests could not run"
);
const catalog = await catalogPromise;
assert.equal(catalog.status, 200);
const body = (await catalog.json()) as { data?: Array<{ id?: string }> };
assert.ok(Array.isArray(body.data));
assert.ok(body.data.some((model) => model.id === "auto/best-coding"));
assert.equal(modelsCatalog.__getCatalogBuilderRunsForTest(), 1);
}
);
test(
"#9199 cold catalog snapshots models.dev pricing once and yields before enrichment",
{ timeout: 30_000 },
async () => {
await settingsDb.updateSettings({
requireLogin: true,
requireAuthForModels: true,
password: "",
});
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "openai-catalog-9199",
apiKey: "sk-openai-catalog-9199",
isActive: true,
testStatus: "active",
rateLimitedUntil: null,
providerSpecificData: {},
});
const apiKey = await apiKeysDb.createApiKey("catalog pricing", "machine-pricing-9199");
const db = core.getDbInstance();
db.prepare("INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES (?, ?, ?)").run(
"models_dev_pricing",
"openai",
JSON.stringify({ "gpt-4o": { input: 1.25, output: 5 } })
);
const originalPrepare = db.prepare;
const callPrepare = originalPrepare.bind(db);
let pricingReads = 0;
let catalogSettled = false;
let heartbeatScheduled = false;
let resolveHeartbeat!: (response: Response) => void;
let rejectHeartbeat!: (error: unknown) => void;
const heartbeatPromise = new Promise<Response>((resolve, reject) => {
resolveHeartbeat = resolve;
rejectHeartbeat = reject;
});
(db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => {
const normalized = String(sql).replace(/\s+/g, " ").trim();
if (
normalized === "SELECT key, value FROM key_value WHERE namespace = 'models_dev_pricing'"
) {
pricingReads++;
if (!heartbeatScheduled) {
heartbeatScheduled = true;
setImmediate(() => {
healthRoute.GET().then(resolveHeartbeat, rejectHeartbeat);
});
}
}
return callPrepare(sql);
}) as typeof db.prepare;
let catalog!: Response;
let heartbeat!: Response;
let catalogSettledBeforeHeartbeat = true;
try {
const catalogPromise = modelsRoute
.GET(
new Request("http://localhost/v1/models?prefix=alias", {
headers: { Authorization: `Bearer ${apiKey.key}` },
})
)
.then((response) => {
catalogSettled = true;
return response;
});
heartbeat = await heartbeatPromise;
catalogSettledBeforeHeartbeat = catalogSettled;
catalog = await catalogPromise;
} finally {
(db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare;
}
assert.equal(catalog.status, 200);
assert.equal(heartbeat.status, 200);
assert.equal(pricingReads, 1, "one catalog build must parse one build-local pricing snapshot");
assert.equal(
catalogSettledBeforeHeartbeat,
false,
"a heartbeat queued by the pricing read must run before catalog enrichment publishes"
);
const body = (await catalog.json()) as {
data?: Array<{
root?: string;
owned_by?: string;
pricing?: { input?: number; output?: number };
}>;
};
assert.ok(Array.isArray(body.data));
const priced = body.data.find(
(model) => model.root === "gpt-4o" && model.owned_by === "openai"
);
assert.deepEqual(priced?.pricing, { input: 1.25, output: 5 });
}
);

164
tests/unit/auto-combo-context-advertising.test.ts
View File

@@ -37,6 +37,7 @@ const core = await import("../../src/lib/db/core.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");
const virtualFactory = await import("../../open-sse/services/autoCombo/virtualFactory.ts");
const suffixComposition = await import("../../open-sse/services/autoCombo/suffixComposition.ts");
const contextManager = await import("../../open-sse/services/contextManager.ts");
const combosAutoRoute = await import("../../src/app/api/combos/auto/route.ts");
@@ -77,6 +78,169 @@ test("computeAdvertisedLimits returns MAX of candidates' known context windows",
);
});
test("#9199 computeAdvertisedLimits reuses build-local prepared capability values", () => {
const result = virtualFactory.computeAdvertisedLimits([
{
provider: "prepared-provider",
model: "prepared-model",
resolvedContextLength: 654321,
resolvedMaxOutputTokens: 12345,
},
]);
assert.deepEqual(result, {
contextLength: 654321,
maxOutputTokens: 12345,
});
});
test("#9199 prepared auto inputs resolve each candidate capability before materialization", async () => {
const prepared = await virtualFactory.prepareVirtualAutoComboInputs({
includeResolvedCapabilities: true,
});
const candidates = [...prepared.regularCandidates, ...prepared.familyCandidates];
assert.ok(candidates.length > 0, "the built-in no-auth registry should provide candidates");
assert.ok(
candidates.every(
(candidate) =>
Object.hasOwn(candidate, "resolvedContextLength") &&
Object.hasOwn(candidate, "resolvedMaxOutputTokens") &&
Object.hasOwn(candidate, "resolvedSupportsVision") &&
Object.hasOwn(candidate, "resolvedReasoning") &&
Object.hasOwn(candidate, "resolvedSupportsThinking")
),
"every prepared candidate should carry a build-local capability snapshot"
);
});
test("#9199 catalog capability preparation bulk-loads each capability table once", async () => {
const db = core.getDbInstance();
const originalPrepare = db.prepare;
const callPrepare = originalPrepare.bind(db);
const counts = {
synced: 0,
capabilityOverrides: 0,
contextOverrides: 0,
};
(db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => {
const normalized = String(sql).replace(/\s+/g, " ").trim();
if (normalized.includes("FROM model_capabilities")) counts.synced++;
if (normalized.includes("FROM model_capability_overrides")) counts.capabilityOverrides++;
if (normalized.includes("FROM model_context_overrides")) counts.contextOverrides++;
return callPrepare(sql);
}) as typeof db.prepare;
try {
const prepared = await virtualFactory.prepareVirtualAutoComboInputs({
includeResolvedCapabilities: true,
});
assert.ok(prepared.regularCandidates.length + prepared.familyCandidates.length > 0);
} finally {
(db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare;
}
assert.deepEqual(counts, {
synced: 1,
capabilityOverrides: 1,
contextOverrides: 1,
});
});
test("#9199 default runtime preparation does not retain catalog-only capabilities", async () => {
const prepared = await virtualFactory.prepareVirtualAutoComboInputs();
const candidates = [...prepared.regularCandidates, ...prepared.familyCandidates];
assert.ok(candidates.length > 0);
assert.ok(
candidates.every((candidate) => !Object.hasOwn(candidate, "resolvedContextLength")),
"runtime routing should keep its existing on-demand capability resolution"
);
});
test("#9199 default runtime preparation does not bulk-load capability tables", async () => {
const db = core.getDbInstance();
const originalPrepare = db.prepare;
const callPrepare = originalPrepare.bind(db);
const counts = {
synced: 0,
capabilityOverrides: 0,
contextOverrides: 0,
};
(db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => {
const normalized = String(sql).replace(/\s+/g, " ").trim();
if (normalized.includes("FROM model_capabilities")) counts.synced++;
if (normalized.includes("FROM model_capability_overrides")) counts.capabilityOverrides++;
if (normalized.includes("FROM model_context_overrides")) counts.contextOverrides++;
return callPrepare(sql);
}) as typeof db.prepare;
try {
const prepared = await virtualFactory.prepareVirtualAutoComboInputs();
assert.ok(prepared.regularCandidates.length + prepared.familyCandidates.length > 0);
} finally {
(db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare;
}
assert.deepEqual(counts, {
synced: 0,
capabilityOverrides: 0,
contextOverrides: 0,
});
});
test("#9199 capability preparation yields before publishing the snapshot", async () => {
let heartbeatRan = false;
const preparation = virtualFactory.prepareVirtualAutoComboInputs({
includeResolvedCapabilities: true,
});
setImmediate(() => {
heartbeatRan = true;
});
await preparation;
assert.equal(heartbeatRan, true);
});
test("#9199 category filters reuse prepared capability flags", () => {
const filter = suffixComposition.buildAutoCandidateFilter("reasoning");
assert.ok(filter);
assert.equal(
filter({
provider: "openai",
model: "gpt-5.4",
resolvedReasoning: false,
resolvedSupportsThinking: false,
}),
false
);
});
test("#9199 materialization passes prepared capability flags into category filters", async () => {
const candidate = {
provider: "openai",
connectionId: null,
allowedConnectionIds: ["prepared-connection"],
model: "gpt-5.4",
modelStr: "openai/gpt-5.4",
costPer1MTokens: 0,
resolvedContextLength: 400000,
resolvedMaxOutputTokens: 64000,
resolvedSupportsVision: false,
resolvedReasoning: false,
resolvedSupportsThinking: false,
};
const combo = await virtualFactory.createVirtualAutoComboFromPrepared(
{ regularCandidates: [candidate], familyCandidates: [candidate] },
undefined,
{ category: "reasoning" }
);
assert.equal(combo.models.length, 0);
});
test("computeAdvertisedLimits returns null limits for an empty candidate pool", () => {
const { computeAdvertisedLimits } = virtualFactory as unknown as {
computeAdvertisedLimits: (candidates: Array<{ provider: string; model: string }>) => {

530
tests/unit/model-capability-resolution-snapshot-9199.test.ts Normal file
View File

@@ -0,0 +1,530 @@
/**
* #9199 — build-local ModelCapabilityResolutionSnapshot parity.
*
* Snapshot-backed resolution must match ordinary on-demand resolvers for the
* same pure chain (synced + max_token override + context override + alias
* fallback + missing data). The snapshot must not flip models.dev's module-
* global all-row cache.
*/
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cap-snapshot-9199-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET ?? "cap-snapshot-9199-secret";
// Isolate getTokenLimit from host env CONTEXT_LENGTH_* overrides.
const originalContextLengthEnv = new Map(
Object.entries(process.env).filter(([key]) => key.startsWith("CONTEXT_LENGTH_"))
);
for (const key of originalContextLengthEnv.keys()) delete process.env[key];
const core = await import("../../src/lib/db/core.ts");
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
const capabilityOverrides = await import("../../src/lib/db/modelCapabilityOverrides.ts");
const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts");
const modelCapabilities = await import("../../src/lib/modelCapabilities.ts");
const contextManager = await import("../../open-sse/services/contextManager.ts");
const model = await import("../../open-sse/services/model.ts");
const { MODEL_SPECS } = await import("../../src/shared/constants/modelSpecs.ts");
const { REGISTRY } = await import("../../open-sse/config/providerRegistry.ts");
test.after(() => {
core.resetDbInstance();
for (const key of Object.keys(process.env)) {
if (key.startsWith("CONTEXT_LENGTH_")) delete process.env[key];
}
for (const [key, value] of originalContextLengthEnv) process.env[key] = value;
try {
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
} catch {
// best-effort cleanup
}
});
function seedFixture() {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
core.getDbInstance();
modelsDevSync.saveModelsDevCapabilities({
// Direct provider/model hit used by ordinary + snapshot paths.
"parity-provider": {
"parity-model": {
tool_call: true,
reasoning: true,
attachment: false,
structured_output: null,
temperature: true,
modalities_input: '["text"]',
modalities_output: '["text"]',
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: null,
open_weights: null,
limit_context: 111111,
limit_input: 100000,
limit_output: 2222,
interleaved_field: null,
},
},
// Alias-side storage only — canonical `opencode` must still resolve via fallback.
"opencode-zen": {
"zen-only-model": {
tool_call: true,
reasoning: false,
attachment: false,
structured_output: null,
temperature: true,
modalities_input: '["text"]',
modalities_output: '["text"]',
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: null,
open_weights: null,
limit_context: 333333,
limit_input: 300000,
limit_output: 4444,
interleaved_field: null,
},
},
});
assert.equal(
capabilityOverrides.setModelCapabilityOverride(
"parity-provider/parity-model",
"max_token",
99999
),
true
);
// Malformed capability override row must be filtered from list/bulk maps.
core
.getDbInstance()
.prepare(
"INSERT OR REPLACE INTO model_capability_overrides " +
"(provider, model_id, override_key, override_value, refreshed_at) " +
"VALUES (?, ?, ?, ?, datetime('now'))"
)
.run("parity-provider", "parity-model-bad", "max_token", "not-a-number");
// Collision pairs that a delimiter-composed key would merge: (a, b\0c) vs (a\0b, c).
assert.equal(
capabilityOverrides.setModelCapabilityOverride("a/b\u0000c", "max_token", 10101),
true
);
assert.equal(
capabilityOverrides.setModelCapabilityOverride("a\u0000b/c", "max_token", 20202),
true
);
assert.equal(contextOverrides.setModelContextOverride("a", "b\u0000c", 30303, "manual"), true);
assert.equal(contextOverrides.setModelContextOverride("a\u0000b", "c", 40404, "manual"), true);
// Real provider-model alias from open-sse/services/model.ts PROVIDER_MODEL_ALIASES.
// Override is stored only under the raw alias id, not the canonical model.
const aliasCanonical = model.resolveCanonicalProviderModel("github", "claude-4.5-opus");
assert.equal(aliasCanonical.provider, "github");
assert.equal(aliasCanonical.model, "claude-opus-4-5-20251101");
assert.notEqual(aliasCanonical.model, "claude-4.5-opus");
assert.equal(
capabilityOverrides.setModelCapabilityOverride("github/claude-4.5-opus", "max_token", 77777),
true
);
assert.equal(
capabilityOverrides.getModelCapabilityOverride(
"github",
"claude-opus-4-5-20251101",
"max_token"
),
null,
"override must exist only under the raw alias model id"
);
assert.equal(
contextOverrides.setModelContextOverride("parity-provider", "parity-model", 555555, "manual"),
true
);
}
function insertRawCapability(provider: string, modelId: string): void {
modelsDevSync.ensureCapabilitiesTable();
core
.getDbInstance()
.prepare(
"INSERT OR REPLACE INTO model_capabilities " +
"(provider, model_id, tool_call, reasoning, attachment, structured_output, temperature, " +
"modalities_input, modalities_output, knowledge_cutoff, release_date, last_updated, " +
"status, family, open_weights, limit_context, limit_input, limit_output, interleaved_field, last_synced) " +
"VALUES (?, ?, 1, 1, 0, NULL, 1, '[]', '[]', NULL, NULL, NULL, NULL, NULL, NULL, 111111, 100000, 2222, NULL, datetime('now'))"
)
.run(provider, modelId);
}
function pickParityFields(
resolved: ReturnType<typeof modelCapabilities.getResolvedModelCapabilities>
) {
return {
provider: resolved.provider,
model: resolved.model,
rawModel: resolved.rawModel,
toolCalling: resolved.toolCalling,
reasoning: resolved.reasoning,
supportsThinking: resolved.supportsThinking,
supportsTools: resolved.supportsTools,
supportsVision: resolved.supportsVision,
contextWindow: resolved.contextWindow,
maxInputTokens: resolved.maxInputTokens,
maxOutputTokens: resolved.maxOutputTokens,
attachment: resolved.attachment,
modalitiesInput: resolved.modalitiesInput,
modalitiesOutput: resolved.modalitiesOutput,
};
}
function assertOrdinarySnapshotParity(
provider: string,
modelId: string,
snapshot: modelCapabilities.ModelCapabilityResolutionSnapshot,
label: string
) {
const ordinaryCaps = modelCapabilities.getResolvedModelCapabilities({
provider,
model: modelId,
});
const snapshotCaps = modelCapabilities.getResolvedModelCapabilities(
{ provider, model: modelId },
snapshot
);
assert.deepEqual(
pickParityFields(snapshotCaps),
pickParityFields(ordinaryCaps),
`${label}: getResolvedModelCapabilities parity`
);
const ordinaryLimit = contextManager.getTokenLimit(provider, modelId);
const snapshotLimit = contextManager.getTokenLimit(provider, modelId, snapshot);
assert.equal(
snapshotLimit,
ordinaryLimit,
`${label}: getTokenLimit parity (got ${snapshotLimit}, want ${ordinaryLimit})`
);
const ordinaryContext = modelCapabilities.getModelContextLimit(provider, modelId);
const snapshotContext = modelCapabilities.getModelContextLimit(provider, modelId, snapshot);
assert.equal(snapshotContext, ordinaryContext, `${label}: getModelContextLimit parity`);
}
test("#9199 bulk capability rows treat prototype-shaped keys as data", () => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
core.getDbInstance();
insertRawCapability("__proto__", "polluted-model");
insertRawCapability("prototype-provider", "__proto__");
assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);
const ordinary = modelsDevSync.getSyncedCapability("__proto__", "polluted-model");
assert.equal(ordinary?.limit_context, 111111, "ordinary point lookup must support the row");
assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);
try {
const bulk = modelsDevSync.loadAllSyncedCapabilitiesUncached();
assert.equal(Object.hasOwn(bulk, "__proto__"), true);
assert.equal(bulk["__proto__"]?.["polluted-model"]?.limit_output, 2222);
assert.equal(Object.hasOwn(bulk["prototype-provider"], "__proto__"), true);
assert.equal(bulk["prototype-provider"]?.["__proto__"]?.limit_context, 111111);
assert.equal(Object.hasOwn(Object.prototype, "polluted-model"), false);
} finally {
delete (Object.prototype as Record<string, unknown>)["polluted-model"];
}
});
test("#9199 uncached bulk load does not mutate models.dev all-row cache", () => {
seedFixture();
// clearModelsDevCapabilities leaves cachedCapabilitiesLoadedAll=true with {}.
modelsDevSync.clearModelsDevCapabilities();
assert.equal(modelsDevSync.getSyncedCapability("parity-provider", "parity-model"), null);
// Insert behind the module-global cache via raw SQL.
const db = core.getDbInstance();
insertRawCapability("parity-provider", "parity-model");
// Ordinary path still sees the empty module cache.
assert.equal(modelsDevSync.getSyncedCapability("parity-provider", "parity-model"), null);
const bulk = modelsDevSync.loadAllSyncedCapabilitiesUncached();
assert.equal(bulk["parity-provider"]?.["parity-model"]?.limit_context, 111111);
// Uncached bulk must not publish into the module-global cache.
assert.equal(
modelsDevSync.getSyncedCapability("parity-provider", "parity-model"),
null,
"loadAllSyncedCapabilitiesUncached must not flip cachedCapabilitiesLoadedAll data"
);
const originalPrepare = db.prepare;
const callPrepare = originalPrepare.bind(db);
const counts = { synced: 0, capabilityOverrides: 0, contextOverrides: 0 };
(db as unknown as { prepare: typeof db.prepare }).prepare = ((sql: string) => {
const normalized = String(sql).replace(/\s+/g, " ").trim();
if (normalized.includes("FROM model_capabilities")) counts.synced++;
if (normalized.includes("FROM model_capability_overrides")) counts.capabilityOverrides++;
if (normalized.includes("FROM model_context_overrides")) counts.contextOverrides++;
return callPrepare(sql);
}) as typeof db.prepare;
try {
const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();
assert.equal(snapshot.synced["parity-provider"]?.["parity-model"]?.limit_output, 2222);
} finally {
(db as unknown as { prepare: typeof db.prepare }).prepare = originalPrepare;
}
assert.deepEqual(counts, { synced: 1, capabilityOverrides: 1, contextOverrides: 1 });
assert.equal(
modelsDevSync.getSyncedCapability("parity-provider", "parity-model"),
null,
"snapshot creation must remain build-local"
);
});
test("#9199 nested override maps keep delimiter-colliding pairs distinct", () => {
seedFixture();
const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();
// Delimiter-composed keys would merge these pairs; nested maps must not.
assert.equal(snapshot.maxTokenOverrides.get("a")?.get("b\u0000c"), 10101);
assert.equal(snapshot.maxTokenOverrides.get("a\u0000b")?.get("c"), 20202);
assert.notEqual(
snapshot.maxTokenOverrides.get("a")?.get("b\u0000c"),
snapshot.maxTokenOverrides.get("a\u0000b")?.get("c")
);
assert.equal(snapshot.contextOverrides.get("a")?.get("b\u0000c"), 30303);
assert.equal(snapshot.contextOverrides.get("a\u0000b")?.get("c"), 40404);
assert.notEqual(
snapshot.contextOverrides.get("a")?.get("b\u0000c"),
snapshot.contextOverrides.get("a\u0000b")?.get("c")
);
assert.equal(
capabilityOverrides.getModelCapabilityOverride(
"a",
"b\u0000c",
"max_token",
snapshot.maxTokenOverrides
),
10101
);
assert.equal(
capabilityOverrides.getModelCapabilityOverride(
"a\u0000b",
"c",
"max_token",
snapshot.maxTokenOverrides
),
20202
);
assert.equal(
contextOverrides.getModelContextOverride("a", "b\u0000c", snapshot.contextOverrides),
30303
);
assert.equal(
contextOverrides.getModelContextOverride("a\u0000b", "c", snapshot.contextOverrides),
40404
);
assertOrdinarySnapshotParity("a", "b\u0000c", snapshot, "collision pair (a, b\\0c)");
assertOrdinarySnapshotParity("a\u0000b", "c", snapshot, "collision pair (a\\0b, c)");
assert.equal(
modelCapabilities.getResolvedModelCapabilities({ provider: "a", model: "b\u0000c" }, snapshot)
.maxOutputTokens,
10101
);
assert.equal(
modelCapabilities.getResolvedModelCapabilities({ provider: "a\u0000b", model: "c" }, snapshot)
.maxOutputTokens,
20202
);
assert.equal(modelCapabilities.getModelContextLimit("a", "b\u0000c", snapshot), 30303);
assert.equal(modelCapabilities.getModelContextLimit("a\u0000b", "c", snapshot), 40404);
});
test("#9199 snapshot-backed resolution matches ordinary resolvers across resolution paths", () => {
seedFixture();
const snapshot = modelCapabilities.createModelCapabilityResolutionSnapshot();
// Static MODEL_SPECS pin (not seeded in models.dev / overrides).
const staticSpec = MODEL_SPECS["gpt-4o-mini"];
assert.ok(staticSpec, "gpt-4o-mini must remain a real MODEL_SPECS entry");
assert.equal(typeof staticSpec.maxOutputTokens, "number");
assert.equal(typeof staticSpec.contextWindow, "number");
assert.equal(
MODEL_SPECS["glm-5-turbo"],
undefined,
"glm-5-turbo must stay absent from MODEL_SPECS so its case exercises PROVIDER_MODELS"
);
// Provider registry defaultContextLength path: no per-model catalog hit.
const defaultContextProvider = "gitlab-duo";
const defaultContextModel = "no-catalog-model-9199";
assert.equal(
REGISTRY[defaultContextProvider]?.defaultContextLength,
128000,
"gitlab-duo must keep a real provider-registry defaultContextLength"
);
assert.equal(
modelCapabilities.getModelContextLimit(defaultContextProvider, defaultContextModel),
null,
"default-context path requires no resolved per-model contextWindow"
);
// Model-name heuristic path: unknown provider with a claude-shaped model id.
const heuristicProvider = "unknown-provider-9199";
const heuristicModel = "custom-claude-lab";
assert.equal(REGISTRY[heuristicProvider], undefined);
assert.equal(
modelCapabilities.getModelContextLimit(heuristicProvider, heuristicModel),
null,
"heuristic path requires no resolved per-model contextWindow"
);
const cases: Array<{ provider: string; model: string; label: string }> = [
{ provider: "parity-provider", model: "parity-model", label: "direct synced + both overrides" },
{ provider: "opencode", model: "zen-only-model", label: "alias fallback to opencode-zen" },
{
provider: "github",
model: "claude-4.5-opus",
label: "canonical model alias with rawModel-only max_token override",
},
{
provider: "glm",
model: "glm-5-turbo",
label: "registry-backed model (PROVIDER_MODELS context/output limits)",
},
{
provider: "missing-provider",
model: "gpt-4o-mini",
label: "static MODEL_SPECS-backed model",
},
{
provider: defaultContextProvider,
model: defaultContextModel,
label: "provider registry defaultContextLength path",
},
{
provider: heuristicProvider,
model: heuristicModel,
label: "model-name heuristic/default path",
},
{ provider: "missing-provider", model: "missing-model", label: "missing synced + overrides" },
];
for (const entry of cases) {
assertOrdinarySnapshotParity(entry.provider, entry.model, snapshot, entry.label);
}
// Explicit override/precedence pins so the test is not only structural.
assert.equal(
modelCapabilities.getResolvedModelCapabilities(
{ provider: "parity-provider", model: "parity-model" },
snapshot
).maxOutputTokens,
99999,
"max_token override must win over synced limit_output"
);
assert.equal(
modelCapabilities.getModelContextLimit("parity-provider", "parity-model", snapshot),
555555,
"context override must win over synced limit_context / resolved contextWindow"
);
assert.equal(
modelCapabilities.getResolvedModelCapabilities(
{ provider: "opencode", model: "zen-only-model" },
snapshot
).contextWindow,
333333,
"canonical opencode must resolve capabilities stored under opencode-zen"
);
const aliasResolved = modelCapabilities.getResolvedModelCapabilities(
{ provider: "github", model: "claude-4.5-opus" },
snapshot
);
assert.equal(aliasResolved.rawModel, "claude-4.5-opus");
assert.equal(aliasResolved.model, "claude-opus-4-5-20251101");
assert.equal(
aliasResolved.maxOutputTokens,
77777,
"max_token override stored only under rawModel must still apply after alias resolution"
);
const canonicalOnly = modelCapabilities.getResolvedModelCapabilities(
{ provider: "github", model: "claude-opus-4-5-20251101" },
snapshot
);
assert.equal(canonicalOnly.rawModel, "claude-opus-4-5-20251101");
assert.equal(canonicalOnly.model, "claude-opus-4-5-20251101");
assert.notEqual(
canonicalOnly.maxOutputTokens,
77777,
"canonical id alone must not invent the rawModel-only max_token override"
);
// Registry-backed pins: glm-5-turbo is absent from MODEL_SPECS.
const registryResolved = modelCapabilities.getResolvedModelCapabilities(
{ provider: "glm", model: "glm-5-turbo" },
snapshot
);
assert.equal(registryResolved.contextWindow, 200000);
assert.equal(registryResolved.maxOutputTokens, 131072);
// Static MODEL_SPECS pin without a provider registry hit for this missing provider.
assert.equal(
modelCapabilities.getResolvedModelCapabilities(
{ provider: "missing-provider", model: "gpt-4o-mini" },
snapshot
).maxOutputTokens,
staticSpec.maxOutputTokens
);
assert.equal(
modelCapabilities.getResolvedModelCapabilities(
{ provider: "missing-provider", model: "gpt-4o-mini" },
snapshot
).contextWindow,
staticSpec.contextWindow
);
// Provider defaultContextLength path (getTokenLimit step 3).
assert.equal(
contextManager.getTokenLimit(defaultContextProvider, defaultContextModel, snapshot),
128000
);
// Model-name heuristic path (getTokenLimit step 4 → DEFAULT_LIMITS.claude).
assert.equal(contextManager.getTokenLimit(heuristicProvider, heuristicModel, snapshot), 200000);
// Missing path remains a pure default/fallback, not a catalog hit.
assert.equal(
modelCapabilities.getResolvedModelCapabilities(
{ provider: "missing-provider", model: "missing-model" },
snapshot
).contextWindow,
null
);
assert.equal(contextManager.getTokenLimit("missing-provider", "missing-model", snapshot), 128000);
assert.equal(
snapshot.maxTokenOverrides.get("parity-provider")?.has("parity-model-bad") ?? false,
false,
"malformed max_token rows must be filtered from the bulk map"
);
});

276
tests/unit/model-catalog-runtime-invalidation.test.ts Normal file
View File

@@ -0,0 +1,276 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(
path.join(os.tmpdir(), "omniroute-model-catalog-runtime-invalidation-")
);
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET =
process.env.API_KEY_SECRET || "catalog-runtime-invalidation-test-secret";
const core = await import("../../src/lib/db/core.ts");
const apiKeysDb = await import("../../src/lib/db/apiKeys.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const settingsDb = await import("../../src/lib/db/settings.ts");
const capabilityOverrides = await import("../../src/lib/db/modelCapabilityOverrides.ts");
const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts");
const readCache = await import("../../src/lib/db/readCache.ts");
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts");
const auth = await import("../../src/sse/services/auth.ts");
async function resetStorage() {
core.resetDbInstance();
apiKeysDb.resetApiKeyState();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
v1ModelsCatalog.__resetCatalogBuilderRunsForTest();
}
async function seedOpenAiConnection() {
return providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "openai-catalog-invalidation",
apiKey: "sk-test",
isActive: true,
testStatus: "active",
providerSpecificData: {},
});
}
function catalogRequest() {
return new Request("http://localhost/api/v1/models?prefix=alias");
}
test.beforeEach(async () => {
await resetStorage();
});
test.after(async () => {
core.resetDbInstance();
apiKeysDb.resetApiKeyState();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
});
test("session-affinity bookkeeping preserves the published model catalog", async () => {
await settingsDb.updateSettings({ sessionAffinityTtlMs: 60_000 });
const connection = await seedOpenAiConnection();
const firstResponse = await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const firstBody = await firstResponse.text();
const firstSelection = await auth.getProviderCredentials("openai", null, null, "gpt-5.4-mini", {
sessionKey: "catalog-runtime-affinity-session",
forcedConnectionId: connection.id as string,
});
const secondSelection = await auth.getProviderCredentials("openai", null, null, "gpt-5.4-mini", {
sessionKey: "catalog-runtime-affinity-session",
forcedConnectionId: connection.id as string,
});
assert.equal(firstSelection?.connectionId, connection.id);
assert.equal(secondSelection?.connectionId, connection.id);
const persisted = await providersDb.getProviderConnectionById(connection.id as string);
assert.equal(
persisted?.consecutiveUseCount,
2,
"reusing a cached affinity connection must keep its usage counter current"
);
assert.equal(typeof persisted?.lastUsedAt, "string");
const secondResponse = await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const secondBody = await secondResponse.text();
assert.equal(secondBody, firstBody);
assert.equal(
v1ModelsCatalog.__getCatalogBuilderRunsForTest(),
1,
"runtime-only affinity bookkeeping must not force a second catalog build"
);
});
test("catalog-affecting connection changes still rebuild the published catalog", async () => {
const connection = await seedOpenAiConnection();
const firstResponse = await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const firstBody = (await firstResponse.json()) as { data: Array<{ id: string }> };
assert.equal(
firstBody.data.some((model) => model.id === "openai/gpt-5.4-mini"),
true
);
await providersDb.updateProviderConnection(connection.id as string, {
providerSpecificData: {
excludedModels: ["gpt-5.4*"],
},
});
const secondResponse = await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const secondBody = (await secondResponse.json()) as { data: Array<{ id: string }> };
assert.equal(
secondBody.data.some((model) => model.id === "openai/gpt-5.4-mini"),
false
);
assert.equal(
v1ModelsCatalog.__getCatalogBuilderRunsForTest(),
2,
"catalog-affecting connection changes must keep hard invalidation"
);
});
test("#9199 capability data writes advance the model-catalog generation", () => {
const expectGenerationAdvance = (label: string, mutate: () => void) => {
const before = readCache.getModelCatalogCacheVersion();
mutate();
assert.ok(
readCache.getModelCatalogCacheVersion() > before,
`${label} must hard-invalidate the published model catalog`
);
};
const expectGenerationUnchanged = (label: string, mutate: () => void) => {
const before = readCache.getModelCatalogCacheVersion();
mutate();
assert.equal(
readCache.getModelCatalogCacheVersion(),
before,
`${label} must not invalidate when no capability row changed`
);
};
expectGenerationAdvance("setModelCapabilityOverride", () => {
assert.equal(
capabilityOverrides.setModelCapabilityOverride("openai/gpt-5.4-mini", "max_token", 64000),
true
);
});
expectGenerationAdvance("removeModelCapabilityOverride", () => {
assert.equal(
capabilityOverrides.removeModelCapabilityOverride("openai/gpt-5.4-mini", "max_token"),
true
);
});
expectGenerationAdvance("setModelContextOverride", () => {
assert.equal(contextOverrides.setModelContextOverride("openai", "gpt-5.4-mini", 400000), true);
});
expectGenerationAdvance("removeModelContextOverride", () => {
assert.equal(contextOverrides.removeModelContextOverride("openai", "gpt-5.4-mini"), true);
});
expectGenerationUnchanged("empty saveModelsDevCapabilities", () => {
modelsDevSync.saveModelsDevCapabilities({});
});
expectGenerationAdvance("saveModelsDevCapabilities", () => {
modelsDevSync.saveModelsDevCapabilities({
openai: {
"gpt-5.4-mini": {
tool_call: true,
reasoning: true,
attachment: false,
structured_output: true,
temperature: true,
modalities_input: '["text"]',
modalities_output: '["text"]',
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: "gpt",
open_weights: false,
limit_context: 400000,
limit_input: 380000,
limit_output: 64000,
interleaved_field: null,
},
},
});
});
expectGenerationAdvance("clearModelsDevCapabilities", () => {
modelsDevSync.clearModelsDevCapabilities();
});
expectGenerationUnchanged("empty clearModelsDevCapabilities", () => {
modelsDevSync.clearModelsDevCapabilities();
});
});
test("#9199 a capability mutation during preparation detaches the obsolete generation", async () => {
let firstSettled = false;
const firstPromise = v1ModelsCatalog
.getUnifiedModelsResponse(catalogRequest())
.then((response) => {
firstSettled = true;
return response;
});
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(
firstSettled,
false,
"the mutation must occur while capability preparation is active"
);
assert.equal(
capabilityOverrides.setModelCapabilityOverride("openai/gpt-5.4-mini", "max_token", 64000),
true
);
const secondPromise = v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
await Promise.all([firstPromise, secondPromise]);
assert.equal(
v1ModelsCatalog.__getCatalogBuilderRunsForTest(),
2,
"a post-mutation caller must not join capability work from the old generation"
);
await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
assert.equal(
v1ModelsCatalog.__getCatalogBuilderRunsForTest(),
2,
"obsolete capability work must not repopulate the current cache generation"
);
});
test("#9199 a mutation during a cooperative catalog build detaches the obsolete generation", async () => {
await settingsDb.updateSettings({ blockedProviders: [] });
let firstSettled = false;
const firstPromise = v1ModelsCatalog
.getUnifiedModelsResponse(catalogRequest())
.then((response) => {
firstSettled = true;
return response;
});
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(firstSettled, false, "the mutation must occur while the first build is in flight");
await settingsDb.updateSettings({ blockedProviders: ["auto"] });
const secondPromise = v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const [firstResponse, secondResponse] = await Promise.all([firstPromise, secondPromise]);
const firstBody = (await firstResponse.json()) as { data: Array<{ id: string }> };
const secondBody = (await secondResponse.json()) as { data: Array<{ id: string }> };
const thirdResponse = await v1ModelsCatalog.getUnifiedModelsResponse(catalogRequest());
const thirdBody = (await thirdResponse.json()) as { data: Array<{ id: string }> };
const advertisesAuto = (body: { data: Array<{ id: string }> }) =>
body.data.some((model) => model.id === "auto/best-coding");
assert.equal(advertisesAuto(firstBody), true, "the pre-mutation caller keeps its own snapshot");
assert.equal(
advertisesAuto(secondBody),
false,
"a post-mutation caller must not join the obsolete in-flight build"
);
assert.equal(
advertisesAuto(thirdBody),
false,
"the obsolete build must not repopulate the current cache generation"
);
assert.equal(
v1ModelsCatalog.__getCatalogBuilderRunsForTest(),
2,
"the current generation must publish from a distinct builder run"
);
});