diff --git a/changelog.d/features/8908-model-token-limit-overrides.md b/changelog.d/features/8908-model-token-limit-overrides.md new file mode 100644 index 0000000000..8d32a9e1f0 --- /dev/null +++ b/changelog.d/features/8908-model-token-limit-overrides.md @@ -0,0 +1 @@ +- **feat(models):** add exact per-model `context_length`, `max_input_tokens`, and `max_output_tokens` overrides across model discovery and runtime enforcement, with automatic migration from the retired output-only `max_token` key ([#8908](https://github.com/diegosouzapw/OmniRoute/pull/8908)) — thanks @xz-dev diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 913716487e..b74ec33890 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -139,6 +139,7 @@ import { supportsMaxTokens, getResolvedModelCapabilities, getExplicitModelOutputCap, + resolveInputTokenCapForGate, } from "@/lib/modelCapabilities.ts"; import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts"; @@ -1857,11 +1858,6 @@ export async function handleChatCore({ } } - // Key the lookup by { provider, model } — the bare-string form resolves to - // `provider: null`, which skips both the registry cap and the operator's - // `max_token` capability override (#6524), the documented escape hatch for a - // wrong synced `limit_output`. Clamping against a stale spec while the operator - // raised the ceiling would silently truncate output. const modelOutputCap = toPositiveInteger( getExplicitModelOutputCap({ provider, model: effectiveModel }) ); @@ -1870,13 +1866,15 @@ export async function handleChatCore({ finalEstimatedInputTokens, finalContextLimit, targetFormat === FORMATS.CLAUDE && sourceFormat !== FORMATS.CLAUDE ? DEFAULT_MAX_TOKENS : 0, - modelOutputCap + modelOutputCap, + toPositiveInteger(resolveInputTokenCapForGate({ provider, model: effectiveModel }, { isCombo })) ); if (!outputBudget.ok) { + const exceededInputCap = outputBudget.maxInputTokens !== undefined; const message = - `Input exceeds the context window for ${provider}/${effectiveModel}: ` + - `estimated ${outputBudget.estimatedInputTokens} input tokens, limit ${outputBudget.contextLimit}. ` + - "Reduce the prompt or route to a model with a larger context window."; + `Input exceeds ${exceededInputCap ? "maximum input tokens" : "context window"} for ${provider}/${effectiveModel}: ` + + `estimated ${outputBudget.estimatedInputTokens} input tokens, ${exceededInputCap ? `max input ${outputBudget.maxInputTokens}` : `limit ${outputBudget.contextLimit}`}. ` + + `Reduce the prompt or route to a model with a larger ${exceededInputCap ? "input limit" : "context window"}.`; log?.warn?.("CONTEXT", message); trackPendingRequest(model, provider, connectionId, false); return createErrorResult( diff --git a/open-sse/handlers/chatCore/outputTokenBudget.ts b/open-sse/handlers/chatCore/outputTokenBudget.ts index 62752e42f4..3adb97fd81 100644 --- a/open-sse/handlers/chatCore/outputTokenBudget.ts +++ b/open-sse/handlers/chatCore/outputTokenBudget.ts @@ -15,6 +15,7 @@ export type OutputTokenBudgetResult = ok: false; estimatedInputTokens: number; contextLimit: number; + maxInputTokens?: number | null; }; type OutputTokenAdjustment = { field: string; value?: number; remove?: boolean }; @@ -74,19 +75,43 @@ function adjustOutputTokenFields( * cap limits how much is requested, not whether the request fits. Absent / * null / non-positive cap values leave behavior byte-identical to before this * parameter existed (fail-open). + * + * `maxInputTokenCap` (the model's own input ceiling, `maxInputTokens`) is an + * additional, independent input-only bound enforced on the accept/reject + * decision. The total-window check (`contextLimit - input >= 1`) stays in place + * and remains responsible for reserving output room; the input cap never + * double-counts a requested output. Absent / null / non-positive input caps + * leave behavior byte-identical (fail-open). */ export function enforceOutputTokenBudget( body: Record | null | undefined, estimatedInputTokens: number, contextLimit: number, defaultOutputTokens = 0, - maxOutputTokenCap?: number | null + maxOutputTokenCap?: number | null, + maxInputTokenCap?: number | null ): OutputTokenBudgetResult { const normalizedInputTokens = Math.max(0, Math.ceil(estimatedInputTokens)); const normalizedContextLimit = Math.max(1, Math.floor(contextLimit)); const normalizedDefaultOutputTokens = Math.max(0, Math.floor(defaultOutputTokens)); const availableOutputTokens = normalizedContextLimit - normalizedInputTokens; + // Independent input-only ceiling: reject when the prompt alone exceeds the + // model's declared max input, regardless of remaining output room. + const normalizedInputCap = maxInputTokenCap == null ? null : Math.floor(maxInputTokenCap); + if ( + normalizedInputCap !== null && + normalizedInputCap > 0 && + normalizedInputTokens > normalizedInputCap + ) { + return { + ok: false, + estimatedInputTokens: normalizedInputTokens, + contextLimit: normalizedContextLimit, + maxInputTokens: normalizedInputCap, + }; + } + if (availableOutputTokens < 1) { return { ok: false, diff --git a/open-sse/services/combo/contextOverrideGate.ts b/open-sse/services/combo/contextOverrideGate.ts index 605c27c13f..4f03978a45 100644 --- a/open-sse/services/combo/contextOverrideGate.ts +++ b/open-sse/services/combo/contextOverrideGate.ts @@ -16,14 +16,13 @@ * pool to one provider and producing a hard 503 with no fallback once that * provider's quota is exhausted. An operator-set or auto-discovered override * reflects the real capacity, so it supersedes both catalog limits. Uses the - * raw override (`getModelContextOverride` returns `null` when none is set) — + * resolved exact override (`getResolvedModelContextOverride` returns `null` when none is set) — * NOT `getModelContextLimitForModelString`, which falls back to * `contextWindow` and would therefore bypass the `maxInputTokens` cap for * every model, not just overridden ones. */ -import { getModelContextOverride } from "../../../src/lib/db/modelContextOverrides"; -import { parseModel } from "../model.ts"; +import { getResolvedModelContextOverride } from "../../../src/lib/modelCapabilities"; /** * Resolve the context-fit verdict from a persisted per-model override, if one @@ -36,8 +35,7 @@ function resolveContextOverrideVerdict( requiredContextTokens: number ): boolean | undefined { if (!modelStr) return undefined; - const parsed = parseModel(modelStr); - const override = getModelContextOverride(parsed.provider, parsed.model); + const override = getResolvedModelContextOverride(modelStr); if (override == null) return undefined; return override >= requiredContextTokens; } diff --git a/src/app/(dashboard)/dashboard/settings/components/ModelCapabilityOverridesTab.tsx b/src/app/(dashboard)/dashboard/settings/components/ModelCapabilityOverridesTab.tsx index 04d182b6b5..a4d3c2cb14 100644 --- a/src/app/(dashboard)/dashboard/settings/components/ModelCapabilityOverridesTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/ModelCapabilityOverridesTab.tsx @@ -5,7 +5,7 @@ import { useTranslations } from "next-intl"; import { Card, Button } from "@/shared/components"; import { matchesSearch } from "@/shared/utils/turkishText"; -type ModelOverrideKey = "max_token"; +type ModelOverrideKey = "context_length" | "max_input_tokens" | "max_output_tokens"; type StatusTone = "success" | "error" | "info"; type ModelOverrideTarget = { @@ -337,7 +337,7 @@ function ModelOverrideForm({ onSave: (target: string, key: ModelOverrideKey, value: number) => void; }) { const t = useTranslations("settings"); - const [key, setKey] = useState("max_token"); + const [key, setKey] = useState("context_length"); const [value, setValue] = useState(""); const numericValue = Number(value); const saveDisabled = !activeTarget || !Number.isInteger(numericValue) || numericValue <= 0; @@ -349,7 +349,9 @@ function ModelOverrideForm({ onChange={(event) => setKey(event.target.value as ModelOverrideKey)} className="sm:w-40 px-2 py-2 text-xs bg-bg-base border border-border rounded-md focus:outline-none focus:border-primary" > - + + + ; +type PublicOverride = Omit & { key: PublicOverrideKey }; + +function listPublicOverrides(): PublicOverride[] { + const capabilityOverrides = listModelCapabilityOverrides() as PublicOverride[]; + const contextOverrides = listModelContextOverrides().map((override): PublicOverride => ({ + provider: override.provider, + modelId: override.modelId, + target: `${override.provider}/${override.modelId}`, + key: "context_length", + value: override.realContext, + refreshedAt: override.refreshedAt, + })); + return [...capabilityOverrides, ...contextOverrides].sort((left, right) => + right.refreshedAt.localeCompare(left.refreshedAt) + ); +} const upsertOverrideSchema = z.object({ target: z.string().min(3), @@ -33,7 +56,7 @@ export async function GET(request: Request) { const authError = await requireManagementAuth(request); if (authError) return authError; - return NextResponse.json({ overrides: listModelCapabilityOverrides() }); + return NextResponse.json({ overrides: listPublicOverrides() }); } export async function PATCH(request: Request) { @@ -57,12 +80,20 @@ export async function PATCH(request: Request) { return NextResponse.json({ error: "Invalid model capability override" }, { status: 400 }); } - const written = setModelCapabilityOverride(target, parsed.data.key, parsed.data.value); + const targetParts = target.split(/\/(.*)/s); + const written = + parsed.data.key === "context_length" + ? setModelContextOverride(targetParts[0], targetParts[1], parsed.data.value, "manual") + : setModelCapabilityOverride( + target, + parsed.data.key as ModelCapabilityOverrideKey, + parsed.data.value + ); if (!written) { return NextResponse.json({ error: "Invalid model capability override" }, { status: 400 }); } - return NextResponse.json({ overrides: listModelCapabilityOverrides() }); + return NextResponse.json({ overrides: listPublicOverrides() }); } export async function DELETE(request: Request) { @@ -78,6 +109,11 @@ export async function DELETE(request: Request) { return NextResponse.json({ error: "target and key are required" }, { status: 400 }); } - removeModelCapabilityOverride(target, parsedKey.data as ModelCapabilityOverrideKey); - return NextResponse.json({ overrides: listModelCapabilityOverrides() }); + if (parsedKey.data === "context_length") { + const targetParts = target.split(/\/(.*)/s); + removeModelContextOverride(targetParts[0], targetParts[1]); + } else { + removeModelCapabilityOverride(target, parsedKey.data as ModelCapabilityOverrideKey); + } + return NextResponse.json({ overrides: listPublicOverrides() }); } diff --git a/src/app/api/v1/models/catalogCache.ts b/src/app/api/v1/models/catalogCache.ts index 316120293c..1cff2fa64f 100644 --- a/src/app/api/v1/models/catalogCache.ts +++ b/src/app/api/v1/models/catalogCache.ts @@ -66,7 +66,17 @@ export const CATALOG_STALE_WHILE_REVALIDATE_MS = 30_000; export const CATALOG_CACHE_TTL_MS_DEFAULT = 60_000; const catalogCache = new Map(); -const catalogInFlight = new Map>(); + +/** + * An in-flight build is bound to the catalog-state generation it started from + * (`getModelCatalogCacheVersion()` at launch). After a write invalidates the + * catalog, the generation moves on: a stale in-flight build must neither be + * joined by new requests nor repopulate the now-current cache when it finishes. + * It still resolves to its own original caller (that request legitimately waits + * on it), just without being persisted. + */ +type InFlightBuild = { generation: number; promise: Promise }; +const catalogInFlight = new Map(); let _catalogBuilderRuns = 0; @@ -90,10 +100,12 @@ function dropCatalogCacheIfStateChanged(): void { if (currentVersion === lastSeenCatalogCacheVersion) return; lastSeenCatalogCacheVersion = currentVersion; catalogCache.clear(); - // Deliberately NOT clearing catalogInFlight: an in-flight build already reads live - // DB/settings state as of when it started, so letting it finish and populate the - // (now-current) cache entry is correct — clearing it would just force a redundant - // second builder run for requests that arrive mid-flight. + // Deliberately NOT clearing catalogInFlight: an in-flight build bound to the + // previous generation is left to finish for its original caller, but the + // generation check in the join path (below) keeps new requests from joining + // it, and the generation check in storePayload keeps it from repopulating + // the now-current cache. Clearing it here would just detach the entry while + // the build still ran — wasted work with no correctness gain. } // Header sources mix Title-Case keys (diagnostic/cors headers built by app code) with @@ -116,14 +128,27 @@ export function mergeCatalogHeaders( return merged; } -function storePayload(cacheKey: string, payload: CatalogPayload): CachedCatalog { +/** + * Persist a freshly built payload — but only when the build still belongs to the + * current catalog-state generation. A build that started before a write + * invalidation (its `buildGeneration` is older than `getModelCatalogCacheVersion()`) + * returns its entry to its original caller but must NOT repopulate the cache: the + * payload reflects pre-write state and caching it would serve stale data. + */ +function storePayload( + cacheKey: string, + payload: CatalogPayload, + buildGeneration: number +): CachedCatalog { const entry: CachedCatalog = { body: payload.body, headers: payload.headers, status: payload.status, expiresAt: Date.now() + payload.cacheTTL, }; - catalogCache.set(cacheKey, entry); + if (buildGeneration === getModelCatalogCacheVersion()) { + catalogCache.set(cacheKey, entry); + } return entry; } @@ -151,10 +176,11 @@ function scheduleBackgroundRefresh( ): void { if (catalogInFlight.has(cacheKey)) return; // a refresh for this key is already running + const generation = getModelCatalogCacheVersion(); const refreshPromise: Promise = new Promise((resolve, reject) => { setTimeout(() => { runBuilder(buildPayload, request) - .then((payload) => resolve(storePayload(cacheKey, payload))) + .then((payload) => resolve(storePayload(cacheKey, payload, generation))) .catch((err) => { console.error( `[catalog] Background stale-while-revalidate refresh failed for key "${cacheKey}":`, @@ -170,11 +196,12 @@ function scheduleBackgroundRefresh( // observes the failure. refreshPromise.catch(() => {}); - catalogInFlight.set(cacheKey, refreshPromise); + catalogInFlight.set(cacheKey, { generation, promise: refreshPromise }); refreshPromise .catch(() => {}) .finally(() => { - if (catalogInFlight.get(cacheKey) === refreshPromise) catalogInFlight.delete(cacheKey); + if (catalogInFlight.get(cacheKey)?.promise === refreshPromise) + catalogInFlight.delete(cacheKey); }); } @@ -229,16 +256,24 @@ export async function resolveCachedCatalogResponse( }); } + const currentGeneration = getModelCatalogCacheVersion(); let inflight = catalogInFlight.get(cacheKey); - if (!inflight) { - inflight = runBuilder(buildPayload, request).then((payload) => storePayload(cacheKey, payload)); + // Only join an in-flight build from the CURRENT generation. A build bound to an + // older (pre-write) generation reflects stale state, so a new request starts a + // fresh build instead of joining it. + if (!inflight || inflight.generation !== currentGeneration) { + const generation = currentGeneration; + const promise = runBuilder(buildPayload, request).then((payload) => + storePayload(cacheKey, payload, generation) + ); + inflight = { generation, promise }; catalogInFlight.set(cacheKey, inflight); - inflight.finally(() => { - if (catalogInFlight.get(cacheKey) === inflight) catalogInFlight.delete(cacheKey); + promise.finally(() => { + if (catalogInFlight.get(cacheKey)?.promise === promise) catalogInFlight.delete(cacheKey); }); } - const payload = await inflight; + const payload = await inflight.promise; return new Response(payload.body, { status: payload.status, headers: mergeCatalogHeaders(corsHeaders, payload.headers, diagnosticHeaders), @@ -284,7 +319,7 @@ export function __setCatalogCacheEntryForTest(request: Request, entry: CachedCat /** Awaits any background refresh in flight, instead of guessing at a real-time sleep. */ export async function __flushCatalogBackgroundRefreshForTest(): Promise { - await Promise.all([...catalogInFlight.values()].map((p) => p.catch(() => {}))); + await Promise.all([...catalogInFlight.values()].map((entry) => entry.promise.catch(() => {}))); } /** @@ -301,5 +336,10 @@ export async function __flushCatalogBackgroundRefreshForTest(): Promise { export function __forceCatalogInFlightRejectionForTest(request: Request, error: unknown): void { const rejected: Promise = Promise.reject(error); rejected.catch(() => {}); // mark as handled — avoids an unhandledRejection warning - catalogInFlight.set(buildCatalogCacheKey(request), rejected); + // Bind to the current generation so the cold path still joins it (a stale + // generation would be skipped as pre-write state and never awaited). + catalogInFlight.set(buildCatalogCacheKey(request), { + generation: getModelCatalogCacheVersion(), + promise: rejected, + }); } diff --git a/src/lib/contextWindowResolver.ts b/src/lib/contextWindowResolver.ts index bcd6cc132f..8e7124ea85 100644 --- a/src/lib/contextWindowResolver.ts +++ b/src/lib/contextWindowResolver.ts @@ -91,8 +91,12 @@ export async function runContextWindowReconcile(): Promise { const byProvider = await getAllSyncedAvailableModels(); const discovered = toDiscoveredWindows(byProvider); return reconcileContextWindows(discovered, { + // Compare against the override-free catalog view. A persisted override must + // never feed back into the comparison that (re)writes it, or the reconciler + // oscillates (write → equal → remove → differ → write ...). getCatalogWindow: (provider, modelId) => - getResolvedModelCapabilities({ provider, model: modelId }).contextWindow, + getResolvedModelCapabilities({ provider, model: modelId }, { persistedOverrides: false }) + .contextWindow, getExistingSource: (provider, modelId) => getModelContextOverrideRecord(provider, modelId)?.source ?? null, writeAuto: (provider, modelId, window) => { diff --git a/src/lib/db/migrations/135_migrate_model_capability_max_token.sql b/src/lib/db/migrations/135_migrate_model_capability_max_token.sql new file mode 100644 index 0000000000..be8fbecfda --- /dev/null +++ b/src/lib/db/migrations/135_migrate_model_capability_max_token.sql @@ -0,0 +1,24 @@ +-- 135_migrate_model_capability_max_token.sql +-- `max_token` historically meant the maximum output token count. Promote legacy +-- rows to the explicit `max_output_tokens` key without replacing an operator's +-- existing modern value, then remove the retired key. + +INSERT INTO model_capability_overrides ( + provider, + model_id, + override_key, + override_value, + refreshed_at +) +SELECT + provider, + model_id, + 'max_output_tokens', + override_value, + refreshed_at +FROM model_capability_overrides +WHERE override_key = 'max_token' +ON CONFLICT (provider, model_id, override_key) DO NOTHING; + +DELETE FROM model_capability_overrides +WHERE override_key = 'max_token'; diff --git a/src/lib/db/modelCapabilityOverrides.ts b/src/lib/db/modelCapabilityOverrides.ts index 36ad9364e8..387f76698a 100644 --- a/src/lib/db/modelCapabilityOverrides.ts +++ b/src/lib/db/modelCapabilityOverrides.ts @@ -1,6 +1,7 @@ import { getDbInstance } from "./core"; +import { invalidateDbCache } from "./readCache"; -export type ModelCapabilityOverrideKey = "max_token"; +export type ModelCapabilityOverrideKey = "max_input_tokens" | "max_output_tokens"; export interface ModelCapabilityOverride { provider: string; @@ -20,7 +21,7 @@ interface OverrideRow { } function isSupportedKey(value: unknown): value is ModelCapabilityOverrideKey { - return value === "max_token"; + return value === "max_input_tokens" || value === "max_output_tokens"; } function isPositiveInteger(value: unknown): value is number { @@ -99,6 +100,7 @@ export function setModelCapabilityOverride( "VALUES (?, ?, ?, ?, datetime('now'))" ) .run(parsedTarget.provider, parsedTarget.modelId, key, JSON.stringify(value)); + invalidateDbCache("model-capabilities"); return true; } @@ -115,6 +117,7 @@ export function removeModelCapabilityOverride( "WHERE provider = ? AND model_id = ? AND override_key = ?" ) .run(parsedTarget.provider, parsedTarget.modelId, key); + if (info.changes > 0) invalidateDbCache("model-capabilities"); return info.changes > 0; } diff --git a/src/lib/db/modelContextOverrides.ts b/src/lib/db/modelContextOverrides.ts index 40b7326a1e..7c531cc7d3 100644 --- a/src/lib/db/modelContextOverrides.ts +++ b/src/lib/db/modelContextOverrides.ts @@ -1,4 +1,5 @@ import { getDbInstance } from "./core"; +import { invalidateDbCache } from "./readCache"; /** * Feature 5004 — self-correcting context-window overrides. @@ -36,7 +37,10 @@ function isPositiveInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value > 0; } -function normalizeKey(provider: unknown, modelId: unknown): { provider: string; modelId: string } | null { +function normalizeKey( + provider: unknown, + modelId: unknown +): { provider: string; modelId: string } | null { const p = typeof provider === "string" ? provider.trim() : ""; const m = typeof modelId === "string" ? modelId.trim() : ""; if (!p || !m) return null; @@ -104,6 +108,7 @@ export function setModelContextOverride( "VALUES (?, ?, ?, ?, datetime('now'))" ) .run(key.provider, key.modelId, realContext, normalizedSource); + invalidateDbCache("model-capabilities"); return true; } @@ -114,6 +119,7 @@ export function removeModelContextOverride(provider: string, modelId: string): b const info = getDbInstance() .prepare("DELETE FROM model_context_overrides WHERE provider = ? AND model_id = ?") .run(key.provider, key.modelId); + if (info.changes > 0) invalidateDbCache("model-capabilities"); return info.changes > 0; } diff --git a/src/lib/db/readCache.ts b/src/lib/db/readCache.ts index b8f520fe19..79fff8c2d2 100644 --- a/src/lib/db/readCache.ts +++ b/src/lib/db/readCache.ts @@ -102,9 +102,7 @@ export async function getCachedPricing(): Promise> { export async function getCachedProviderConnections( filter?: Record ): Promise { - const cacheKey = filter && Object.keys(filter).length > 0 - ? JSON.stringify(filter) - : "all"; + const cacheKey = filter && Object.keys(filter).length > 0 ? JSON.stringify(filter) : "all"; const cached = connectionsCache.get(cacheKey); if (cached) return cached; @@ -136,7 +134,10 @@ export async function getCachedRawProviderConnections( return rows; } -const connectionByIdCache = new TTLCache | null>(CONNECTIONS_TTL_MS, 10_000); +const connectionByIdCache = new TTLCache | null>( + CONNECTIONS_TTL_MS, + 10_000 +); const nodesCache = new TTLCache<(Record | null)[]>(CONNECTIONS_TTL_MS); /** @@ -263,7 +264,7 @@ export function getModelCatalogCacheVersion(): number { * cannot be selectively invalidated). */ export function invalidateDbCache( - scope?: "settings" | "pricing" | "connections" | "combos" | "nodes", + scope?: "settings" | "pricing" | "connections" | "combos" | "nodes" | "model-capabilities", id?: string ): void { if (!scope || scope === "settings") settingsCache.invalidate(); diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index 3deb72832b..c8152b18fb 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -100,6 +100,16 @@ type CapabilityInput = type SyncedCapabilities = ReturnType; +/** + * Controls whether persisted operator/discovery overrides participate in resolution. + * Omit it (the public default) to resolve effective runtime capabilities. Catalog + * reconciliation alone uses `persistedOverrides: false` to compare discovery with + * static/synced catalog data without feeding an existing override back into itself. + */ +export interface ResolveModelCapabilitiesOptions { + persistedOverrides?: boolean; +} + export interface ResolvedModelCapabilities { provider: string | null; model: string | null; @@ -494,7 +504,7 @@ function resolveVisionCapability( } /** - * Issue #6524: an operator-set `max_token` capability override (see + * Issue #6524: an operator-set `max_output_tokens` capability override (see * `src/lib/db/modelCapabilityOverrides.ts`) is the manual escape hatch for a * wrong/stale synced `limit_output` value (e.g. a provider's models.dev catalog * row reporting `limit_output` equal to `limit_context`). It already won over the @@ -502,22 +512,66 @@ function resolveVisionCapability( * makes `getExplicitModelOutputCap()` (used by the reasoning-token-buffer clamp) * consult the same override so both read paths agree. */ -function getMaxTokenCapabilityOverride(resolved: { +/** + * Exact-match capability override lookup with intentional raw-alias fallback. + * + * An override may be stored under either the canonical model id or the exact + * provider-scoped raw alias the operator used (e.g. `github/claude-opus-4.5` + * resolving to canonical `claude-opus-4-5-20251101`). We consult the canonical + * id first, then the raw alias — both are exact provider/model matches. There is + * deliberately NO suffix/effort/family inheritance: an override for + * `codex/gpt-5.6` never applies to `codex/gpt-5.6-high`. + */ +function getCapabilityOverride( + resolved: { provider: string | null; model: string | null; rawModel: string | null }, + key: "max_input_tokens" | "max_output_tokens" +): number | null { + const canonical = getModelCapabilityOverride(resolved.provider, resolved.model, key); + if (canonical !== null) return canonical; + return resolved.rawModel && resolved.rawModel !== resolved.model + ? getModelCapabilityOverride(resolved.provider, resolved.rawModel, key) + : null; +} + +function getContextOverride(resolved: { provider: string | null; model: string | null; rawModel: string | null; }): number | null { - return ( - getModelCapabilityOverride(resolved.provider, resolved.model, "max_token") ?? - (resolved.rawModel && resolved.rawModel !== resolved.model - ? getModelCapabilityOverride(resolved.provider, resolved.rawModel, "max_token") - : null) - ); + const canonical = getModelContextOverride(resolved.provider, resolved.model); + if (canonical !== null) return canonical; + return resolved.rawModel && resolved.rawModel !== resolved.model + ? getModelContextOverride(resolved.provider, resolved.rawModel) + : null; +} + +/** + * Resolve a persisted context override by canonical id, then by the exact raw + * alias supplied by the caller. Neither lookup inherits to related models. + */ +export function getResolvedModelContextOverride(input: CapabilityInput): number | null { + return getContextOverride(resolveCapabilityInput(input)); +} + +function getInputTokenCapabilityOverride(resolved: { + provider: string | null; + model: string | null; + rawModel: string | null; +}): number | null { + return getCapabilityOverride(resolved, "max_input_tokens"); +} + +function getOutputTokenCapabilityOverride(resolved: { + provider: string | null; + model: string | null; + rawModel: string | null; +}): number | null { + return getCapabilityOverride(resolved, "max_output_tokens"); } export function getExplicitModelOutputCap(input: CapabilityInput): number | null { const resolved = resolveCapabilityInput(input); - const maxTokenOverride = getMaxTokenCapabilityOverride(resolved); + const maxTokenOverride = getOutputTokenCapabilityOverride(resolved); if (maxTokenOverride !== null) return maxTokenOverride; const synced = getSyncedCapabilityForResolved( @@ -534,7 +588,13 @@ export function getExplicitModelOutputCap(input: CapabilityInput): number | null return spec?.maxOutputTokens ?? null; } -export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedModelCapabilities { +export function getResolvedModelCapabilities( + input: CapabilityInput, + options?: ResolveModelCapabilitiesOptions +): ResolvedModelCapabilities { + // Reconciliation / auto-discovery needs the override-free catalog view so a + // persisted override never feeds back into the comparison that (re)writes it. + const usePersistedOverrides = options?.persistedOverrides !== false; const resolved = resolveCapabilityInput(input); const spec = getStaticSpec(resolved.model, resolved.rawModel); const registryModel = getRegistryModel(resolved.provider, resolved.model); @@ -585,14 +645,23 @@ export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedMo resolved.model, resolved.rawModel ); + // A persisted context-window override (operator-set or auto-discovered) + // reflects the real *total* window and wins over every static/synced source. + // `maxInputTokens` still follows its own precedence chain; only when that + // chain has no narrower source does it naturally fall back to this window. + const persistedContextWindow = usePersistedOverrides ? getContextOverride(resolved) : null; const contextWindow = + persistedContextWindow ?? authoritativeContextWindow ?? synced?.limit_context ?? (typeof registryModel?.contextLength === "number" ? registryModel.contextLength : null) ?? spec?.contextWindow ?? null; - const maxTokenOverride = getMaxTokenCapabilityOverride(resolved); + const maxInputOverride = usePersistedOverrides ? getInputTokenCapabilityOverride(resolved) : null; + const maxTokenOverride = usePersistedOverrides + ? getOutputTokenCapabilityOverride(resolved) + : null; // Vision consults leaf static metadata for path-shaped ids; other capability // fields keep using the non-leaf `spec` from getStaticSpec() above. @@ -628,11 +697,22 @@ export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedMo structuredOutput: synced?.structured_output ?? null, temperature: synced?.temperature ?? null, contextWindow, - maxInputTokens: - (typeof registryModel?.maxInputTokens === "number" ? registryModel.maxInputTokens : null) ?? - authoritativeContextWindow ?? - synced?.limit_input ?? - contextWindow, + maxInputTokens: (() => { + // Input cap is input-only. An explicit `max_input_tokens` override wins; + // otherwise fall back to the existing per-source input limits, then to the + // total window. The effective cap can never exceed the total window + // (input + output), so clamp it — but never double-count a requested + // output against this input cap. + const candidate = + maxInputOverride ?? + (typeof registryModel?.maxInputTokens === "number" ? registryModel.maxInputTokens : null) ?? + authoritativeContextWindow ?? + synced?.limit_input ?? + contextWindow; + return candidate !== null && contextWindow !== null + ? Math.min(candidate, contextWindow) + : candidate; + })(), maxOutputTokens: maxTokenOverride ?? synced?.limit_output ?? @@ -657,6 +737,51 @@ export function getResolvedModelCapabilities(input: CapabilityInput): ResolvedMo }; } +/** + * Input cap enforced at the request-time hard gate, with explicit combo semantics. + * + * Feature 5004 lets a raw, exact `model_context_overrides` entry supersede a + * deliberately smaller catalog/client input hint for COMBO routing: the combo + * compatibility filter (`open-sse/services/combo/contextOverrideGate.ts`) already + * rescues such targets, so the final hard gate in handleChatCore must not turn + * around and reject the rescued target on the very hint the filter bypassed. + * + * Semantics (deliberately narrow — NO suffix/effort/family inheritance): + * - An explicit `max_input_tokens` capability override is ALWAYS enforced, for + * direct and combo requests alike. It is the operator's input-only ceiling and + * must never be bypassed by a context-window override. + * - Otherwise, for a COMBO request with an exact persisted context override, that + * context override is the input cap (it reflects the real window; the smaller + * catalog hint does not apply). Direct requests ignore this branch. + * - Otherwise the canonical `maxInputTokens` chain applies (registry input hint → + * authoritative window → synced limit_input → total window), clamped to the + * total window. + * + * `isCombo` selects the combo-rescue branch; pass `false`/omit for direct calls. + * The returned cap is still only an *input* bound — the total-window/output + * reserve check is enforced separately by `enforceOutputTokenBudget`. + */ +export function resolveInputTokenCapForGate( + input: CapabilityInput, + { isCombo = false }: { isCombo?: boolean } = {} +): number | null { + const resolved = resolveCapabilityInput(input); + + // 1. An explicit `max_input_tokens` override always wins and is never bypassed. + const explicitInputOverride = getInputTokenCapabilityOverride(resolved); + if (explicitInputOverride !== null) return explicitInputOverride; + + // 2. Combo rescue: an exact persisted context override supersedes the smaller + // catalog/client input hint (mirrors contextOverrideGate.evaluateContextLimit). + if (isCombo) { + const contextOverride = getContextOverride(resolved); + if (contextOverride !== null) return contextOverride; + } + + // 3. Canonical chain (already clamped to the total window by the resolver). + return getResolvedModelCapabilities(input).maxInputTokens; +} + export function supportsToolCalling(input: CapabilityInput): boolean { if (typeof input === "string" && !String(input || "").trim()) return false; return getResolvedModelCapabilities(input).toolCalling; @@ -732,9 +857,5 @@ export function getModelContextLimit( typeof providerOrInput === "string" && modelId !== undefined ? getResolvedModelCapabilities({ provider: providerOrInput, model: modelId }) : getResolvedModelCapabilities(providerOrInput); - // Feature 5004: a persisted override (operator-set or auto-discovered) wins over the - // static catalog / models.dev sync. `getResolvedModelCapabilities` stays override-free - // so the reconciler can compare the catalog value against provider-declared windows. - const override = getModelContextOverride(resolved.provider, resolved.model); - return override ?? resolved.contextWindow; + return resolved.contextWindow; } diff --git a/src/lib/modelMetadataRegistry.ts b/src/lib/modelMetadataRegistry.ts index e0d49968b7..72fd26c67f 100644 --- a/src/lib/modelMetadataRegistry.ts +++ b/src/lib/modelMetadataRegistry.ts @@ -2,7 +2,11 @@ import { randomUUID } from "node:crypto"; import { parseModel } from "@omniroute/open-sse/services/model.ts"; import { getModelInfo } from "@/sse/services/model"; import { getModelAliases } from "@/lib/db/models"; -import { getResolvedModelCapabilities, isNonChatCatalogSurface } from "@/lib/modelCapabilities"; +import { + getResolvedModelCapabilities, + getResolvedModelContextOverride, + isNonChatCatalogSurface, +} from "@/lib/modelCapabilities"; import { getAuthoritativeContextWindow, getAuthoritativeProviderContextWindow, @@ -418,6 +422,7 @@ export function enrichCatalogModelEntry( getAuthoritativeContextWindow(metadata.model) ?? getAuthoritativeContextWindow(model); const specialtySurface = isNonChatCatalogSurface(entry.type); + const persistedContextWindow = getResolvedModelContextOverride({ provider, model }); const capabilityFields = { ...(typeof metadata.capabilities.vision === "boolean" ? { vision: metadata.capabilities.vision } @@ -486,16 +491,21 @@ export function enrichCatalogModelEntry( if ( !specialtySurface && - (typeof nextEntry.context_length !== "number" || authoritativeContextWindow !== null) && + (typeof nextEntry.context_length !== "number" || + authoritativeContextWindow !== null || + persistedContextWindow !== null) && typeof metadata.limits.contextWindow === "number" ) { nextEntry.context_length = metadata.limits.contextWindow; + } else if (specialtySurface && persistedContextWindow !== null) { + // Exact persisted overrides are authoritative for every surface of the model. + nextEntry.context_length = persistedContextWindow; } else if ( specialtySurface && authoritativeContextWindow !== null && typeof authoritativeContextWindow === "number" ) { - // Only authoritative static windows may decorate specialty rows. + // Only authoritative static windows may otherwise decorate specialty rows. nextEntry.context_length = authoritativeContextWindow; } else if (specialtySurface && typeof nextEntry.context_length === "number") { // Keep an explicit source-provided context if the emitter already set one. diff --git a/tests/unit/chatcore-combo-context-override-rescue.test.ts b/tests/unit/chatcore-combo-context-override-rescue.test.ts new file mode 100644 index 0000000000..67b716ed3c --- /dev/null +++ b/tests/unit/chatcore-combo-context-override-rescue.test.ts @@ -0,0 +1,137 @@ +// Black-box regression for the Feature-5004 combo context-override rescue at the +// handleChatCore final input gate. +// +// The combo compatibility filter (open-sse/services/combo/contextOverrideGate.ts) +// deliberately lets a raw, exact `model_context_overrides` entry supersede a smaller +// catalog/client input hint so a capable fallback target is not wrongly dropped for a +// large prompt. But handleChatCore then enforced the canonical `maxInputTokens` again +// at the final hard boundary — so a target the filter just rescued was rejected before +// the upstream fetch (#5004 combo rescue defeated at the gate). +// +// Fix: the final gate resolves the input cap through `resolveInputTokenCapForGate`, +// which mirrors the filter's semantics — an exact context override supersedes the +// smaller catalog input hint for combo requests, while an explicit `max_input_tokens` +// capability override is ALWAYS enforced (combo and direct alike). No inheritance is +// generalized. +// +// GitHub's Claude alias canonicalizes before capability lookup, so a prompt sized +// between the catalog input hint and an exact raw-alias context override exercises +// the rescue the filter performs. These tests drive the public handleChatCore with a +// temp DB and an upstream fetch stub and assert on the +// dispatch signal (did the upstream fetch fire, and was the rejection the 400 +// input-cap error) — the precise contract of the gate. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-combo-ctx-rescue-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts"); +const capabilityOverrides = await import("../../src/lib/db/modelCapabilityOverrides.ts"); +const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); + +const PROVIDER = "github"; +const MODEL = "claude-opus-4.5"; +const CONTEXT_OVERRIDE = 1_000_000; +// ~3.7M chars ≈ 925k estimated tokens — above the catalog input hint, below the override. +const BIG_PROMPT = "x".repeat(3_700_000); + +const originalFetch = globalThis.fetch; +let fetchCalls = 0; +const silentLog = { debug() {}, info() {}, warn() {}, error() {} }; + +function buildRequest(isCombo: boolean, content: string) { + const body = { + model: MODEL, + messages: [{ role: "user", content }], + max_tokens: 10, + stream: false, + }; + return { + body, + modelInfo: { provider: PROVIDER, model: MODEL, extendedContext: false }, + credentials: { + apiKey: "sk-test", + providerSpecificData: { baseUrl: "https://combo-ctx-rescue.example.test" }, + }, + clientRawRequest: { + endpoint: "/v1/chat/completions", + body, + headers: new Headers({ accept: "application/json" }), + }, + userAgent: "unit-test", + isCombo, + log: silentLog, + }; +} + +test.before(() => { + core.resetDbInstance(); + globalThis.fetch = async () => { + fetchCalls += 1; + return new Response( + JSON.stringify({ + id: "chatcmpl-combo-ctx", + choices: [ + { index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; +}); + +test.after(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test.beforeEach(() => { + fetchCalls = 0; + // Isolate each scenario: clear any overrides a prior test wrote so ordering does + // not couple the assertions. + contextOverrides.removeModelContextOverride(PROVIDER, MODEL); + capabilityOverrides.removeModelCapabilityOverride(`${PROVIDER}/${MODEL}`, "max_input_tokens"); +}); + +test("raw-alias combo request rescued by an exact context override dispatches", async () => { + assert.equal(contextOverrides.setModelContextOverride(PROVIDER, MODEL, CONTEXT_OVERRIDE), true); + // The prompt exceeds the catalog input hint but fits the context override. A combo + // request must NOT be rejected at the gate — the upstream fetch must fire. + const result = await handleChatCore(buildRequest(true, BIG_PROMPT)); + assert.ok(fetchCalls > 0, "rescued combo target must reach the upstream fetch"); + assert.notEqual( + result.status, + 400, + "rescued combo target must not be rejected with the input-cap 400" + ); +}); + +test("an exact raw-alias max_input_tokens override is enforced even in a combo", async () => { + // An explicit max_input_tokens capability override must NEVER be bypassed by a + // context override — combo included. + assert.equal(contextOverrides.setModelContextOverride(PROVIDER, MODEL, CONTEXT_OVERRIDE), true); + assert.equal( + capabilityOverrides.setModelCapabilityOverride( + `${PROVIDER}/${MODEL}`, + "max_input_tokens", + 1000 + ), + true + ); + const result = await handleChatCore(buildRequest(true, BIG_PROMPT)); + assert.equal(result.status, 400); + assert.equal( + fetchCalls, + 0, + "explicit max_input override must reject with zero fetch even in combo" + ); + assert.match(JSON.stringify(result), /maximum input tokens/i); +}); diff --git a/tests/unit/chatcore-model-output-cap-wiring.test.ts b/tests/unit/chatcore-model-output-cap-wiring.test.ts index f50caa3958..0331aeda67 100644 --- a/tests/unit/chatcore-model-output-cap-wiring.test.ts +++ b/tests/unit/chatcore-model-output-cap-wiring.test.ts @@ -1,46 +1,33 @@ -// Wiring guard for the model-output-cap clamp: output-token-budget-model-cap.test.ts -// exercises enforceOutputTokenBudget() directly and therefore cannot catch a -// callsite regression — drop the cap argument in handleChatCore and every one of -// those unit tests still passes. This test drives handleChatCore() end to end -// (stubbed fetch, temp DB) and asserts the body actually dispatched upstream. -// -// The cap is supplied through an operator `max_token` capability override -// (src/lib/db/modelCapabilityOverrides.ts, issue #6524) rather than a catalog -// model, which pins two things at once and keeps the test independent of -// provider-catalog drift: -// 1. the clamp runs on the single-model (non-combo) path; -// 2. the cap lookup is keyed by { provider, model } — the bare-string form -// resolves to provider: null, and the override table is keyed by provider, -// so a string-keyed lookup silently misses it and no clamp happens. +// Direct output-budget tests cannot prove handleChatCore passes capability caps +// through its non-combo runtime path. This drives that public handler with a +// temporary DB and upstream fetch stub. import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-output-cap-")); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-model-cap-wiring-")); process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const overridesDb = await import("../../src/lib/db/modelCapabilityOverrides.ts"); const { handleChatCore } = await import("../../open-sse/handlers/chatCore.ts"); -// Distinctive enough that it can never collide with a provider registered in -// open-sse/config/providerRegistry.ts, so nothing but the override supplies a cap. const PROVIDER = "capwire-testprov"; const MODEL = "capwire-testmodel"; const OUTPUT_CAP = 1000; +const INPUT_CAP = 10; const REQUESTED_MAX_TOKENS = 50_000; - const originalFetch = globalThis.fetch; let dispatchedBody: Record | null = null; - +let fetchCalls = 0; const silentLog = { debug() {}, info() {}, warn() {}, error() {} }; -function buildRequest(maxTokens: number) { +function buildRequest(maxTokens: number, content = "hello") { const body = { model: MODEL, - messages: [{ role: "user", content: "hello" }], + messages: [{ role: "user", content }], max_tokens: maxTokens, stream: false, }; @@ -65,12 +52,15 @@ function buildRequest(maxTokens: number) { test.before(() => { core.resetDbInstance(); assert.equal( - overridesDb.setModelCapabilityOverride(`${PROVIDER}/${MODEL}`, "max_token", OUTPUT_CAP), - true, - "the operator override must be persisted for this test to mean anything" + overridesDb.setModelCapabilityOverride(`${PROVIDER}/${MODEL}`, "max_output_tokens", OUTPUT_CAP), + true + ); + assert.equal( + overridesDb.setModelCapabilityOverride(`${PROVIDER}/${MODEL}`, "max_input_tokens", INPUT_CAP), + true ); - globalThis.fetch = async (_input: RequestInfo | URL, init?: RequestInit) => { + fetchCalls += 1; dispatchedBody = init?.body ? JSON.parse(String(init.body)) : null; return new Response( JSON.stringify({ @@ -93,25 +83,34 @@ test.after(() => { test("handleChatCore clamps an over-cap max_tokens to the model's output cap before dispatch", async () => { dispatchedBody = null; + fetchCalls = 0; await handleChatCore(buildRequest(REQUESTED_MAX_TOKENS)); - - assert.ok(dispatchedBody, "expected the request to reach the upstream fetch"); - assert.equal( - dispatchedBody?.max_tokens, - OUTPUT_CAP, - `expected max_tokens clamped to the ${OUTPUT_CAP}-token operator cap, got ${dispatchedBody?.max_tokens}` - ); + assert.equal(fetchCalls, 1); + assert.equal(dispatchedBody?.max_tokens, OUTPUT_CAP); }); test("handleChatCore leaves a max_tokens below the cap untouched", async () => { dispatchedBody = null; + fetchCalls = 0; const underCap = OUTPUT_CAP - 1; await handleChatCore(buildRequest(underCap)); + assert.equal(fetchCalls, 1); + assert.equal(dispatchedBody?.max_tokens, underCap); +}); - assert.ok(dispatchedBody, "expected the request to reach the upstream fetch"); - assert.equal( - dispatchedBody?.max_tokens, - underCap, - "a request below the cap must never be raised to it" - ); +test("handleChatCore rejects input over the model input cap before upstream dispatch", async () => { + dispatchedBody = null; + fetchCalls = 0; + const result = await handleChatCore(buildRequest(1, "x".repeat(200))); + assert.equal(result.status, 400); + assert.equal(fetchCalls, 0, "input-cap rejection must not call the upstream"); + assert.match(JSON.stringify(result), /maximum input tokens/i); +}); + +test("handleChatCore dispatches input within the model input cap", async () => { + dispatchedBody = null; + fetchCalls = 0; + await handleChatCore(buildRequest(1, "x")); + assert.equal(fetchCalls, 1); + assert.ok(dispatchedBody, "input below the cap must reach the upstream"); }); diff --git a/tests/unit/context-window-reconcile-persisted-overrides.test.ts b/tests/unit/context-window-reconcile-persisted-overrides.test.ts new file mode 100644 index 0000000000..13285d398f --- /dev/null +++ b/tests/unit/context-window-reconcile-persisted-overrides.test.ts @@ -0,0 +1,55 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync( + path.join(os.tmpdir(), "omniroute-context-reconcile-runtime-") +); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const models = await import("../../src/lib/db/models.ts"); +const overrides = await import("../../src/lib/db/modelContextOverrides.ts"); +const { runContextWindowReconcile } = await import("../../src/lib/contextWindowResolver.ts"); + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("runContextWindowReconcile retains an auto override across repeated synced discovery", async () => { + // gpt-4o's static catalog window is 128K; discovery reports a real 372K. + // This uses the live DB discovery and resolver seams, not injected pure deps. + await models.replaceSyncedAvailableModelsForConnection("openai", "reconcile-test", [ + { id: "gpt-4o", inputTokenLimit: 372000 }, + ]); + + const first = await runContextWindowReconcile(); + assert.deepEqual(first, { scanned: 1, written: 1, removed: 0, skippedManual: 0 }); + assert.deepEqual( + (() => { + const record = overrides.getModelContextOverrideRecord("openai", "gpt-4o"); + return record && { realContext: record.realContext, source: record.source }; + })(), + { realContext: 372000, source: "auto:discovery" } + ); + + const second = await runContextWindowReconcile(); + assert.deepEqual(second, { scanned: 1, written: 1, removed: 0, skippedManual: 0 }); + assert.deepEqual( + (() => { + const record = overrides.getModelContextOverrideRecord("openai", "gpt-4o"); + return record && { realContext: record.realContext, source: record.source }; + })(), + { realContext: 372000, source: "auto:discovery" }, + "the persisted auto override remains rather than being removed after its first write" + ); +}); diff --git a/tests/unit/model-capability-overrides.test.ts b/tests/unit/model-capability-overrides.test.ts index 2857327e8f..788b48c9cf 100644 --- a/tests/unit/model-capability-overrides.test.ts +++ b/tests/unit/model-capability-overrides.test.ts @@ -10,6 +10,8 @@ process.env.DATA_DIR = moduleDataDir; const coreDb = await import("../../src/lib/db/core.ts"); const caps = await import("../../src/lib/modelCapabilities.ts"); const overrides = await import("../../src/lib/db/modelCapabilityOverrides.ts"); +const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts"); +const route = await import("../../src/app/api/model-capability-overrides/route.ts"); beforeEach(() => { coreDb.resetDbInstance(); @@ -23,8 +25,18 @@ after(() => { fs.rmSync(moduleDataDir, { recursive: true, force: true }); }); +function patchOverride(key: string, value: unknown) { + return route.PATCH( + new Request("http://localhost/api/model-capability-overrides", { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ target: "codex/gpt-5.6", key, value }), + }) + ); +} + describe("model capability overrides", () => { - it("stores, lists, removes, and applies a provider/model max_token override", () => { + it("stores, lists, removes, and applies an exact max_output_tokens override", () => { const withoutOverride = caps.getResolvedModelCapabilities({ provider: "openai", model: "gpt-4o", @@ -32,7 +44,7 @@ describe("model capability overrides", () => { const distinct = (withoutOverride ?? 0) + 12345; assert.equal( - overrides.setModelCapabilityOverride("openai/gpt-4o", "max_token", distinct), + overrides.setModelCapabilityOverride("openai/gpt-4o", "max_output_tokens", distinct), true ); assert.deepEqual( @@ -41,32 +53,56 @@ describe("model capability overrides", () => { key: entry.key, value: entry.value, })), - [{ target: "openai/gpt-4o", key: "max_token", value: distinct }] + [{ target: "openai/gpt-4o", key: "max_output_tokens", value: distinct }] ); - assert.equal( caps.getResolvedModelCapabilities({ provider: "openai", model: "gpt-4o" }).maxOutputTokens, distinct ); assert.notEqual( caps.getResolvedModelCapabilities({ provider: "anthropic", model: "gpt-4o" }).maxOutputTokens, - distinct, - "override must be scoped by provider/model, not bare model id" + distinct + ); + assert.equal( + overrides.removeModelCapabilityOverride("openai/gpt-4o", "max_output_tokens"), + true ); - - assert.equal(overrides.removeModelCapabilityOverride("openai/gpt-4o", "max_token"), true); assert.equal( caps.getResolvedModelCapabilities({ provider: "openai", model: "gpt-4o" }).maxOutputTokens, withoutOverride ); }); + it("uses exact input/output overrides, clamps input to context, and isolates effort variants", () => { + const target = "codex/gpt-5.6"; + const variant = "codex/gpt-5.6-high"; + assert.equal(contextOverrides.setModelContextOverride("codex", "gpt-5.6", 372000), true); + assert.equal(overrides.setModelCapabilityOverride(target, "max_input_tokens", 999999), true); + assert.equal(overrides.setModelCapabilityOverride(target, "max_output_tokens", 123456), true); + + const base = caps.getResolvedModelCapabilities(target); + assert.deepEqual( + { + contextWindow: base.contextWindow, + maxInputTokens: base.maxInputTokens, + maxOutputTokens: base.maxOutputTokens, + }, + { contextWindow: 372000, maxInputTokens: 372000, maxOutputTokens: 123456 } + ); + assert.equal(overrides.removeModelCapabilityOverride(target, "max_output_tokens"), true); + assert.notEqual(caps.getResolvedModelCapabilities(target).maxOutputTokens, 123456); + + const effort = caps.getResolvedModelCapabilities(variant); + assert.notEqual(effort.contextWindow, 372000); + assert.notEqual(effort.maxInputTokens, 372000); + assert.notEqual(effort.maxOutputTokens, 123456); + }); + it("applies overrides stored under provider-scoped model aliases", () => { assert.equal( - overrides.setModelCapabilityOverride("github/claude-opus-4.5", "max_token", 77777), + overrides.setModelCapabilityOverride("github/claude-opus-4.5", "max_output_tokens", 77777), true ); - assert.equal( caps.getResolvedModelCapabilities({ provider: "github", model: "claude-opus-4.5" }) .maxOutputTokens, @@ -74,10 +110,143 @@ describe("model capability overrides", () => { ); }); + it("accepts exactly the three public token-limit keys through the API", async () => { + assert.equal( + contextOverrides.setModelContextOverride("codex", "gpt-5.6", 272000, "auto:discovery"), + true + ); + const discoveredResponse = await route.GET( + new Request("http://localhost/api/model-capability-overrides") + ); + const discoveredPayload = (await discoveredResponse.json()) as { + overrides: Array<{ target: string; key: string; value: number }>; + }; + assert.ok( + discoveredPayload.overrides.some( + (override) => + override.target === "codex/gpt-5.6" && + override.key === "context_length" && + override.value === 272000 + ), + "the unified surface exposes an auto-discovered context window before manual replacement" + ); + + assert.equal((await patchOverride("context_length", 372000)).status, 200); + assert.equal((await patchOverride("max_input_tokens", 353400)).status, 200); + assert.equal((await patchOverride("max_output_tokens", 128000)).status, 200); + assert.equal((await patchOverride("max_token", 77777)).status, 400, "legacy key"); + assert.equal((await patchOverride("unknown", 1)).status, 400, "unsupported key"); + assert.equal((await patchOverride("max_input_tokens", 0)).status, 400, "non-positive integer"); + assert.equal( + (await patchOverride("max_input_tokens", Number.POSITIVE_INFINITY)).status, + 400, + "JSON serializes Infinity as null; route rejects the resulting non-number" + ); + + const response = await route.GET( + new Request("http://localhost/api/model-capability-overrides") + ); + const payload = (await response.json()) as { + overrides: Array<{ target: string; key: string; value: number }>; + }; + assert.deepEqual( + payload.overrides + .map(({ target, key, value }) => ({ target, key, value })) + .sort((left, right) => left.key.localeCompare(right.key)), + [ + { target: "codex/gpt-5.6", key: "context_length", value: 372000 }, + { target: "codex/gpt-5.6", key: "max_input_tokens", value: 353400 }, + { target: "codex/gpt-5.6", key: "max_output_tokens", value: 128000 }, + ] + ); + + assert.equal( + contextOverrides.getModelContextOverrideRecord("codex", "gpt-5.6")?.source, + "manual" + ); + + const resolved = caps.getResolvedModelCapabilities("codex/gpt-5.6"); + assert.deepEqual( + { + contextWindow: resolved.contextWindow, + maxInputTokens: resolved.maxInputTokens, + maxOutputTokens: resolved.maxOutputTokens, + }, + { contextWindow: 372000, maxInputTokens: 353400, maxOutputTokens: 128000 } + ); + + const removed = await route.DELETE( + new Request( + "http://localhost/api/model-capability-overrides?target=codex/gpt-5.6&key=context_length", + { method: "DELETE" } + ) + ); + assert.equal(removed.status, 200); + assert.equal(contextOverrides.getModelContextOverride("codex", "gpt-5.6"), null); + + const rejectedDelete = await route.DELETE( + new Request( + "http://localhost/api/model-capability-overrides?target=codex/gpt-5.6&key=max_token", + { method: "DELETE" } + ) + ); + assert.equal(rejectedDelete.status, 400); + }); + it("rejects invalid targets and non-positive values", () => { - assert.equal(overrides.setModelCapabilityOverride("gpt-4o", "max_token", 1000), false); - assert.equal(overrides.setModelCapabilityOverride("openai/gpt-4o", "max_token", 0), false); - assert.equal(overrides.setModelCapabilityOverride("openai/gpt-4o", "max_token", 1.5), false); + assert.equal(overrides.setModelCapabilityOverride("gpt-4o", "max_output_tokens", 1000), false); + assert.equal( + overrides.setModelCapabilityOverride("openai/gpt-4o", "max_output_tokens", 0), + false + ); + assert.equal( + overrides.setModelCapabilityOverride("openai/gpt-4o", "max_output_tokens", 1.5), + false + ); assert.deepEqual(overrides.listModelCapabilityOverrides(), []); }); + + it("migrates legacy max_token rows without replacing a modern output override", () => { + const db = coreDb.getDbInstance(); + const insert = db.prepare( + "INSERT INTO model_capability_overrides " + + "(provider, model_id, override_key, override_value, refreshed_at) VALUES (?, ?, ?, ?, ?)" + ); + insert.run("legacy", "legacy-only", "max_token", "64000", "2026-01-01 00:00:00"); + insert.run("legacy", "collision", "max_token", "64000", "2026-01-01 00:00:00"); + insert.run("legacy", "collision", "max_output_tokens", "128000", "2026-02-01 00:00:00"); + + const migration = fs.readFileSync( + path.resolve("src/lib/db/migrations/135_migrate_model_capability_max_token.sql"), + "utf8" + ); + db.exec(migration); + db.exec(migration); + + assert.deepEqual( + db + .prepare( + "SELECT provider, model_id, override_key, override_value, refreshed_at " + + "FROM model_capability_overrides WHERE provider = 'legacy' " + + "ORDER BY model_id, override_key" + ) + .all(), + [ + { + provider: "legacy", + model_id: "collision", + override_key: "max_output_tokens", + override_value: "128000", + refreshed_at: "2026-02-01 00:00:00", + }, + { + provider: "legacy", + model_id: "legacy-only", + override_key: "max_output_tokens", + override_value: "64000", + refreshed_at: "2026-01-01 00:00:00", + }, + ] + ); + }); }); diff --git a/tests/unit/model-context-override-readpath.test.ts b/tests/unit/model-context-override-readpath.test.ts index e9aef36538..9cd6a6ea3e 100644 --- a/tests/unit/model-context-override-readpath.test.ts +++ b/tests/unit/model-context-override-readpath.test.ts @@ -28,8 +28,10 @@ after(() => { describe("getModelContextLimit override precedence (5004)", () => { it("an override wins over the catalog, and removing it falls back to the catalog", () => { // Read the override-free catalog value dynamically (non-brittle for any model). - const catalog = caps.getResolvedModelCapabilities({ provider: "openai", model: "gpt-4o" }) - .contextWindow; + const catalog = caps.getResolvedModelCapabilities({ + provider: "openai", + model: "gpt-4o", + }).contextWindow; const distinct = (catalog ?? 0) + 12345; mco.setModelContextOverride("openai", "gpt-4o", distinct); @@ -49,11 +51,61 @@ describe("getModelContextLimit override precedence (5004)", () => { assert.equal(caps.getModelContextLimit("custom-local", "my-7b-128k"), 131072); }); - it("leaves getResolvedModelCapabilities override-free (so the reconciler sees the catalog)", () => { + it("resolves exact raw aliases after canonical rows, without effort inheritance", () => { + const provider = "github"; + const rawAlias = "claude-opus-4.5"; + const canonical = "claude-opus-4-5-20251101"; + + assert.equal(mco.setModelContextOverride(provider, rawAlias, 333333), true); + assert.equal( + caps.getResolvedModelCapabilities({ provider, model: rawAlias }).contextWindow, + 333333, + "an exact raw alias override must be effective" + ); + assert.notEqual( + caps.getResolvedModelCapabilities( + { provider, model: rawAlias }, + { persistedOverrides: false } + ).contextWindow, + 333333, + "override-free resolution must not read the raw alias row" + ); + + assert.equal(mco.setModelContextOverride(provider, canonical, 444444), true); + assert.equal( + caps.getResolvedModelCapabilities({ provider, model: rawAlias }).contextWindow, + 444444, + "the canonical row must win over the exact raw alias row" + ); + assert.notEqual( + caps.getResolvedModelCapabilities({ provider, model: `${rawAlias}-high` }).contextWindow, + 444444, + "an exact alias override must not inherit to an effort variant" + ); + }); + + it("default getResolvedModelCapabilities reflects the override; persistedOverrides:false returns the catalog", () => { mco.setModelContextOverride("openai", "gpt-4o", 999999, "auto:discovery"); - const catalog = caps.getResolvedModelCapabilities({ provider: "openai", model: "gpt-4o" }) - .contextWindow; - assert.notEqual(catalog, 999999, "getResolvedModelCapabilities must not reflect the override"); - assert.equal(caps.getModelContextLimit("openai", "gpt-4o"), 999999, "but getModelContextLimit does"); + // Default resolution is the effective runtime view: the persisted context + // override wins over the catalog. + const effective = caps.getResolvedModelCapabilities({ + provider: "openai", + model: "gpt-4o", + }).contextWindow; + assert.equal(effective, 999999, "default resolution must reflect the persisted override"); + + // The override-free catalog view (used by the reconciler) excludes the override. + const catalog = caps.getResolvedModelCapabilities( + { provider: "openai", model: "gpt-4o" }, + { persistedOverrides: false } + ).contextWindow; + assert.notEqual(catalog, 999999, "persistedOverrides:false must return the catalog value"); + + // getModelContextLimit always follows the effective override. + assert.equal( + caps.getModelContextLimit("openai", "gpt-4o"), + 999999, + "getModelContextLimit reflects the override" + ); }); }); diff --git a/tests/unit/model-token-limit-catalog.test.ts b/tests/unit/model-token-limit-catalog.test.ts new file mode 100644 index 0000000000..b92649a01d --- /dev/null +++ b/tests/unit/model-token-limit-catalog.test.ts @@ -0,0 +1,164 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-token-limit-catalog-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const contextOverrides = await import("../../src/lib/db/modelContextOverrides.ts"); +const capabilityOverrides = await import("../../src/lib/db/modelCapabilityOverrides.ts"); +const models = await import("../../src/lib/db/models.ts"); +const providers = await import("../../src/lib/db/providers.ts"); +const catalog = await import("../../src/app/api/v1/models/catalog.ts"); + +const TARGET = "openai/gpt-5.6"; +const LIMITS = { context: 372000, input: 353400, output: 128000 }; + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + catalog.__resetCatalogBuilderRunsForTest(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +async function getModel(target = TARGET) { + const response = await catalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as { data: Array> }; + return body.data.find((model) => model.id === target); +} + +test("v1 model catalog projects effective context, input, and output overrides and invalidates on delete", async () => { + assert.equal(contextOverrides.setModelContextOverride("openai", "gpt-5.6", LIMITS.context), true); + assert.equal( + capabilityOverrides.setModelCapabilityOverride(TARGET, "max_input_tokens", LIMITS.input), + true + ); + assert.equal( + capabilityOverrides.setModelCapabilityOverride(TARGET, "max_output_tokens", LIMITS.output), + true + ); + await providers.createProviderConnection({ + provider: "openai", + authType: "api_key", + name: "token-limit-catalog", + apiKey: "sk-test", + }); + + const initial = await getModel(); + assert.ok(initial); + assert.deepEqual( + { + context_length: initial.context_length, + max_input_tokens: initial.max_input_tokens, + max_output_tokens: initial.max_output_tokens, + }, + { + context_length: LIMITS.context, + max_input_tokens: LIMITS.input, + max_output_tokens: LIMITS.output, + } + ); + + const sentinelOutput = 111111; + assert.equal( + capabilityOverrides.setModelCapabilityOverride(TARGET, "max_output_tokens", sentinelOutput), + true + ); + assert.equal((await getModel())?.max_output_tokens, sentinelOutput); + assert.equal( + capabilityOverrides.removeModelCapabilityOverride(TARGET, "max_output_tokens"), + true + ); + assert.notEqual((await getModel())?.max_output_tokens, sentinelOutput); +}); + +test("v1 model catalog projects a synced Codex context to both public aliases", async () => { + const target = "codex/gpt-5.6-sol"; + const connection = await providers.createProviderConnection({ + provider: "codex", + authType: "oauth", + name: "codex-token-limit-catalog", + accessToken: "test-access-token", + }); + assert.equal(typeof connection.id, "string"); + await models.replaceSyncedAvailableModelsForConnection("codex", connection.id as string, [ + { + id: "gpt-5.6-sol", + name: "GPT 5.6 Sol", + source: "provider", + supportedEndpoints: ["responses"], + inputTokenLimit: 272000, + outputTokenLimit: 128000, + }, + ]); + assert.equal( + contextOverrides.setModelContextOverride("codex", "gpt-5.6-sol", LIMITS.context), + true + ); + assert.equal( + capabilityOverrides.setModelCapabilityOverride(target, "max_input_tokens", LIMITS.input), + true + ); + assert.equal( + capabilityOverrides.setModelCapabilityOverride(target, "max_output_tokens", LIMITS.output), + true + ); + + for (const publicId of ["cx/gpt-5.6-sol", "codex/gpt-5.6-sol"]) { + const projected = await getModel(publicId); + assert.ok(projected, `expected ${publicId} in the public catalog`); + assert.deepEqual( + { + context_length: projected.context_length, + max_input_tokens: projected.max_input_tokens, + max_output_tokens: projected.max_output_tokens, + }, + { + context_length: LIMITS.context, + max_input_tokens: LIMITS.input, + max_output_tokens: LIMITS.output, + }, + `${publicId} must retain the persisted Codex token-limit overrides` + ); + } + + const canonical = await getModel("codex/gpt-5.6-sol"); + assert.equal(canonical?.type, "image"); + assert.deepEqual(canonical?.output_modalities, ["image"]); + assert.ok(Array.isArray(canonical?.supported_sizes)); + + assert.equal(contextOverrides.removeModelContextOverride("codex", "gpt-5.6-sol"), true); + assert.equal( + (await getModel("codex/gpt-5.6-sol"))?.context_length, + undefined, + "the specialty row must not inherit the synced chat context after override removal" + ); + assert.equal((await getModel("cx/gpt-5.6-sol"))?.context_length, 272000); +}); + +test("v1 model catalog projects an exact raw-alias context override", async () => { + const target = "github/claude-opus-4.5"; + assert.equal(contextOverrides.setModelContextOverride("github", "claude-opus-4.5", 333333), true); + await providers.createProviderConnection({ + provider: "github", + authType: "api_key", + name: "raw-alias-token-limit-catalog", + apiKey: "ghp-test", + }); + + assert.equal( + (await getModel(target))?.context_length, + 333333, + "the catalog entry keeps its raw alias and must project that exact override" + ); +}); diff --git a/tests/unit/output-token-budget-model-cap.test.ts b/tests/unit/output-token-budget-model-cap.test.ts index dd7dc42b17..4015f90189 100644 --- a/tests/unit/output-token-budget-model-cap.test.ts +++ b/tests/unit/output-token-budget-model-cap.test.ts @@ -118,3 +118,73 @@ test("a sub-token cap is treated as absent, never as a cap of zero", () => { assert.equal(result.body.max_tokens, 8_000, "sub-token cap must leave the request untouched"); assert.deepEqual(result.adjustedFields, []); }); + +test("rejects when estimated input exceeds the model input cap, even with output room", () => { + // Input cap is input-only and independent of the (larger) total window. + const result = enforceOutputTokenBudget( + { max_tokens: 1_000 }, + 354_000, + 372_000, + 0, + null, + 353_400 + ); + + assert.equal(result.ok, false); + if (result.ok) return; + assert.equal(result.estimatedInputTokens, 354_000); + assert.equal(result.maxInputTokens, 353_400); +}); + +test("accepts input at the model input cap without double-counting requested output", () => { + // estimatedInput == input cap; a large requested output must not be re-counted + // against the input-only cap (only the total window bounds input + output). + const result = enforceOutputTokenBudget( + { max_tokens: 128_000 }, + 353_400, + 372_000, + 0, + null, + 353_400 + ); + + assert.equal(result.ok, true); + if (!result.ok) return; + // Output clamped to remaining window room (372000 - 353400 = 18600). + assert.equal(result.body.max_tokens, 18_600); +}); + +test("input cap is fail-open when absent, null, or non-positive", () => { + const base = enforceOutputTokenBudget({ max_tokens: 1_000 }, 100_000, 200_000, 0, null); + const undefinedCap = enforceOutputTokenBudget( + { max_tokens: 1_000 }, + 100_000, + 200_000, + 0, + null, + undefined + ); + const nullCap = enforceOutputTokenBudget({ max_tokens: 1_000 }, 100_000, 200_000, 0, null, null); + const zeroCap = enforceOutputTokenBudget({ max_tokens: 1_000 }, 100_000, 200_000, 0, null, 0); + + assert.deepEqual(undefinedCap, base); + assert.deepEqual(nullCap, base); + assert.deepEqual(zeroCap, base); + assert.equal(base.ok, true); +}); + +test("the total window still rejects when input fits the input cap but no output room remains", () => { + // Input (150k) fits a 353k input cap but leaves no room in the 128k window. + const result = enforceOutputTokenBudget( + { max_tokens: 1_000 }, + 150_000, + 128_000, + 0, + null, + 353_400 + ); + + assert.equal(result.ok, false); + if (result.ok) return; + assert.equal(result.maxInputTokens, undefined, "window rejection, not input-cap rejection"); +}); diff --git a/tests/unit/repro-6524.test.ts b/tests/unit/repro-6524.test.ts index 1d4ac3a626..d99dcec855 100644 --- a/tests/unit/repro-6524.test.ts +++ b/tests/unit/repro-6524.test.ts @@ -11,14 +11,14 @@ * `getExplicitModelOutputCap` clamp source): the clamp math itself is correct, but * `getExplicitModelOutputCap()` only ever read the unvalidated synced * `limit_output` (or registry/static fallbacks) — it ignored the operator-settable - * `max_token` capability override (`src/lib/db/modelCapabilityOverrides.ts`, + * `max_output_tokens` capability override (`src/lib/db/modelCapabilityOverrides.ts`, * `/api/model-capability-overrides`) that `getResolvedModelCapabilities()` already * consulted. That inconsistency meant an operator manually correcting a bad synced * output cap (the existing, already-shipped remediation path for wrong catalog * data) had no effect on the reasoning buffer, which kept inflating past the real * cap regardless. * - * Fix: `getExplicitModelOutputCap()` now checks the same `max_token` override + * Fix: `getExplicitModelOutputCap()` now checks the same `max_output_tokens` override * before falling back to synced/registry/static data, via a helper shared with * `getResolvedModelCapabilities()`. */ @@ -32,15 +32,12 @@ const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-repro-652 process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); -const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import( - "../../src/lib/modelsDevSync.ts" -); -const { setModelCapabilityOverride, removeModelCapabilityOverride } = await import( - "../../src/lib/db/modelCapabilityOverrides.ts" -); -const { resolveReasoningBufferedMaxTokens } = await import( - "../../open-sse/services/reasoningTokenBuffer.ts" -); +const { saveModelsDevCapabilities, clearModelsDevCapabilities } = + await import("../../src/lib/modelsDevSync.ts"); +const { setModelCapabilityOverride, removeModelCapabilityOverride } = + await import("../../src/lib/db/modelCapabilityOverrides.ts"); +const { resolveReasoningBufferedMaxTokens } = + await import("../../open-sse/services/reasoningTokenBuffer.ts"); const PROVIDER = "ollama-cloud"; const MODEL = "deepseek-v4-flash"; @@ -87,10 +84,10 @@ test("#6524: with only the (wrong) synced catalog data, the buffer no longer inf assert.equal(result, 64000); }); -test("#6524: an operator-set max_token override now clamps the reasoning buffer to the real cap", () => { +test("#6524: an operator-set max_output_tokens override clamps the reasoning buffer", () => { assert.ok( - setModelCapabilityOverride(TARGET, "max_token", REAL_UPSTREAM_OUTPUT_CAP), - "expected the max_token override to be written" + setModelCapabilityOverride(TARGET, "max_output_tokens", REAL_UPSTREAM_OUTPUT_CAP), + "expected the max_output_tokens override to be written" ); try { const result = resolveReasoningBufferedMaxTokens(TARGET, 64000); @@ -100,6 +97,6 @@ test("#6524: an operator-set max_token override now clamps the reasoning buffer `(reproduces reported 64000 -> 96000 inflation, upstream then 400s)` ); } finally { - removeModelCapabilityOverride(TARGET, "max_token"); + removeModelCapabilityOverride(TARGET, "max_output_tokens"); } }); diff --git a/tests/unit/v1-models-catalog-generation-race.test.ts b/tests/unit/v1-models-catalog-generation-race.test.ts new file mode 100644 index 0000000000..1c1e519164 --- /dev/null +++ b/tests/unit/v1-models-catalog-generation-race.test.ts @@ -0,0 +1,153 @@ +// Regression guard — catalog in-flight build survived invalidation and repopulated +// the cache with pre-write state (generation race). +// +// catalogCache coalesces concurrent GET /v1/models onto one in-flight builder and +// drops completed entries when invalidateDbCache() bumps modelCatalogCacheVersion. +// But the in-flight map was keyed only by request shape: a build that STARTED before +// a write could still be JOINED by a request that arrived after it, and when the +// stale build finished it called storePayload unconditionally — repopulating the +// now-current cache with a body built from pre-write DB state. +// +// Fix: every in-flight build is bound to the catalog-state generation it started +// from. New requests only join a current-generation build, and storePayload only +// persists a payload whose build generation still equals the live version. A stale +// build still resolves to its own original caller (that request is legitimately +// waiting on it) but never overwrites the fresh cache. +// +// These tests drive resolveCachedCatalogResponse directly with a deferred builder so +// the interleaving is fully deterministic — no sleeps, no real-time races. + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-catalog-genrace-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-genrace-test-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const readCache = await import("../../src/lib/db/readCache.ts"); +const catalogCache = await import("../../src/app/api/v1/models/catalogCache.ts"); + +const HEADER_SOURCES = { corsHeaders: {}, diagnosticHeaders: {} }; + +function makeRequest() { + return new Request("http://localhost/v1/models"); +} + +function makePayload(body: string): catalogCache.CatalogPayload { + return { body, headers: { "content-type": "application/json" }, status: 200, cacheTTL: 60_000 }; +} + +/** A builder whose single in-flight promise resolves only when release() is called. */ +function deferredBuilder(body: string) { + let release!: (payload: catalogCache.CatalogPayload) => void; + let calls = 0; + const gate = new Promise((resolve) => { + release = resolve; + }); + const builder = () => { + calls += 1; + return gate; + }; + return { + builder, + release: () => release(makePayload(body)), + calls: () => calls, + }; +} + +test.beforeEach(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + catalogCache.__resetCatalogBuilderRunsForTest(); +}); + +test.after(() => { + core.resetDbInstance(); + apiKeysDb.resetApiKeyState(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("a build that started before invalidation is not joined and does not repopulate the cache", async () => { + // 1. Start the OLD build and hold it in flight. + const oldBuild = deferredBuilder("OLD"); + const oldResponsePromise = catalogCache.resolveCachedCatalogResponse( + makeRequest(), + HEADER_SOURCES, + oldBuild.builder + ); + // Let the cold path register its in-flight entry (it awaits inflight.promise). + await Promise.resolve(); + assert.equal(oldBuild.calls(), 1, "first request starts the old build"); + + // 2. Invalidate the catalog state (simulates a settings/connections/combos write). + readCache.invalidateDbCache("model-capabilities"); + + // 3. A new request after the write must NOT join the stale old build — it starts a + // FRESH build at the new generation. + const newBuild = deferredBuilder("NEW"); + const newResponsePromise = catalogCache.resolveCachedCatalogResponse( + makeRequest(), + HEADER_SOURCES, + newBuild.builder + ); + await Promise.resolve(); + assert.equal( + newBuild.calls(), + 1, + "post-invalidation request must run a fresh builder, not join the stale old build" + ); + + // 4. Release the NEW build first — it is current-generation and must populate the cache. + newBuild.release(); + const newResponse = await newResponsePromise; + assert.equal(await newResponse.text(), "NEW"); + + // 5. Now release the OLD (stale) build. It still resolves to its original caller, + // but must NOT overwrite the fresh cache entry. + oldBuild.release(); + const oldResponse = await oldResponsePromise; + assert.equal( + await oldResponse.text(), + "OLD", + "stale build still returns its own payload to its original caller" + ); + + // 6. The next request must be served from cache with the FRESH body — proving the + // stale old build never repopulated the cache. A cache hit means no builder runs. + const freshBuild = deferredBuilder("SHOULD-NOT-RUN"); + const cachedResponse = await catalogCache.resolveCachedCatalogResponse( + makeRequest(), + HEADER_SOURCES, + freshBuild.builder + ); + assert.equal(await cachedResponse.text(), "NEW", "cache must hold the fresh payload"); + assert.equal(freshBuild.calls(), 0, "a cache hit must not run the builder again"); +}); + +test("concurrent requests in the SAME generation still coalesce onto one build", async () => { + const build = deferredBuilder("SHARED"); + const p1 = catalogCache.resolveCachedCatalogResponse( + makeRequest(), + HEADER_SOURCES, + build.builder + ); + const p2 = catalogCache.resolveCachedCatalogResponse( + makeRequest(), + HEADER_SOURCES, + build.builder + ); + await Promise.resolve(); + assert.equal(build.calls(), 1, "same-generation concurrent requests share one build"); + + build.release(); + const [r1, r2] = await Promise.all([p1, p2]); + assert.equal(await r1.text(), "SHARED"); + assert.equal(await r2.text(), "SHARED"); +});