diff --git a/open-sse/services/combo/comboStructure.ts b/open-sse/services/combo/comboStructure.ts index 0b7f8cb0fa..cdd6bec347 100644 --- a/open-sse/services/combo/comboStructure.ts +++ b/open-sse/services/combo/comboStructure.ts @@ -18,6 +18,7 @@ import { getResolvedModelCapabilities } from "../modelCapabilities.ts"; import { parseModel } from "../model.ts"; import { dedupeTargetsByExecutionKey, isRecord } from "./comboData.ts"; import { getTargetProvider, MAX_COMBO_DEPTH } from "./comboPredicates.ts"; +import { evaluateContextLimit } from "./contextOverrideGate.ts"; import { hasEstimableContent } from "./knownContextOverflow.ts"; import { normalizeModelEntry, @@ -487,57 +488,13 @@ function exceedsKnownOutputLimit( return maxOutputTokens < requestedOutputTokens; } -/** - * Decide whether a target's known context limit accommodates the request. - * - * `maxInputTokens` is an **input-only** cap — the requested output reserve is - * already enforced separately against `maxOutputTokens` (see - * `exceedsKnownOutputLimit`), so it must NOT be re-counted here. Comparing - * `maxInputTokens` against `estimatedInputTokens + requestedOutputTokens` - * double-counted the output reserve and shrank the effective input allowance - * (#7039). - * - * `contextWindow` is the total window, so input + output must both fit. - * - * Returns `true` when the known limit accommodates the request, `false` when - * it is known to be too small, and `null` when no limit metadata is known. - */ -function evaluateContextLimit( - capabilities: { maxInputTokens?: number | null; contextWindow?: number | null }, - requirements: { estimatedInputTokens: number; requiredContextTokens: number } -): boolean | null { - const hasMaxInput = capabilities.maxInputTokens != null; - const hasContextWindow = capabilities.contextWindow != null; - - // Neither limit is known — cannot judge. - if (!hasMaxInput && !hasContextWindow) return null; - - // The input-only cap must accommodate the estimated input. - const inputFits = hasMaxInput - ? capabilities.maxInputTokens! >= requirements.estimatedInputTokens - : true; - - // The total window must accommodate input + requested output. The output - // reserve is enforced separately via `maxOutputTokens`, but when a model - // exposes both `maxInputTokens` and `contextWindow` the two must not be - // checked in isolation: a request whose input fits `maxInputTokens` but whose - // input + output exceeds `contextWindow` must still be rejected (#7039 - // follow-up — shared-window models where `maxInputTokens` defaults to the - // total window size). - const totalFits = hasContextWindow - ? capabilities.contextWindow! >= requirements.requiredContextTokens - : true; - - return inputFits && totalFits; -} - function hasKnownCompatibleContextLimit( target: ResolvedComboTarget, requirements: RequestCompatibilityRequirements ): boolean { if (requirements.requiredContextTokens <= 0) return false; const capabilities = getResolvedModelCapabilities(target.modelStr); - return evaluateContextLimit(capabilities, requirements) === true; + return evaluateContextLimit(capabilities, requirements, target.modelStr) === true; } function hasOnlyContextWindowFailures(reasons: string[]): boolean { @@ -576,7 +533,7 @@ function getTargetCompatibilityFailures( failures.push("output_tokens"); } - const contextVerdict = evaluateContextLimit(capabilities, requirements); + const contextVerdict = evaluateContextLimit(capabilities, requirements, target.modelStr); if (requirements.requiredContextTokens > 0 && contextVerdict === false) { failures.push("context_window"); } diff --git a/open-sse/services/combo/contextOverrideGate.ts b/open-sse/services/combo/contextOverrideGate.ts new file mode 100644 index 0000000000..605c27c13f --- /dev/null +++ b/open-sse/services/combo/contextOverrideGate.ts @@ -0,0 +1,94 @@ +/** + * Context-fit evaluation for combo routing's compatibility filter, extracted + * from comboStructure.ts to keep that file under the file-size cap (PR + * #7933's model_context_override fix pushed it over). + * + * evaluateContextLimit() is the single chokepoint both compatibility-check + * call sites in comboStructure.ts (hasKnownCompatibleContextLimit, + * getTargetCompatibilityFailures) go through. It first consults a persisted + * per-model context override, then falls back to the catalog's + * maxInputTokens/contextWindow limits. + * + * Override rationale (Feature 5004): the catalog's `maxInputTokens` can be a + * deliberately smaller *client-facing* hint (e.g. set below the true window so + * coding agents auto-compact — #6191); using it to filter fallback targets + * wrongly drops otherwise-capable providers for large prompts, collapsing the + * pool to one provider and producing a hard 503 with no fallback once that + * provider's quota is exhausted. An operator-set or auto-discovered override + * reflects the real capacity, so it supersedes both catalog limits. Uses the + * raw override (`getModelContextOverride` returns `null` when none is set) — + * NOT `getModelContextLimitForModelString`, which falls back to + * `contextWindow` and would therefore bypass the `maxInputTokens` cap for + * every model, not just overridden ones. + */ + +import { getModelContextOverride } from "../../../src/lib/db/modelContextOverrides"; +import { parseModel } from "../model.ts"; + +/** + * Resolve the context-fit verdict from a persisted per-model override, if one + * is set. Returns `undefined` when there is no `modelStr` or no override + * exists, so the caller falls through to the catalog-based check; otherwise + * returns the fit verdict for the override itself. + */ +function resolveContextOverrideVerdict( + modelStr: string | undefined, + requiredContextTokens: number +): boolean | undefined { + if (!modelStr) return undefined; + const parsed = parseModel(modelStr); + const override = getModelContextOverride(parsed.provider, parsed.model); + if (override == null) return undefined; + return override >= requiredContextTokens; +} + +/** + * Decide whether a target's known context limit accommodates the request. + * + * `maxInputTokens` is an **input-only** cap — the requested output reserve is + * already enforced separately against `maxOutputTokens` (see + * `exceedsKnownOutputLimit` in comboStructure.ts), so it must NOT be + * re-counted here. Comparing `maxInputTokens` against `estimatedInputTokens + + * requestedOutputTokens` double-counted the output reserve and shrank the + * effective input allowance (#7039). + * + * `contextWindow` is the total window, so input + output must both fit. + * + * Returns `true` when the known limit accommodates the request, `false` when + * it is known to be too small, and `null` when no limit metadata is known. + */ +export function evaluateContextLimit( + capabilities: { maxInputTokens?: number | null; contextWindow?: number | null }, + requirements: { estimatedInputTokens: number; requiredContextTokens: number }, + modelStr?: string +): boolean | null { + const overrideVerdict = resolveContextOverrideVerdict( + modelStr, + requirements.requiredContextTokens + ); + if (overrideVerdict !== undefined) return overrideVerdict; + + const hasMaxInput = capabilities.maxInputTokens != null; + const hasContextWindow = capabilities.contextWindow != null; + + // Neither limit is known — cannot judge. + if (!hasMaxInput && !hasContextWindow) return null; + + // The input-only cap must accommodate the estimated input. + const inputFits = hasMaxInput + ? capabilities.maxInputTokens! >= requirements.estimatedInputTokens + : true; + + // The total window must accommodate input + requested output. The output + // reserve is enforced separately via `maxOutputTokens`, but when a model + // exposes both `maxInputTokens` and `contextWindow` the two must not be + // checked in isolation: a request whose input fits `maxInputTokens` but whose + // input + output exceeds `contextWindow` must still be rejected (#7039 + // follow-up — shared-window models where `maxInputTokens` defaults to the + // total window size). + const totalFits = hasContextWindow + ? capabilities.contextWindow! >= requirements.requiredContextTokens + : true; + + return inputFits && totalFits; +} diff --git a/tests/unit/combo-context-window-filter.test.ts b/tests/unit/combo-context-window-filter.test.ts index 342e360ce4..12214ce1ba 100644 --- a/tests/unit/combo-context-window-filter.test.ts +++ b/tests/unit/combo-context-window-filter.test.ts @@ -18,6 +18,8 @@ const { saveModelsDevCapabilities, clearModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); const { filterTargetsByRequestCompatibility, getKnownContextOverflow, handleComboChat } = await import("../../open-sse/services/combo.ts"); +const { setModelContextOverride, removeModelContextOverride } = + await import("../../src/lib/db/modelContextOverrides.ts"); test.after(() => { core.resetDbInstance(); @@ -364,3 +366,53 @@ test("maxInputTokens defaulting to contextWindow still rejects when input + outp ["unit-7039-window/huge"] ); }); + +// A persisted model_context_override (Feature 5004 — operator-set or +// auto-discovered) reflects a target's real usable window and must win over the +// static catalog limit for server-side routing. Otherwise a provider whose +// catalog `maxInputTokens` is a deliberately smaller client-facing hint (set +// below the true window so coding agents auto-compact, #6191) gets wrongly +// dropped for large-context requests, collapsing the fallback pool. +test("model_context_override lets a small-catalog target survive a large-context request", () => { + saveModelsDevCapabilities({ + "unit-override": { + big: capabilityEntry(1_000_000), + capped: capabilityEntry(8_000), + }, + }); + setModelContextOverride("unit-override", "capped", 1_000_000); + try { + const out = filterTargetsByRequestCompatibility( + [target("unit-override/capped"), target("unit-override/big")], + largeContextBody(), + noopLog + ); + assert.deepEqual( + out.map((entry) => entry.modelStr).sort(), + ["unit-override/big", "unit-override/capped"] + ); + } finally { + removeModelContextOverride("unit-override", "capped"); + } +}); + +test("without an override the small-catalog target is still dropped for the large request", () => { + saveModelsDevCapabilities({ + "unit-override": { + big: capabilityEntry(1_000_000), + capped: capabilityEntry(8_000), + }, + }); + // No override: capped (8K) is genuinely too small and must be filtered out, + // guarding the override read-path from masking a real too-small target. + const out = filterTargetsByRequestCompatibility( + [target("unit-override/capped"), target("unit-override/big")], + largeContextBody(), + noopLog + ); + + assert.deepEqual( + out.map((entry) => entry.modelStr), + ["unit-override/big"] + ); +});