diff --git a/changelog.d/fixes/12897-vllm-max-model-len.md b/changelog.d/fixes/12897-vllm-max-model-len.md new file mode 100644 index 0000000000..2a58c084bb --- /dev/null +++ b/changelog.d/fixes/12897-vllm-max-model-len.md @@ -0,0 +1 @@ +- **fix(providers):** vLLM connections now advertise the real context window: model discovery reads `max_model_len` instead of falling back to the 128K default ([#12897](https://github.com/diegosouzapw/OmniRoute/pull/12897), closes [#12858](https://github.com/diegosouzapw/OmniRoute/issues/12858)) diff --git a/src/lib/providerModels/modelDiscovery.ts b/src/lib/providerModels/modelDiscovery.ts index 3f79163d36..f2dcf77c15 100644 --- a/src/lib/providerModels/modelDiscovery.ts +++ b/src/lib/providerModels/modelDiscovery.ts @@ -426,10 +426,16 @@ export function normalizeDiscoveredModels( // Keep the total context window distinct from an explicit maximum-input limit. Existing // providers historically stored context_length as inputTokenLimit, so retain that compatibility // outside Vertex while persisting the separate contextWindow field for new consumers. + // vLLM — and every server that copies its /v1/models shape — reports the + // window as `max_model_len`, the value the engine was actually started with. + // Without it a vLLM model syncs with no window at all and the resolver hands + // out the 128K default, understating a 250K deployment by half. #12858 const contextWindow = firstPositiveNumber( record.context_length, record.contextLength, record.contextWindow, + record.max_model_len, + record.maxModelLen, topProvider.context_length ); const isVertexProvider = providerId === "vertex" || providerId === "vertex-partner"; diff --git a/tests/unit/vllm-max-model-len-12858.test.ts b/tests/unit/vllm-max-model-len-12858.test.ts new file mode 100644 index 0000000000..621f4f0917 --- /dev/null +++ b/tests/unit/vllm-max-model-len-12858.test.ts @@ -0,0 +1,68 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { normalizeDiscoveredModels } from "@/lib/providerModels/modelDiscovery"; + +// Regression guard for #12858. +// +// vLLM's /v1/models reports the context window as `max_model_len` — the value +// the engine was started with — and nothing else. The native `vllm` provider is +// `passthroughModels: true`, and the OpenAI- and Anthropic-compatible custom +// providers pass the same raw records through, so all three connection shapes +// land here. Before the fix no window field matched, the model was persisted +// with no `inputTokenLimit`, and `/v1/models` advertised the 128K provider +// default for a deployment serving 250K. +// +// Same class of bug as #3202 (OpenRouter's `context_length`): the provider +// declares the window under a name the normalizer does not read. + +test("#12858 maps vLLM max_model_len into inputTokenLimit", () => { + const [model] = normalizeDiscoveredModels([ + { id: "qwen3.8", owned_by: "vllm", root: "/models/Qwen3.8-27B-FP8", max_model_len: 250000 }, + ]); + + assert.equal(model.id, "qwen3.8"); + assert.equal(model.inputTokenLimit, 250000); +}); + +test("#12858 accepts the camelCase spelling too", () => { + // `contextLength` is already accepted next to `context_length`; keep the two + // spellings of this field symmetric with that pair. + const [model] = normalizeDiscoveredModels([{ id: "vendor/camel", maxModelLen: 131072 }]); + + assert.equal(model.inputTokenLimit, 131072); +}); + +test("#12858 preserves an explicit inputTokenLimit over max_model_len", () => { + const [model] = normalizeDiscoveredModels([ + { id: "vendor/explicit-wins", inputTokenLimit: 200000, max_model_len: 999999 }, + ]); + + assert.equal(model.inputTokenLimit, 200000); +}); + +test("#12858 keeps context_length ahead of max_model_len when a record has both", () => { + // OpenRouter-style catalogs are the older contract (#3202); a record carrying + // both fields must not change meaning because this one was added. + const [model] = normalizeDiscoveredModels([ + { id: "vendor/both", context_length: 262144, max_model_len: 131072 }, + ]); + + assert.equal(model.inputTokenLimit, 262144); +}); + +test("#12858 ignores a non-positive or non-numeric max_model_len", () => { + const [zero] = normalizeDiscoveredModels([{ id: "vendor/zero", max_model_len: 0 }]); + const [negative] = normalizeDiscoveredModels([{ id: "vendor/negative", max_model_len: -1 }]); + const [text] = normalizeDiscoveredModels([{ id: "vendor/text", max_model_len: "250000" }]); + + assert.equal(zero.inputTokenLimit, undefined); + assert.equal(negative.inputTokenLimit, undefined); + assert.equal(text.inputTokenLimit, undefined); +}); + +test("#12858 leaves inputTokenLimit unset when no window field is present", () => { + const [model] = normalizeDiscoveredModels([{ id: "vendor/no-window" }]); + + assert.equal(model.inputTokenLimit, undefined); +});