diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 6747824f73..285a6e6633 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -190,7 +190,7 @@ "src/app/api/providers/[id]/models/route.ts": 2590, "src/app/api/providers/[id]/test/route.ts": 917, "src/app/api/usage/analytics/route.ts": 941, - "src/app/api/v1/models/catalog.ts": 1577, + "src/app/api/v1/models/catalog.ts": 1615, "src/lib/cloudflaredTunnel.ts": 934, "src/lib/db/apiKeys.ts": 1662, "src/lib/db/core.ts": 1825, diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 2cc2c90c9c..e93dddd351 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -65,6 +65,12 @@ interface CustomModelEntry { supportedEndpoints?: string[]; inputTokenLimit?: number; isHidden?: boolean; + // User-set "vision-capable" flag (persisted by addCustomModel / replaceCustomModels + // in src/lib/db/models.ts). Surfaced into `/v1/models` via + // getCustomVisionCapabilityFields so user-added vision models appear with + // `capabilities.vision: true` even when their id does not match the + // conservative isVisionModelId heuristic. + supportsVision?: boolean; } const FALLBACK_ALIAS_TO_PROVIDER = { @@ -151,6 +157,39 @@ function getVisionCapabilityFields(modelId: string) { }; } +/** + * Vision-capability fields for a user-added custom chat model. Honours an + * explicit `supportsVision` flag on the saved entry (the dashboard "vision- + * capable" toggle) IN ADDITION TO the conservative id-based heuristic used by + * built-in models. Without this, a user who registered e.g. `my-vision-llm` + * and ticked vision saw no `capabilities.vision` in `/v1/models`, so the LLM + * selector and downstream routing treated the model as text-only. + * + * Port of upstream decolua/9router 5e5e78d3. Conservative: an explicit + * `supportsVision === false` wins so users can downgrade a mis-classified + * model (same anti-FP discipline as #4071 / #4072). + */ +export function getCustomVisionCapabilityFields( + entry: { supportsVision?: boolean } | null | undefined, + ...candidateIds: Array +): { capabilities: { vision: true }; input_modalities: string[]; output_modalities: string[] } | null { + if (entry && entry.supportsVision === false) return null; + if (entry && entry.supportsVision === true) { + return { + capabilities: { vision: true }, + input_modalities: ["text", "image"], + output_modalities: ["text"], + }; + } + for (const id of candidateIds) { + if (typeof id === "string" && id) { + const fields = getVisionCapabilityFields(id); + if (fields) return fields; + } + } + return null; +} + function qualifyOpenRouterModelId(modelId: string): string { return modelId.startsWith("openrouter/") ? modelId : `openrouter/${modelId}`; } @@ -1264,7 +1303,7 @@ export async function getUnifiedModelsResponse( } const visionFields = modelType === "chat" - ? getVisionCapabilityFields(aliasId) || getVisionCapabilityFields(modelId) + ? getCustomVisionCapabilityFields(model, aliasId, modelId) : null; if (includeAlias) { @@ -1297,8 +1336,7 @@ export async function getUnifiedModelsResponse( if (models.some((m) => m.id === providerPrefixedId)) continue; const providerVisionFields = modelType === "chat" - ? getVisionCapabilityFields(providerPrefixedId) || - getVisionCapabilityFields(modelId) + ? getCustomVisionCapabilityFields(model, providerPrefixedId, modelId) : null; models.push({ id: providerPrefixedId, diff --git a/tests/unit/llm-selector-custom-vision-models.test.ts b/tests/unit/llm-selector-custom-vision-models.test.ts new file mode 100644 index 0000000000..3d79da12fd --- /dev/null +++ b/tests/unit/llm-selector-custom-vision-models.test.ts @@ -0,0 +1,74 @@ +/** + * Custom vision models in the LLM selector / `/v1/models` catalog + * (port of upstream decolua/9router 5e5e78d3 — "fix: show custom vision + * models in LLM selector and model list"). + * + * Upstream stored an `imageToText` kind on user-added custom models and the + * selector hid them from the default LLM list. OmniRoute already keeps custom + * chat models in the LLM list, but the equivalent gap is the `supportsVision` + * flag on a custom model: the catalog used to only set `capabilities.vision` + * when the model id matched the conservative `isVisionModelId` heuristic + * (`pixtral`, `llava`, `qwen-vl`, …). A user who registered a custom chat + * model called e.g. `my-vision-llm` and explicitly checked "vision-capable" + * still saw no `capabilities.vision` in `/v1/models`, so the LLM selector and + * downstream routing treated it as text-only. + * + * This file pins the OmniRoute-flavoured behaviour: + * • a custom chat model with `supportsVision: true` MUST surface + * `capabilities: { vision: true }` via the catalog helper, regardless of + * whether the model id matches the static vision heuristic. + * • the existing id-based heuristic (`isVisionModelId`) still works. + * • when neither signal is present, no vision fields are emitted (no false + * positives — same conservative guard #4071 / #4072). + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const catalog = await import("../../src/app/api/v1/models/catalog.ts"); + +type VisionFields = { capabilities: { vision: true } } | null; +type Helper = (entry: unknown, ...candidateIds: Array) => VisionFields; + +const getCustomVisionCapabilityFields = ( + catalog as unknown as { getCustomVisionCapabilityFields?: Helper } +).getCustomVisionCapabilityFields; + +test("custom vision helper is exported from the catalog module", () => { + assert.equal( + typeof getCustomVisionCapabilityFields, + "function", + "expected getCustomVisionCapabilityFields to be exported from src/app/api/v1/models/catalog.ts" + ); +}); + +test("supportsVision flag on a custom model surfaces capabilities.vision (LLM selector unblock)", () => { + const helper = getCustomVisionCapabilityFields as Helper; + const entry = { id: "my-vision-llm", supportsVision: true }; + const fields = helper(entry, "openai-compat/my-vision-llm", "my-vision-llm"); + assert.ok(fields, "supportsVision=true must yield vision fields"); + assert.equal(fields?.capabilities.vision, true); +}); + +test("id-based isVisionModelId heuristic still works when supportsVision is unset", () => { + const helper = getCustomVisionCapabilityFields as Helper; + const entry = { id: "pixtral-12b" }; + const fields = helper(entry, "openai-compat/pixtral-12b", "pixtral-12b"); + assert.ok(fields, "well-known vision model id must yield vision fields"); + assert.equal(fields?.capabilities.vision, true); +}); + +test("no signal → no vision fields (avoids #4071-style false positives)", () => { + const helper = getCustomVisionCapabilityFields as Helper; + const entry = { id: "plain-text-llm" }; + const fields = helper(entry, "openai-compat/plain-text-llm", "plain-text-llm"); + assert.equal(fields, null); +}); + +test("explicit supportsVision=false is respected (text-only override)", () => { + const helper = getCustomVisionCapabilityFields as Helper; + // Even if the id matches the heuristic, an explicit false flag wins so the + // user can downgrade a mis-classified model. + const entry = { id: "pixtral-12b", supportsVision: false }; + const fields = helper(entry, "alias/pixtral-12b", "pixtral-12b"); + assert.equal(fields, null); +});