From e780da3578065acc4868a26c99e95a1762201fb5 Mon Sep 17 00:00:00 2001 From: Aref Alapour <125223497+aref-alapour@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:29:58 +0330 Subject: [PATCH] fix(catalog): advertise input_modalities on vision-capable combos (#12799) * fix(catalog): advertise input_modalities on vision-capable combos A combo whose merged capabilities carry vision:true (e.g. an operator-flagged #9195 vision head, or canonical vision with no synced modality data) advertised the boolean with an empty modality set, so models.dev-shaped clients that key off input_modalities still saw a text-only entry. buildComboCatalogMetadata now derives the modalities from the vision verdict it already advertises via visionDerivedModalities() in catalogHelpers; synced modality intersections keep precedence and nothing is derived for unknown or text-only verdicts (fail-closed, same discipline as #4071/#4072). catalog.ts stays at its frozen LOC (spreads collapsed into the helper call). Regression-tested in models-catalog-combo-metadata.test.ts. Refs #12798 * changelog: fragment for #12799 --------- Co-authored-by: aref-alapour Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../12799-combo-vision-derived-modalities.md | 1 + src/app/api/v1/models/catalog.ts | 4 +- src/app/api/v1/models/catalogHelpers.ts | 34 +++++++++ .../models-catalog-combo-metadata.test.ts | 70 +++++++++++++++++++ 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixes/12799-combo-vision-derived-modalities.md diff --git a/changelog.d/fixes/12799-combo-vision-derived-modalities.md b/changelog.d/fixes/12799-combo-vision-derived-modalities.md new file mode 100644 index 0000000000..f4383dc98c --- /dev/null +++ b/changelog.d/fixes/12799-combo-vision-derived-modalities.md @@ -0,0 +1 @@ +- `/v1/models` combos whose merged `capabilities.vision` is `true` now also advertise `input_modalities: ["text","image"]` / `output_modalities: ["text"]` (synced modality intersections keep precedence), so models.dev-shaped clients no longer see a vision combo as text-only. (#12799 — thanks @aref-alapour) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 938db02070..6464408d66 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -103,6 +103,7 @@ import { maybeOmitCatalogModelName, getThinkingCapabilityFields, mergeComboCapabilities, + visionDerivedModalities, getConnectionScopedEffortTiers, type ConnectionScopedReasoningCatalog, memoizeTargetMetadata, @@ -795,8 +796,7 @@ async function buildUnifiedModelsResponseCore( ...(contextLength ? { context_length: contextLength } : {}), ...(maxInputTokens ? { max_input_tokens: maxInputTokens } : {}), ...(maxOutputTokens ? { max_output_tokens: maxOutputTokens } : {}), - ...(inputModalities.length > 0 ? { input_modalities: inputModalities } : {}), - ...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}), + ...visionDerivedModalities(capabilities, inputModalities, outputModalities), // #12798 ...(Object.keys(capabilities).length > 0 ? { capabilities } : {}), }; }; diff --git a/src/app/api/v1/models/catalogHelpers.ts b/src/app/api/v1/models/catalogHelpers.ts index 12b9815b8a..116507b498 100644 --- a/src/app/api/v1/models/catalogHelpers.ts +++ b/src/app/api/v1/models/catalogHelpers.ts @@ -223,6 +223,40 @@ export function mergeComboCapabilities( return capabilities; } +/** + * #12798: a combo can advertise `capabilities.vision: true` while emitting no + * `input_modalities` / `output_modalities`. The merged vision verdict flows + * from the targets canonical capabilities - which honour the #9195 operator + * "Vision capable" override - while the combo-level modality intersection in + * buildComboCatalogMetadata only fills when EVERY known target carries synced + * modality data. An operator-flagged vision head with no synced modalities + * therefore listed `vision: true` next to an empty modality set, so + * models.dev-shaped clients that key off `input_modalities` (not the boolean) + * still saw a text-only entry. Derives the modalities from the already + * advertised vision verdict: mergeComboCapabilities only emits `vision: true` + * when every known target is vision-capable, so this makes no new claim. + * Synced intersections keep precedence; nothing is derived for unknown or + * text-only verdicts. + */ +export function visionDerivedModalities( + capabilities: Record, + syncedInput: string[], + syncedOutput: string[] +): { input_modalities?: string[]; output_modalities?: string[] } { + return { + ...(syncedInput.length > 0 + ? { input_modalities: syncedInput } + : capabilities.vision === true + ? { input_modalities: ["text", "image"] } + : {}), + ...(syncedOutput.length > 0 + ? { output_modalities: syncedOutput } + : capabilities.vision === true + ? { output_modalities: ["text"] } + : {}), + }; +} + /** * Memoize per-target catalog metadata for one catalog build, yielding between misses. * #12046 resolves metadata for every target of every built-in `auto/*` combo, and those diff --git a/tests/unit/models-catalog-combo-metadata.test.ts b/tests/unit/models-catalog-combo-metadata.test.ts index c64b46bd1a..15b419270b 100644 --- a/tests/unit/models-catalog-combo-metadata.test.ts +++ b/tests/unit/models-catalog-combo-metadata.test.ts @@ -628,3 +628,73 @@ test("Ollama Cloud projects native efforts for base, tagged, and combo models", assert.deepEqual(capabilitiesFor(modelId).effort_tiers, narrowEfforts, modelId); } }); + +// #12798: an operator-flagged vision head (the dashboard "Vision capable" +// toggle, #9195) with a synced capability row that carries limits but NO +// modality data merged to `capabilities.vision: true` with an empty modality +// set, so models.dev-shaped clients keying off `input_modalities` still saw a +// text-only combo. The combo must derive its modalities from the vision +// verdict it already advertises. +test("vision-flagged combo derives input modalities from the merged vision verdict", async () => { + await providersDb.createProviderConnection({ + provider: "openai-compatible", + authType: "api_key", + name: "vision-head-provider-12798", + apiKey: "vision-head-test-key", + isActive: true, + testStatus: "active", + providerSpecificData: { baseUrl: "http://127.0.0.1:9/v1" }, + }); + await modelsDb.addCustomModel( + "vision-head-provider-12798", + "custom-vision-head", + "Custom Vision Head", + "manual", + "chat-completions", + ["chat"], + undefined, + {}, + true + ); + const { saveModelsDevCapabilities } = await import("../../src/lib/modelsDevSync.ts"); + saveModelsDevCapabilities({ + "vision-head-provider-12798": { + "custom-vision-head": { + tool_call: true, + reasoning: false, + attachment: null, + structured_output: true, + temperature: true, + modalities_input: null, + modalities_output: null, + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: false, + limit_context: 200000, + limit_input: 200000, + limit_output: 8192, + interleaved_field: null, + }, + }, + }); + await combosDb.createCombo({ + name: "custom-vision-head-combo", + strategy: "auto", + models: ["vision-head-provider-12798/custom-vision-head"], + }); + + const response = await catalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as { data: Array> }; + const combo = body.data.find((item) => item.id === "custom-vision-head-combo"); + + assert.ok(combo, "combo entry missing from /v1/models"); + const comboCapabilities = combo.capabilities as Record; + assert.equal(comboCapabilities.vision, true, "merged vision verdict must be advertised"); + assert.deepEqual(combo.input_modalities, ["text", "image"]); + assert.deepEqual(combo.output_modalities, ["text"]); +});