From f4fd9190b38ab3722baeb45ecf66afd5bd4b2ba8 Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Fri, 11 Sep 2026 18:28:21 -0400 Subject: [PATCH] fix(catalog): degrade unknown combo targets in LCD modalities (#12613) (#13042) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Array.every` on modality arrays meant one unknown target wiped `input_modalities` for the entire combo — degrade-not-wipe is the correct semantic for a least-common-denominator. Upserting OpenRouter's live `architecture.input_modalities` without a full-table replace, and only invalidating cache when SQLite reports a change, are both the careful version. --- Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017). - `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓ - complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline - 531 of 532 focused assertions green across the batch's 46 test files - `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied. ⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here). Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit. --- .../12613-combo-openrouter-modalities.md | 1 + src/app/api/v1/models/catalog.ts | 58 +++--- src/app/api/v1/models/catalogHelpers.ts | 5 + src/app/api/v1/models/catalogOpenrouter.ts | 42 ++++ src/lib/modelsDevSync.ts | 78 +++++++ .../combo-openrouter-modalities-12613.test.ts | 190 ++++++++++++++++++ 6 files changed, 345 insertions(+), 29 deletions(-) create mode 100644 changelog.d/fixes/12613-combo-openrouter-modalities.md create mode 100644 tests/unit/combo-openrouter-modalities-12613.test.ts diff --git a/changelog.d/fixes/12613-combo-openrouter-modalities.md b/changelog.d/fixes/12613-combo-openrouter-modalities.md new file mode 100644 index 0000000000..aaa4782dcb --- /dev/null +++ b/changelog.d/fixes/12613-combo-openrouter-modalities.md @@ -0,0 +1 @@ +- **fix(catalog):** degrade unknown combo targets instead of dropping LCD modalities, and persist OpenRouter `architecture.input_modalities` into the capability snapshot ([#12613](https://github.com/diegosouzapw/OmniRoute/issues/12613)) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 1e13ad8780..4de6cd21fa 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -68,7 +68,12 @@ import { type CatalogEnrichmentSnapshot, } from "@/lib/modelMetadataRegistry"; import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; -import { getModelsDevPricing, getSyncedCapability } from "@/lib/modelsDevSync"; +import { + getModelsDevPricing, + getSyncedCapability, + upsertSyncedCapabilities, +} from "@/lib/modelsDevSync"; +import type { ModelCapabilityEntry } from "@/lib/modelsDevSync"; import { getModelSpec } from "@/shared/constants/modelSpecs"; import { classifyModelSupportedEndpoints } from "@/shared/constants/modelSupportedEndpoints"; import { getModelsCatalogPrefixMode } from "@/shared/utils/featureFlags"; @@ -92,7 +97,7 @@ import { type ComboTargetCatalogMetadata, isPositiveFiniteNumber, parseJsonStringArray, - intersectStringArrays, + intersectKnownStringArrays, minKnownNumber, maybeOmitCatalogModelName, getThinkingCapabilityFields, @@ -106,6 +111,7 @@ import { getOpenRouterModelType, isOpenRouterFreeModel, getOpenRouterDisplayName, + openRouterCapabilityEntry, } from "./catalogOpenrouter"; import { getVisionCapabilityFields, getCustomVisionCapabilityFields } from "./catalogVision"; import { @@ -762,17 +768,12 @@ async function buildUnifiedModelsResponseCore( knownMetadata.map((metadata) => metadata.maxOutputTokens) ); - const inputModalities = knownMetadata.every( - (metadata) => Array.isArray(metadata.inputModalities) && metadata.inputModalities.length > 0 - ) - ? intersectStringArrays(knownMetadata.map((metadata) => metadata.inputModalities || [])) - : []; - const outputModalities = knownMetadata.every( - (metadata) => - Array.isArray(metadata.outputModalities) && metadata.outputModalities.length > 0 - ) - ? intersectStringArrays(knownMetadata.map((metadata) => metadata.outputModalities || [])) - : []; + const inputModalities = intersectKnownStringArrays( + knownMetadata.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : [])) + ); + const outputModalities = intersectKnownStringArrays( + knownMetadata.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : [])) + ); const capabilities = mergeComboCapabilities(knownMetadata); if (targetMetadata.some((metadata) => metadata === null)) { @@ -887,20 +888,12 @@ async function buildUnifiedModelsResponseCore( const knownAutoMeta = autoTargetMetadata.filter( (m): m is ComboTargetCatalogMetadata => m !== null ); - const autoInputModalities = - knownAutoMeta.length > 0 && - knownAutoMeta.every( - (m) => Array.isArray(m.inputModalities) && m.inputModalities.length > 0 - ) - ? intersectStringArrays(knownAutoMeta.map((m) => m.inputModalities || [])) - : []; - const autoOutputModalities = - knownAutoMeta.length > 0 && - knownAutoMeta.every( - (m) => Array.isArray(m.outputModalities) && m.outputModalities.length > 0 - ) - ? intersectStringArrays(knownAutoMeta.map((m) => m.outputModalities || [])) - : []; + const autoInputModalities = intersectKnownStringArrays( + knownAutoMeta.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : [])) + ); + const autoOutputModalities = intersectKnownStringArrays( + knownAutoMeta.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : [])) + ); const autoCapabilities: Record = { tool_calling: true, reasoning: true, @@ -1345,6 +1338,7 @@ async function buildUnifiedModelsResponseCore( ) { try { const openRouterCatalog = await getOpenRouterCatalog(); + const openRouterCaps: Record = {}; for (const openRouterModel of openRouterCatalog.data || []) { if (!openRouterModel?.id || typeof openRouterModel.id !== "string") continue; const qualifiedId = qualifyOpenRouterModelId(openRouterModel.id); @@ -1402,10 +1396,16 @@ async function buildUnifiedModelsResponseCore( ...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}), ...(Object.keys(capabilities).length > 0 ? { capabilities } : {}), }); - - // #9147: OpenRouter catalog can be large — yield periodically. + const capEntry = openRouterCapabilityEntry( + openRouterModel, + inputModalities, + outputModalities, + capabilities + ); + if (capEntry) openRouterCaps[openRouterModel.id] = capEntry; await maybeYieldCatalogBuild(); } + upsertSyncedCapabilities("openrouter", openRouterCaps); } catch (err) { console.error("[catalog] Error loading OpenRouter catalog:", err); } diff --git a/src/app/api/v1/models/catalogHelpers.ts b/src/app/api/v1/models/catalogHelpers.ts index 02a84e32d1..4973a37e89 100644 --- a/src/app/api/v1/models/catalogHelpers.ts +++ b/src/app/api/v1/models/catalogHelpers.ts @@ -91,6 +91,11 @@ export function intersectStringArrays(arrays: string[][]): string[] { }); } +/** LCD over known arrays only. Empty/unknown entries degrade instead of wiping. */ +export function intersectKnownStringArrays(arrays: string[][]): string[] { + return intersectStringArrays(arrays.filter((values) => values.length > 0)); +} + export function minKnownNumber(values: Array): number | undefined { const knownValues = values.filter(isPositiveFiniteNumber); if (knownValues.length === 0) return undefined; diff --git a/src/app/api/v1/models/catalogOpenrouter.ts b/src/app/api/v1/models/catalogOpenrouter.ts index 744f096809..07d5e00adb 100644 --- a/src/app/api/v1/models/catalogOpenrouter.ts +++ b/src/app/api/v1/models/catalogOpenrouter.ts @@ -44,3 +44,45 @@ export function getOpenRouterDisplayName(model: { const name = model.name || model.id || "OpenRouter model"; return isOpenRouterFreeModel(model) && !/\bgr[aá]tis\b/i.test(name) ? `${name} (Grátis)` : name; } + +export function openRouterCapabilityEntry( + model: { + id?: string; + context_length?: number; + top_provider?: { max_completion_tokens?: number }; + }, + inputModalities: string[], + outputModalities: string[], + capabilities: Record +) { + if (inputModalities.length === 0 && outputModalities.length === 0) return null; + return { + tool_call: capabilities.tool_calling === true, + reasoning: capabilities.reasoning === true, + attachment: null, + structured_output: capabilities.structured_output === true, + temperature: null, + modalities_input: JSON.stringify(inputModalities), + modalities_output: JSON.stringify(outputModalities), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: null, + limit_context: + typeof model.context_length === "number" && + Number.isFinite(model.context_length) && + model.context_length > 0 + ? model.context_length + : null, + limit_input: null, + limit_output: + typeof model.top_provider?.max_completion_tokens === "number" && + Number.isFinite(model.top_provider.max_completion_tokens) && + model.top_provider.max_completion_tokens > 0 + ? model.top_provider.max_completion_tokens + : null, + interleaved_field: null, + }; +} diff --git a/src/lib/modelsDevSync.ts b/src/lib/modelsDevSync.ts index 1400ad7ed4..a5ae357243 100644 --- a/src/lib/modelsDevSync.ts +++ b/src/lib/modelsDevSync.ts @@ -544,6 +544,84 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void { if (changed) invalidateDbCache("model-capabilities"); } +/** + * Insert-or-replace one provider's capability rows without wiping the table. + * Used by the OpenRouter live catalog walk so architecture.input_modalities + * survive into the next combo LCD (#12613). + */ +export function upsertSyncedCapabilities( + provider: string, + models: Record +): void { + if (!provider || Object.keys(models).length === 0) return; + const db = getDbInstance(); + ensureCapabilitiesTable(); + const insert = db.prepare(` + INSERT INTO model_capabilities ( + provider, model_id, tool_call, reasoning, attachment, structured_output, + temperature, modalities_input, modalities_output, knowledge_cutoff, + release_date, last_updated, status, family, open_weights, + limit_context, limit_input, limit_output, interleaved_field, last_synced + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(provider, model_id) DO UPDATE SET + tool_call=excluded.tool_call, + reasoning=excluded.reasoning, + attachment=excluded.attachment, + structured_output=excluded.structured_output, + temperature=excluded.temperature, + modalities_input=excluded.modalities_input, + modalities_output=excluded.modalities_output, + knowledge_cutoff=excluded.knowledge_cutoff, + release_date=excluded.release_date, + last_updated=excluded.last_updated, + status=excluded.status, + family=excluded.family, + open_weights=excluded.open_weights, + limit_context=excluded.limit_context, + limit_input=excluded.limit_input, + limit_output=excluded.limit_output, + interleaved_field=excluded.interleaved_field, + last_synced=excluded.last_synced + `); + const now = new Date().toISOString(); + let changed = false; + const tx = db.transaction(() => { + for (const [modelId, cap] of Object.entries(models)) { + const info = insert.run( + provider, + modelId, + cap.tool_call === null ? null : cap.tool_call ? 1 : 0, + cap.reasoning === null ? null : cap.reasoning ? 1 : 0, + cap.attachment === null ? null : cap.attachment ? 1 : 0, + cap.structured_output === null ? null : cap.structured_output ? 1 : 0, + cap.temperature === null ? null : cap.temperature ? 1 : 0, + cap.modalities_input, + cap.modalities_output, + cap.knowledge_cutoff, + cap.release_date, + cap.last_updated, + cap.status, + cap.family, + cap.open_weights === null ? null : cap.open_weights ? 1 : 0, + cap.limit_context, + cap.limit_input, + cap.limit_output, + cap.interleaved_field, + now + ); + if (info.changes > 0) changed = true; + } + }); + tx(); + if (cachedCapabilities) { + cachedCapabilities[provider] = { + ...(cachedCapabilities[provider] || {}), + ...models, + }; + } + if (changed) invalidateDbCache("model-capabilities"); +} + /** * Clear all synced capability data. */ diff --git a/tests/unit/combo-openrouter-modalities-12613.test.ts b/tests/unit/combo-openrouter-modalities-12613.test.ts new file mode 100644 index 0000000000..1fd4f10c99 --- /dev/null +++ b/tests/unit/combo-openrouter-modalities-12613.test.ts @@ -0,0 +1,190 @@ +/** + * #12613 — combo LCD must degrade unknown/empty-modality targets instead of + * dropping the whole intersection. OpenRouter architecture.input_modalities + * must also land in the canonical snapshot so a later combo walk can see them. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-12613-combo-modalities-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-12613-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const combosDb = await import("../../src/lib/db/combos.ts"); +const modelsDevSync = await import("../../src/lib/modelsDevSync.ts"); +const catalog = await import("../../src/app/api/v1/models/catalog.ts"); +const { intersectKnownStringArrays } = + await import("../../src/app/api/v1/models/catalogHelpers.ts"); +const { openRouterCapabilityEntry } = + await import("../../src/app/api/v1/models/catalogOpenrouter.ts"); + +type CatalogEntry = { + id: string; + input_modalities?: string[]; + output_modalities?: string[]; +}; + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + catalog.__resetCatalogBuilderRunsForTest(); +} + +function capability(overrides: Record = {}) { + return { + tool_call: null, + reasoning: null, + attachment: null, + structured_output: null, + temperature: null, + modalities_input: JSON.stringify([]), + modalities_output: JSON.stringify([]), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: null, + limit_context: null, + limit_input: null, + limit_output: null, + interleaved_field: null, + ...overrides, + }; +} + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(async () => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("#12613 intersectKnownStringArrays ignores empty unknown arrays", () => { + assert.deepEqual(intersectKnownStringArrays([["text", "image"], []]), ["text", "image"]); + assert.deepEqual(intersectKnownStringArrays([["text", "image"], ["text"]]), ["text"]); + assert.deepEqual(intersectKnownStringArrays([[], []]), []); + assert.deepEqual(intersectKnownStringArrays([]), []); +}); + +test("#12613 combo with one unknown target keeps known vision modalities", async () => { + await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "openai-12613", + apiKey: "sk-test-12613", + isActive: true, + testStatus: "active", + providerSpecificData: {}, + }); + + modelsDevSync.saveModelsDevCapabilities({ + openai: { + "gpt-4o": capability({ + tool_call: true, + reasoning: false, + attachment: true, + structured_output: true, + temperature: true, + modalities_input: JSON.stringify(["text", "image"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 128000, + limit_input: 128000, + limit_output: 16384, + }), + }, + }); + + await combosDb.createCombo({ + name: "vision-plus-unknown-12613", + strategy: "priority", + models: ["openai/gpt-4o", "openai/totally-unknown-model-12613"], + }); + + const response = await catalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + if (response.status !== 200) { + const errBody = await response.text(); + assert.fail(`catalog ${response.status}: ${errBody.slice(0, 500)}`); + } + const body = (await response.json()) as { data: CatalogEntry[] }; + const combo = body.data.find((m) => m.id === "vision-plus-unknown-12613"); + assert.ok(combo, "combo must be listed"); + assert.ok( + Array.isArray(combo.input_modalities) && combo.input_modalities.includes("image"), + `unknown target must not drop combo image modality, got ${JSON.stringify(combo.input_modalities)}` + ); + assert.ok( + Array.isArray(combo.output_modalities) && combo.output_modalities.includes("text"), + `unknown target must not drop combo text output, got ${JSON.stringify(combo.output_modalities)}` + ); +}); + +test("#12613 upsertSyncedCapabilities writes OpenRouter modalities without wiping others", () => { + modelsDevSync.saveModelsDevCapabilities({ + openai: { + "gpt-4o": capability({ + modalities_input: JSON.stringify(["text", "image"]), + modalities_output: JSON.stringify(["text"]), + }), + }, + }); + modelsDevSync.upsertSyncedCapabilities("openrouter", { + "openai/gpt-4o": capability({ + tool_call: true, + modalities_input: JSON.stringify(["text", "image"]), + modalities_output: JSON.stringify(["text"]), + }), + }); + const caps = modelsDevSync.getSyncedCapabilities(); + assert.ok(caps.openai?.["gpt-4o"], "openai rows must survive upsert"); + assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [ + "text", + "image", + ]); +}); + +test("#12613 openRouterCapabilityEntry rejects non-positive limits", () => { + const entry = openRouterCapabilityEntry( + { id: "x", context_length: -1, top_provider: { max_completion_tokens: Number.NaN } }, + ["text"], + ["text"], + { tool_calling: true } + ); + assert.equal(entry?.limit_context, null); + assert.equal(entry?.limit_output, null); +}); + +test("#12613 upsertSyncedCapabilities refreshes limit_output on conflict", async () => { + modelsDevSync.upsertSyncedCapabilities("openrouter", { + "openai/gpt-4o": { + ...capability(), + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_output: 100, + }, + }); + modelsDevSync.upsertSyncedCapabilities("openrouter", { + "openai/gpt-4o": { + ...capability(), + modalities_input: JSON.stringify(["text", "image"]), + modalities_output: JSON.stringify(["text"]), + limit_output: 200, + }, + }); + const caps = modelsDevSync.getSyncedCapabilities("openrouter", "openai/gpt-4o"); + assert.equal(caps.openrouter["openai/gpt-4o"].limit_output, 200); + assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [ + "text", + "image", + ]); +});