fix(catalog): degrade unknown combo targets in LCD modalities (#12613) (#13042)

`Array.every` on modality arrays meant one unknown target wiped `input_modalities` for the entire combo — degrade-not-wipe is the correct semantic for a least-common-denominator. Upserting OpenRouter's live `architecture.input_modalities` without a full-table replace, and only invalidating cache when SQLite reports a change, are both the careful version.

---

Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017).

- `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓
- complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline
- 531 of 532 focused assertions green across the batch's 46 test files
- `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR

The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied.

⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here).

Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit.
This commit is contained in:
Bob.Hou
2026-09-11 18:28:21 -04:00
committed by GitHub
parent 171421439d
commit f4fd9190b3
6 changed files with 345 additions and 29 deletions

View File

@@ -0,0 +1 @@
- **fix(catalog):** degrade unknown combo targets instead of dropping LCD modalities, and persist OpenRouter `architecture.input_modalities` into the capability snapshot ([#12613](https://github.com/diegosouzapw/OmniRoute/issues/12613))

View File

@@ -68,7 +68,12 @@ import {
type CatalogEnrichmentSnapshot,
} from "@/lib/modelMetadataRegistry";
import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
import { getModelsDevPricing, getSyncedCapability } from "@/lib/modelsDevSync";
import {
getModelsDevPricing,
getSyncedCapability,
upsertSyncedCapabilities,
} from "@/lib/modelsDevSync";
import type { ModelCapabilityEntry } from "@/lib/modelsDevSync";
import { getModelSpec } from "@/shared/constants/modelSpecs";
import { classifyModelSupportedEndpoints } from "@/shared/constants/modelSupportedEndpoints";
import { getModelsCatalogPrefixMode } from "@/shared/utils/featureFlags";
@@ -92,7 +97,7 @@ import {
type ComboTargetCatalogMetadata,
isPositiveFiniteNumber,
parseJsonStringArray,
intersectStringArrays,
intersectKnownStringArrays,
minKnownNumber,
maybeOmitCatalogModelName,
getThinkingCapabilityFields,
@@ -106,6 +111,7 @@ import {
getOpenRouterModelType,
isOpenRouterFreeModel,
getOpenRouterDisplayName,
openRouterCapabilityEntry,
} from "./catalogOpenrouter";
import { getVisionCapabilityFields, getCustomVisionCapabilityFields } from "./catalogVision";
import {
@@ -762,17 +768,12 @@ async function buildUnifiedModelsResponseCore(
knownMetadata.map((metadata) => metadata.maxOutputTokens)
);
const inputModalities = knownMetadata.every(
(metadata) => Array.isArray(metadata.inputModalities) && metadata.inputModalities.length > 0
)
? intersectStringArrays(knownMetadata.map((metadata) => metadata.inputModalities || []))
: [];
const outputModalities = knownMetadata.every(
(metadata) =>
Array.isArray(metadata.outputModalities) && metadata.outputModalities.length > 0
)
? intersectStringArrays(knownMetadata.map((metadata) => metadata.outputModalities || []))
: [];
const inputModalities = intersectKnownStringArrays(
knownMetadata.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : []))
);
const outputModalities = intersectKnownStringArrays(
knownMetadata.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : []))
);
const capabilities = mergeComboCapabilities(knownMetadata);
if (targetMetadata.some((metadata) => metadata === null)) {
@@ -887,20 +888,12 @@ async function buildUnifiedModelsResponseCore(
const knownAutoMeta = autoTargetMetadata.filter(
(m): m is ComboTargetCatalogMetadata => m !== null
);
const autoInputModalities =
knownAutoMeta.length > 0 &&
knownAutoMeta.every(
(m) => Array.isArray(m.inputModalities) && m.inputModalities.length > 0
)
? intersectStringArrays(knownAutoMeta.map((m) => m.inputModalities || []))
: [];
const autoOutputModalities =
knownAutoMeta.length > 0 &&
knownAutoMeta.every(
(m) => Array.isArray(m.outputModalities) && m.outputModalities.length > 0
)
? intersectStringArrays(knownAutoMeta.map((m) => m.outputModalities || []))
: [];
const autoInputModalities = intersectKnownStringArrays(
knownAutoMeta.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : []))
);
const autoOutputModalities = intersectKnownStringArrays(
knownAutoMeta.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : []))
);
const autoCapabilities: Record<string, boolean | string[]> = {
tool_calling: true,
reasoning: true,
@@ -1345,6 +1338,7 @@ async function buildUnifiedModelsResponseCore(
) {
try {
const openRouterCatalog = await getOpenRouterCatalog();
const openRouterCaps: Record<string, ModelCapabilityEntry> = {};
for (const openRouterModel of openRouterCatalog.data || []) {
if (!openRouterModel?.id || typeof openRouterModel.id !== "string") continue;
const qualifiedId = qualifyOpenRouterModelId(openRouterModel.id);
@@ -1402,10 +1396,16 @@ async function buildUnifiedModelsResponseCore(
...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}),
...(Object.keys(capabilities).length > 0 ? { capabilities } : {}),
});
// #9147: OpenRouter catalog can be large — yield periodically.
const capEntry = openRouterCapabilityEntry(
openRouterModel,
inputModalities,
outputModalities,
capabilities
);
if (capEntry) openRouterCaps[openRouterModel.id] = capEntry;
await maybeYieldCatalogBuild();
}
upsertSyncedCapabilities("openrouter", openRouterCaps);
} catch (err) {
console.error("[catalog] Error loading OpenRouter catalog:", err);
}

View File

@@ -91,6 +91,11 @@ export function intersectStringArrays(arrays: string[][]): string[] {
});
}
/** LCD over known arrays only. Empty/unknown entries degrade instead of wiping. */
export function intersectKnownStringArrays(arrays: string[][]): string[] {
return intersectStringArrays(arrays.filter((values) => values.length > 0));
}
export function minKnownNumber(values: Array<number | undefined>): number | undefined {
const knownValues = values.filter(isPositiveFiniteNumber);
if (knownValues.length === 0) return undefined;

View File

@@ -44,3 +44,45 @@ export function getOpenRouterDisplayName(model: {
const name = model.name || model.id || "OpenRouter model";
return isOpenRouterFreeModel(model) && !/\bgr[aá]tis\b/i.test(name) ? `${name} (Grátis)` : name;
}
export function openRouterCapabilityEntry(
model: {
id?: string;
context_length?: number;
top_provider?: { max_completion_tokens?: number };
},
inputModalities: string[],
outputModalities: string[],
capabilities: Record<string, boolean>
) {
if (inputModalities.length === 0 && outputModalities.length === 0) return null;
return {
tool_call: capabilities.tool_calling === true,
reasoning: capabilities.reasoning === true,
attachment: null,
structured_output: capabilities.structured_output === true,
temperature: null,
modalities_input: JSON.stringify(inputModalities),
modalities_output: JSON.stringify(outputModalities),
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: null,
open_weights: null,
limit_context:
typeof model.context_length === "number" &&
Number.isFinite(model.context_length) &&
model.context_length > 0
? model.context_length
: null,
limit_input: null,
limit_output:
typeof model.top_provider?.max_completion_tokens === "number" &&
Number.isFinite(model.top_provider.max_completion_tokens) &&
model.top_provider.max_completion_tokens > 0
? model.top_provider.max_completion_tokens
: null,
interleaved_field: null,
};
}

View File

@@ -544,6 +544,84 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
if (changed) invalidateDbCache("model-capabilities");
}
/**
* Insert-or-replace one provider's capability rows without wiping the table.
* Used by the OpenRouter live catalog walk so architecture.input_modalities
* survive into the next combo LCD (#12613).
*/
export function upsertSyncedCapabilities(
provider: string,
models: Record<string, ModelCapabilityEntry>
): void {
if (!provider || Object.keys(models).length === 0) return;
const db = getDbInstance();
ensureCapabilitiesTable();
const insert = db.prepare(`
INSERT INTO model_capabilities (
provider, model_id, tool_call, reasoning, attachment, structured_output,
temperature, modalities_input, modalities_output, knowledge_cutoff,
release_date, last_updated, status, family, open_weights,
limit_context, limit_input, limit_output, interleaved_field, last_synced
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(provider, model_id) DO UPDATE SET
tool_call=excluded.tool_call,
reasoning=excluded.reasoning,
attachment=excluded.attachment,
structured_output=excluded.structured_output,
temperature=excluded.temperature,
modalities_input=excluded.modalities_input,
modalities_output=excluded.modalities_output,
knowledge_cutoff=excluded.knowledge_cutoff,
release_date=excluded.release_date,
last_updated=excluded.last_updated,
status=excluded.status,
family=excluded.family,
open_weights=excluded.open_weights,
limit_context=excluded.limit_context,
limit_input=excluded.limit_input,
limit_output=excluded.limit_output,
interleaved_field=excluded.interleaved_field,
last_synced=excluded.last_synced
`);
const now = new Date().toISOString();
let changed = false;
const tx = db.transaction(() => {
for (const [modelId, cap] of Object.entries(models)) {
const info = insert.run(
provider,
modelId,
cap.tool_call === null ? null : cap.tool_call ? 1 : 0,
cap.reasoning === null ? null : cap.reasoning ? 1 : 0,
cap.attachment === null ? null : cap.attachment ? 1 : 0,
cap.structured_output === null ? null : cap.structured_output ? 1 : 0,
cap.temperature === null ? null : cap.temperature ? 1 : 0,
cap.modalities_input,
cap.modalities_output,
cap.knowledge_cutoff,
cap.release_date,
cap.last_updated,
cap.status,
cap.family,
cap.open_weights === null ? null : cap.open_weights ? 1 : 0,
cap.limit_context,
cap.limit_input,
cap.limit_output,
cap.interleaved_field,
now
);
if (info.changes > 0) changed = true;
}
});
tx();
if (cachedCapabilities) {
cachedCapabilities[provider] = {
...(cachedCapabilities[provider] || {}),
...models,
};
}
if (changed) invalidateDbCache("model-capabilities");
}
/**
* Clear all synced capability data.
*/

View File

@@ -0,0 +1,190 @@
/**
* #12613 — combo LCD must degrade unknown/empty-modality targets instead of
* dropping the whole intersection. OpenRouter architecture.input_modalities
* must also land in the canonical snapshot so a later combo walk can see them.
*/
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-12613-combo-modalities-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-12613-secret";
const core = await import("../../src/lib/db/core.ts");
const providersDb = await import("../../src/lib/db/providers.ts");
const combosDb = await import("../../src/lib/db/combos.ts");
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
const catalog = await import("../../src/app/api/v1/models/catalog.ts");
const { intersectKnownStringArrays } =
await import("../../src/app/api/v1/models/catalogHelpers.ts");
const { openRouterCapabilityEntry } =
await import("../../src/app/api/v1/models/catalogOpenrouter.ts");
type CatalogEntry = {
id: string;
input_modalities?: string[];
output_modalities?: string[];
};
async function resetStorage() {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
catalog.__resetCatalogBuilderRunsForTest();
}
function capability(overrides: Record<string, unknown> = {}) {
return {
tool_call: null,
reasoning: null,
attachment: null,
structured_output: null,
temperature: null,
modalities_input: JSON.stringify([]),
modalities_output: JSON.stringify([]),
knowledge_cutoff: null,
release_date: null,
last_updated: null,
status: null,
family: null,
open_weights: null,
limit_context: null,
limit_input: null,
limit_output: null,
interleaved_field: null,
...overrides,
};
}
test.beforeEach(async () => {
await resetStorage();
});
test.after(async () => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("#12613 intersectKnownStringArrays ignores empty unknown arrays", () => {
assert.deepEqual(intersectKnownStringArrays([["text", "image"], []]), ["text", "image"]);
assert.deepEqual(intersectKnownStringArrays([["text", "image"], ["text"]]), ["text"]);
assert.deepEqual(intersectKnownStringArrays([[], []]), []);
assert.deepEqual(intersectKnownStringArrays([]), []);
});
test("#12613 combo with one unknown target keeps known vision modalities", async () => {
await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "openai-12613",
apiKey: "sk-test-12613",
isActive: true,
testStatus: "active",
providerSpecificData: {},
});
modelsDevSync.saveModelsDevCapabilities({
openai: {
"gpt-4o": capability({
tool_call: true,
reasoning: false,
attachment: true,
structured_output: true,
temperature: true,
modalities_input: JSON.stringify(["text", "image"]),
modalities_output: JSON.stringify(["text"]),
limit_context: 128000,
limit_input: 128000,
limit_output: 16384,
}),
},
});
await combosDb.createCombo({
name: "vision-plus-unknown-12613",
strategy: "priority",
models: ["openai/gpt-4o", "openai/totally-unknown-model-12613"],
});
const response = await catalog.getUnifiedModelsResponse(
new Request("http://localhost/api/v1/models")
);
if (response.status !== 200) {
const errBody = await response.text();
assert.fail(`catalog ${response.status}: ${errBody.slice(0, 500)}`);
}
const body = (await response.json()) as { data: CatalogEntry[] };
const combo = body.data.find((m) => m.id === "vision-plus-unknown-12613");
assert.ok(combo, "combo must be listed");
assert.ok(
Array.isArray(combo.input_modalities) && combo.input_modalities.includes("image"),
`unknown target must not drop combo image modality, got ${JSON.stringify(combo.input_modalities)}`
);
assert.ok(
Array.isArray(combo.output_modalities) && combo.output_modalities.includes("text"),
`unknown target must not drop combo text output, got ${JSON.stringify(combo.output_modalities)}`
);
});
test("#12613 upsertSyncedCapabilities writes OpenRouter modalities without wiping others", () => {
modelsDevSync.saveModelsDevCapabilities({
openai: {
"gpt-4o": capability({
modalities_input: JSON.stringify(["text", "image"]),
modalities_output: JSON.stringify(["text"]),
}),
},
});
modelsDevSync.upsertSyncedCapabilities("openrouter", {
"openai/gpt-4o": capability({
tool_call: true,
modalities_input: JSON.stringify(["text", "image"]),
modalities_output: JSON.stringify(["text"]),
}),
});
const caps = modelsDevSync.getSyncedCapabilities();
assert.ok(caps.openai?.["gpt-4o"], "openai rows must survive upsert");
assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [
"text",
"image",
]);
});
test("#12613 openRouterCapabilityEntry rejects non-positive limits", () => {
const entry = openRouterCapabilityEntry(
{ id: "x", context_length: -1, top_provider: { max_completion_tokens: Number.NaN } },
["text"],
["text"],
{ tool_calling: true }
);
assert.equal(entry?.limit_context, null);
assert.equal(entry?.limit_output, null);
});
test("#12613 upsertSyncedCapabilities refreshes limit_output on conflict", async () => {
modelsDevSync.upsertSyncedCapabilities("openrouter", {
"openai/gpt-4o": {
...capability(),
modalities_input: JSON.stringify(["text"]),
modalities_output: JSON.stringify(["text"]),
limit_output: 100,
},
});
modelsDevSync.upsertSyncedCapabilities("openrouter", {
"openai/gpt-4o": {
...capability(),
modalities_input: JSON.stringify(["text", "image"]),
modalities_output: JSON.stringify(["text"]),
limit_output: 200,
},
});
const caps = modelsDevSync.getSyncedCapabilities("openrouter", "openai/gpt-4o");
assert.equal(caps.openrouter["openai/gpt-4o"].limit_output, 200);
assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [
"text",
"image",
]);
});