mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 02:42:24 +03:00
`Array.every` on modality arrays meant one unknown target wiped `input_modalities` for the entire combo — degrade-not-wipe is the correct semantic for a least-common-denominator. Upserting OpenRouter's live `architecture.input_modalities` without a full-table replace, and only invalidating cache when SQLite reports a change, are both the careful version. --- Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017). - `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓ - complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline - 531 of 532 focused assertions green across the batch's 46 test files - `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied. ⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here). Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit.
This commit is contained in:
1
changelog.d/fixes/12613-combo-openrouter-modalities.md
Normal file
1
changelog.d/fixes/12613-combo-openrouter-modalities.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(catalog):** degrade unknown combo targets instead of dropping LCD modalities, and persist OpenRouter `architecture.input_modalities` into the capability snapshot ([#12613](https://github.com/diegosouzapw/OmniRoute/issues/12613))
|
||||
@@ -68,7 +68,12 @@ import {
|
||||
type CatalogEnrichmentSnapshot,
|
||||
} from "@/lib/modelMetadataRegistry";
|
||||
import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
|
||||
import { getModelsDevPricing, getSyncedCapability } from "@/lib/modelsDevSync";
|
||||
import {
|
||||
getModelsDevPricing,
|
||||
getSyncedCapability,
|
||||
upsertSyncedCapabilities,
|
||||
} from "@/lib/modelsDevSync";
|
||||
import type { ModelCapabilityEntry } from "@/lib/modelsDevSync";
|
||||
import { getModelSpec } from "@/shared/constants/modelSpecs";
|
||||
import { classifyModelSupportedEndpoints } from "@/shared/constants/modelSupportedEndpoints";
|
||||
import { getModelsCatalogPrefixMode } from "@/shared/utils/featureFlags";
|
||||
@@ -92,7 +97,7 @@ import {
|
||||
type ComboTargetCatalogMetadata,
|
||||
isPositiveFiniteNumber,
|
||||
parseJsonStringArray,
|
||||
intersectStringArrays,
|
||||
intersectKnownStringArrays,
|
||||
minKnownNumber,
|
||||
maybeOmitCatalogModelName,
|
||||
getThinkingCapabilityFields,
|
||||
@@ -106,6 +111,7 @@ import {
|
||||
getOpenRouterModelType,
|
||||
isOpenRouterFreeModel,
|
||||
getOpenRouterDisplayName,
|
||||
openRouterCapabilityEntry,
|
||||
} from "./catalogOpenrouter";
|
||||
import { getVisionCapabilityFields, getCustomVisionCapabilityFields } from "./catalogVision";
|
||||
import {
|
||||
@@ -762,17 +768,12 @@ async function buildUnifiedModelsResponseCore(
|
||||
knownMetadata.map((metadata) => metadata.maxOutputTokens)
|
||||
);
|
||||
|
||||
const inputModalities = knownMetadata.every(
|
||||
(metadata) => Array.isArray(metadata.inputModalities) && metadata.inputModalities.length > 0
|
||||
)
|
||||
? intersectStringArrays(knownMetadata.map((metadata) => metadata.inputModalities || []))
|
||||
: [];
|
||||
const outputModalities = knownMetadata.every(
|
||||
(metadata) =>
|
||||
Array.isArray(metadata.outputModalities) && metadata.outputModalities.length > 0
|
||||
)
|
||||
? intersectStringArrays(knownMetadata.map((metadata) => metadata.outputModalities || []))
|
||||
: [];
|
||||
const inputModalities = intersectKnownStringArrays(
|
||||
knownMetadata.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : []))
|
||||
);
|
||||
const outputModalities = intersectKnownStringArrays(
|
||||
knownMetadata.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : []))
|
||||
);
|
||||
|
||||
const capabilities = mergeComboCapabilities(knownMetadata);
|
||||
if (targetMetadata.some((metadata) => metadata === null)) {
|
||||
@@ -887,20 +888,12 @@ async function buildUnifiedModelsResponseCore(
|
||||
const knownAutoMeta = autoTargetMetadata.filter(
|
||||
(m): m is ComboTargetCatalogMetadata => m !== null
|
||||
);
|
||||
const autoInputModalities =
|
||||
knownAutoMeta.length > 0 &&
|
||||
knownAutoMeta.every(
|
||||
(m) => Array.isArray(m.inputModalities) && m.inputModalities.length > 0
|
||||
)
|
||||
? intersectStringArrays(knownAutoMeta.map((m) => m.inputModalities || []))
|
||||
: [];
|
||||
const autoOutputModalities =
|
||||
knownAutoMeta.length > 0 &&
|
||||
knownAutoMeta.every(
|
||||
(m) => Array.isArray(m.outputModalities) && m.outputModalities.length > 0
|
||||
)
|
||||
? intersectStringArrays(knownAutoMeta.map((m) => m.outputModalities || []))
|
||||
: [];
|
||||
const autoInputModalities = intersectKnownStringArrays(
|
||||
knownAutoMeta.map((m) => (Array.isArray(m.inputModalities) ? m.inputModalities : []))
|
||||
);
|
||||
const autoOutputModalities = intersectKnownStringArrays(
|
||||
knownAutoMeta.map((m) => (Array.isArray(m.outputModalities) ? m.outputModalities : []))
|
||||
);
|
||||
const autoCapabilities: Record<string, boolean | string[]> = {
|
||||
tool_calling: true,
|
||||
reasoning: true,
|
||||
@@ -1345,6 +1338,7 @@ async function buildUnifiedModelsResponseCore(
|
||||
) {
|
||||
try {
|
||||
const openRouterCatalog = await getOpenRouterCatalog();
|
||||
const openRouterCaps: Record<string, ModelCapabilityEntry> = {};
|
||||
for (const openRouterModel of openRouterCatalog.data || []) {
|
||||
if (!openRouterModel?.id || typeof openRouterModel.id !== "string") continue;
|
||||
const qualifiedId = qualifyOpenRouterModelId(openRouterModel.id);
|
||||
@@ -1402,10 +1396,16 @@ async function buildUnifiedModelsResponseCore(
|
||||
...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}),
|
||||
...(Object.keys(capabilities).length > 0 ? { capabilities } : {}),
|
||||
});
|
||||
|
||||
// #9147: OpenRouter catalog can be large — yield periodically.
|
||||
const capEntry = openRouterCapabilityEntry(
|
||||
openRouterModel,
|
||||
inputModalities,
|
||||
outputModalities,
|
||||
capabilities
|
||||
);
|
||||
if (capEntry) openRouterCaps[openRouterModel.id] = capEntry;
|
||||
await maybeYieldCatalogBuild();
|
||||
}
|
||||
upsertSyncedCapabilities("openrouter", openRouterCaps);
|
||||
} catch (err) {
|
||||
console.error("[catalog] Error loading OpenRouter catalog:", err);
|
||||
}
|
||||
|
||||
@@ -91,6 +91,11 @@ export function intersectStringArrays(arrays: string[][]): string[] {
|
||||
});
|
||||
}
|
||||
|
||||
/** LCD over known arrays only. Empty/unknown entries degrade instead of wiping. */
|
||||
export function intersectKnownStringArrays(arrays: string[][]): string[] {
|
||||
return intersectStringArrays(arrays.filter((values) => values.length > 0));
|
||||
}
|
||||
|
||||
export function minKnownNumber(values: Array<number | undefined>): number | undefined {
|
||||
const knownValues = values.filter(isPositiveFiniteNumber);
|
||||
if (knownValues.length === 0) return undefined;
|
||||
|
||||
@@ -44,3 +44,45 @@ export function getOpenRouterDisplayName(model: {
|
||||
const name = model.name || model.id || "OpenRouter model";
|
||||
return isOpenRouterFreeModel(model) && !/\bgr[aá]tis\b/i.test(name) ? `${name} (Grátis)` : name;
|
||||
}
|
||||
|
||||
export function openRouterCapabilityEntry(
|
||||
model: {
|
||||
id?: string;
|
||||
context_length?: number;
|
||||
top_provider?: { max_completion_tokens?: number };
|
||||
},
|
||||
inputModalities: string[],
|
||||
outputModalities: string[],
|
||||
capabilities: Record<string, boolean>
|
||||
) {
|
||||
if (inputModalities.length === 0 && outputModalities.length === 0) return null;
|
||||
return {
|
||||
tool_call: capabilities.tool_calling === true,
|
||||
reasoning: capabilities.reasoning === true,
|
||||
attachment: null,
|
||||
structured_output: capabilities.structured_output === true,
|
||||
temperature: null,
|
||||
modalities_input: JSON.stringify(inputModalities),
|
||||
modalities_output: JSON.stringify(outputModalities),
|
||||
knowledge_cutoff: null,
|
||||
release_date: null,
|
||||
last_updated: null,
|
||||
status: null,
|
||||
family: null,
|
||||
open_weights: null,
|
||||
limit_context:
|
||||
typeof model.context_length === "number" &&
|
||||
Number.isFinite(model.context_length) &&
|
||||
model.context_length > 0
|
||||
? model.context_length
|
||||
: null,
|
||||
limit_input: null,
|
||||
limit_output:
|
||||
typeof model.top_provider?.max_completion_tokens === "number" &&
|
||||
Number.isFinite(model.top_provider.max_completion_tokens) &&
|
||||
model.top_provider.max_completion_tokens > 0
|
||||
? model.top_provider.max_completion_tokens
|
||||
: null,
|
||||
interleaved_field: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -544,6 +544,84 @@ export function saveModelsDevCapabilities(data: CapabilitiesByProvider): void {
|
||||
if (changed) invalidateDbCache("model-capabilities");
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert-or-replace one provider's capability rows without wiping the table.
|
||||
* Used by the OpenRouter live catalog walk so architecture.input_modalities
|
||||
* survive into the next combo LCD (#12613).
|
||||
*/
|
||||
export function upsertSyncedCapabilities(
|
||||
provider: string,
|
||||
models: Record<string, ModelCapabilityEntry>
|
||||
): void {
|
||||
if (!provider || Object.keys(models).length === 0) return;
|
||||
const db = getDbInstance();
|
||||
ensureCapabilitiesTable();
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO model_capabilities (
|
||||
provider, model_id, tool_call, reasoning, attachment, structured_output,
|
||||
temperature, modalities_input, modalities_output, knowledge_cutoff,
|
||||
release_date, last_updated, status, family, open_weights,
|
||||
limit_context, limit_input, limit_output, interleaved_field, last_synced
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(provider, model_id) DO UPDATE SET
|
||||
tool_call=excluded.tool_call,
|
||||
reasoning=excluded.reasoning,
|
||||
attachment=excluded.attachment,
|
||||
structured_output=excluded.structured_output,
|
||||
temperature=excluded.temperature,
|
||||
modalities_input=excluded.modalities_input,
|
||||
modalities_output=excluded.modalities_output,
|
||||
knowledge_cutoff=excluded.knowledge_cutoff,
|
||||
release_date=excluded.release_date,
|
||||
last_updated=excluded.last_updated,
|
||||
status=excluded.status,
|
||||
family=excluded.family,
|
||||
open_weights=excluded.open_weights,
|
||||
limit_context=excluded.limit_context,
|
||||
limit_input=excluded.limit_input,
|
||||
limit_output=excluded.limit_output,
|
||||
interleaved_field=excluded.interleaved_field,
|
||||
last_synced=excluded.last_synced
|
||||
`);
|
||||
const now = new Date().toISOString();
|
||||
let changed = false;
|
||||
const tx = db.transaction(() => {
|
||||
for (const [modelId, cap] of Object.entries(models)) {
|
||||
const info = insert.run(
|
||||
provider,
|
||||
modelId,
|
||||
cap.tool_call === null ? null : cap.tool_call ? 1 : 0,
|
||||
cap.reasoning === null ? null : cap.reasoning ? 1 : 0,
|
||||
cap.attachment === null ? null : cap.attachment ? 1 : 0,
|
||||
cap.structured_output === null ? null : cap.structured_output ? 1 : 0,
|
||||
cap.temperature === null ? null : cap.temperature ? 1 : 0,
|
||||
cap.modalities_input,
|
||||
cap.modalities_output,
|
||||
cap.knowledge_cutoff,
|
||||
cap.release_date,
|
||||
cap.last_updated,
|
||||
cap.status,
|
||||
cap.family,
|
||||
cap.open_weights === null ? null : cap.open_weights ? 1 : 0,
|
||||
cap.limit_context,
|
||||
cap.limit_input,
|
||||
cap.limit_output,
|
||||
cap.interleaved_field,
|
||||
now
|
||||
);
|
||||
if (info.changes > 0) changed = true;
|
||||
}
|
||||
});
|
||||
tx();
|
||||
if (cachedCapabilities) {
|
||||
cachedCapabilities[provider] = {
|
||||
...(cachedCapabilities[provider] || {}),
|
||||
...models,
|
||||
};
|
||||
}
|
||||
if (changed) invalidateDbCache("model-capabilities");
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all synced capability data.
|
||||
*/
|
||||
|
||||
190
tests/unit/combo-openrouter-modalities-12613.test.ts
Normal file
190
tests/unit/combo-openrouter-modalities-12613.test.ts
Normal file
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* #12613 — combo LCD must degrade unknown/empty-modality targets instead of
|
||||
* dropping the whole intersection. OpenRouter architecture.input_modalities
|
||||
* must also land in the canonical snapshot so a later combo walk can see them.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-12613-combo-modalities-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.API_KEY_SECRET = process.env.API_KEY_SECRET || "catalog-12613-secret";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const combosDb = await import("../../src/lib/db/combos.ts");
|
||||
const modelsDevSync = await import("../../src/lib/modelsDevSync.ts");
|
||||
const catalog = await import("../../src/app/api/v1/models/catalog.ts");
|
||||
const { intersectKnownStringArrays } =
|
||||
await import("../../src/app/api/v1/models/catalogHelpers.ts");
|
||||
const { openRouterCapabilityEntry } =
|
||||
await import("../../src/app/api/v1/models/catalogOpenrouter.ts");
|
||||
|
||||
type CatalogEntry = {
|
||||
id: string;
|
||||
input_modalities?: string[];
|
||||
output_modalities?: string[];
|
||||
};
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
catalog.__resetCatalogBuilderRunsForTest();
|
||||
}
|
||||
|
||||
function capability(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
tool_call: null,
|
||||
reasoning: null,
|
||||
attachment: null,
|
||||
structured_output: null,
|
||||
temperature: null,
|
||||
modalities_input: JSON.stringify([]),
|
||||
modalities_output: JSON.stringify([]),
|
||||
knowledge_cutoff: null,
|
||||
release_date: null,
|
||||
last_updated: null,
|
||||
status: null,
|
||||
family: null,
|
||||
open_weights: null,
|
||||
limit_context: null,
|
||||
limit_input: null,
|
||||
limit_output: null,
|
||||
interleaved_field: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test.beforeEach(async () => {
|
||||
await resetStorage();
|
||||
});
|
||||
|
||||
test.after(async () => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
||||
});
|
||||
|
||||
test("#12613 intersectKnownStringArrays ignores empty unknown arrays", () => {
|
||||
assert.deepEqual(intersectKnownStringArrays([["text", "image"], []]), ["text", "image"]);
|
||||
assert.deepEqual(intersectKnownStringArrays([["text", "image"], ["text"]]), ["text"]);
|
||||
assert.deepEqual(intersectKnownStringArrays([[], []]), []);
|
||||
assert.deepEqual(intersectKnownStringArrays([]), []);
|
||||
});
|
||||
|
||||
test("#12613 combo with one unknown target keeps known vision modalities", async () => {
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "openai-12613",
|
||||
apiKey: "sk-test-12613",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
providerSpecificData: {},
|
||||
});
|
||||
|
||||
modelsDevSync.saveModelsDevCapabilities({
|
||||
openai: {
|
||||
"gpt-4o": capability({
|
||||
tool_call: true,
|
||||
reasoning: false,
|
||||
attachment: true,
|
||||
structured_output: true,
|
||||
temperature: true,
|
||||
modalities_input: JSON.stringify(["text", "image"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
limit_context: 128000,
|
||||
limit_input: 128000,
|
||||
limit_output: 16384,
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
await combosDb.createCombo({
|
||||
name: "vision-plus-unknown-12613",
|
||||
strategy: "priority",
|
||||
models: ["openai/gpt-4o", "openai/totally-unknown-model-12613"],
|
||||
});
|
||||
|
||||
const response = await catalog.getUnifiedModelsResponse(
|
||||
new Request("http://localhost/api/v1/models")
|
||||
);
|
||||
if (response.status !== 200) {
|
||||
const errBody = await response.text();
|
||||
assert.fail(`catalog ${response.status}: ${errBody.slice(0, 500)}`);
|
||||
}
|
||||
const body = (await response.json()) as { data: CatalogEntry[] };
|
||||
const combo = body.data.find((m) => m.id === "vision-plus-unknown-12613");
|
||||
assert.ok(combo, "combo must be listed");
|
||||
assert.ok(
|
||||
Array.isArray(combo.input_modalities) && combo.input_modalities.includes("image"),
|
||||
`unknown target must not drop combo image modality, got ${JSON.stringify(combo.input_modalities)}`
|
||||
);
|
||||
assert.ok(
|
||||
Array.isArray(combo.output_modalities) && combo.output_modalities.includes("text"),
|
||||
`unknown target must not drop combo text output, got ${JSON.stringify(combo.output_modalities)}`
|
||||
);
|
||||
});
|
||||
|
||||
test("#12613 upsertSyncedCapabilities writes OpenRouter modalities without wiping others", () => {
|
||||
modelsDevSync.saveModelsDevCapabilities({
|
||||
openai: {
|
||||
"gpt-4o": capability({
|
||||
modalities_input: JSON.stringify(["text", "image"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
}),
|
||||
},
|
||||
});
|
||||
modelsDevSync.upsertSyncedCapabilities("openrouter", {
|
||||
"openai/gpt-4o": capability({
|
||||
tool_call: true,
|
||||
modalities_input: JSON.stringify(["text", "image"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
}),
|
||||
});
|
||||
const caps = modelsDevSync.getSyncedCapabilities();
|
||||
assert.ok(caps.openai?.["gpt-4o"], "openai rows must survive upsert");
|
||||
assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [
|
||||
"text",
|
||||
"image",
|
||||
]);
|
||||
});
|
||||
|
||||
test("#12613 openRouterCapabilityEntry rejects non-positive limits", () => {
|
||||
const entry = openRouterCapabilityEntry(
|
||||
{ id: "x", context_length: -1, top_provider: { max_completion_tokens: Number.NaN } },
|
||||
["text"],
|
||||
["text"],
|
||||
{ tool_calling: true }
|
||||
);
|
||||
assert.equal(entry?.limit_context, null);
|
||||
assert.equal(entry?.limit_output, null);
|
||||
});
|
||||
|
||||
test("#12613 upsertSyncedCapabilities refreshes limit_output on conflict", async () => {
|
||||
modelsDevSync.upsertSyncedCapabilities("openrouter", {
|
||||
"openai/gpt-4o": {
|
||||
...capability(),
|
||||
modalities_input: JSON.stringify(["text"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
limit_output: 100,
|
||||
},
|
||||
});
|
||||
modelsDevSync.upsertSyncedCapabilities("openrouter", {
|
||||
"openai/gpt-4o": {
|
||||
...capability(),
|
||||
modalities_input: JSON.stringify(["text", "image"]),
|
||||
modalities_output: JSON.stringify(["text"]),
|
||||
limit_output: 200,
|
||||
},
|
||||
});
|
||||
const caps = modelsDevSync.getSyncedCapabilities("openrouter", "openai/gpt-4o");
|
||||
assert.equal(caps.openrouter["openai/gpt-4o"].limit_output, 200);
|
||||
assert.deepEqual(JSON.parse(caps.openrouter["openai/gpt-4o"].modalities_input), [
|
||||
"text",
|
||||
"image",
|
||||
]);
|
||||
});
|
||||
Reference in New Issue
Block a user