From 78b084502a7e41f140a303e5fa807e40c0f19e33 Mon Sep 17 00:00:00 2001 From: doda Date: Tue, 12 May 2026 01:31:50 +0900 Subject: [PATCH] feat(api): aggregate combo model metadata in catalog --- src/app/api/v1/models/catalog.ts | 258 +++++++++++++++++++++- tests/unit/models-catalog-route.test.ts | 280 ++++++++++++++++++++++++ 2 files changed, 537 insertions(+), 1 deletion(-) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 12fe5b302c..f77c668f1e 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -16,14 +16,18 @@ import { getAllModerationModels } from "@omniroute/open-sse/config/moderationReg import { getAllVideoModels } from "@omniroute/open-sse/config/videoRegistry.ts"; import { getAllMusicModels } from "@omniroute/open-sse/config/musicRegistry.ts"; import { REGISTRY } from "@omniroute/open-sse/config/providerRegistry.ts"; +import { resolveNestedComboTargets } from "@omniroute/open-sse/services/combo.ts"; import { getAllSyncedAvailableModels } from "@/lib/db/models"; import { getCompatibleFallbackModels } from "@/lib/providers/managedAvailableModels"; import { hasEligibleConnectionForModel } from "@/domain/connectionModelRules"; import { INTERNAL_PROXY_ERROR, enrichCatalogModelEntry, + getCanonicalModelMetadata, getCatalogDiagnosticsHeaders, } from "@/lib/modelMetadataRegistry"; +import { getSyncedCapability } from "@/lib/modelsDevSync"; +import { getModelSpec } from "@/shared/constants/modelSpecs"; import { isAuthRequired, isDashboardSessionAuthenticated } from "@/shared/utils/apiAuth"; const FALLBACK_ALIAS_TO_PROVIDER = { @@ -40,6 +44,50 @@ const FALLBACK_ALIAS_TO_PROVIDER = { qw: "qwen", }; +type ComboCatalogTarget = { + modelStr?: string; + provider?: string | null; +}; + +type ComboTargetCatalogMetadata = { + contextLength?: number; + maxInputTokens?: number; + maxOutputTokens?: number; + inputModalities?: string[]; + outputModalities?: string[]; + capabilities: Record; +}; + +function isPositiveFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value > 0; +} + +function parseJsonStringArray(value: unknown): string[] { + if (typeof value !== "string" || value.trim().length === 0) return []; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) + ? parsed.filter((entry): entry is string => typeof entry === "string" && entry.length > 0) + : []; + } catch { + return []; + } +} + +function intersectStringArrays(arrays: string[][]): string[] { + if (arrays.length === 0 || arrays.some((values) => values.length === 0)) return []; + const [first, ...rest] = arrays; + return first.filter((value, index) => { + if (first.indexOf(value) !== index) return false; + return rest.every((values) => values.includes(value)); + }); +} + +function minKnownNumber(values: Array): number | undefined { + if (values.length === 0 || !values.every(isPositiveFiniteNumber)) return undefined; + return Math.min(...values); +} + const VISION_MODEL_KEYWORDS = [ "gpt-4o", "gpt-4.1", @@ -306,6 +354,212 @@ export async function getUnifiedModelsResponse( ); }; + const getRegistryModel = (providerId: string, modelId: string) => { + const alias = providerIdToAlias[providerId] || PROVIDER_ID_TO_ALIAS[providerId] || providerId; + const providerModels = PROVIDER_MODELS[alias] || PROVIDER_MODELS[providerId] || []; + return providerModels.find((model) => model?.id === modelId) || null; + }; + + const getProviderPrefixes = (providerId: string, rawProvider: string) => { + const prefixes = new Set([providerId, rawProvider, providerIdToAlias[providerId]]); + for (const [alias, mappedProviderId] of Object.entries(aliasToProviderId)) { + if (mappedProviderId === providerId) prefixes.add(alias); + } + return [...prefixes].filter( + (prefix): prefix is string => typeof prefix === "string" && prefix.length > 0 + ); + }; + + const getComboTargetModelId = (target: ComboCatalogTarget) => { + const rawProvider = typeof target.provider === "string" ? target.provider.trim() : ""; + const modelStr = typeof target.modelStr === "string" ? target.modelStr.trim() : ""; + if (!rawProvider || rawProvider === "unknown" || !modelStr) return null; + + const providerId = resolveCanonicalProviderId(rawProvider); + if (!providerId || providerId === "unknown") return null; + + for (const prefix of getProviderPrefixes(providerId, rawProvider)) { + const prefixWithSlash = `${prefix}/`; + if (modelStr.startsWith(prefixWithSlash)) { + const modelId = modelStr.slice(prefixWithSlash.length).trim(); + return modelId ? { providerId, modelId } : null; + } + } + + return { providerId, modelId: modelStr }; + }; + + const getComboTargetCatalogMetadata = ( + target: ComboCatalogTarget + ): ComboTargetCatalogMetadata | null => { + const targetModel = getComboTargetModelId(target); + if (!targetModel) return null; + + const canonical = getCanonicalModelMetadata({ + provider: targetModel.providerId, + model: targetModel.modelId, + }); + if (!canonical) return null; + + const source = canonical.metadata.source; + if (!source.providerRegistry && !source.staticSpec && !source.syncedCapability) return null; + + const providerId = canonical.provider || targetModel.providerId; + const modelId = canonical.model || targetModel.modelId; + const synced = getSyncedCapability(providerId, modelId); + const spec = getModelSpec(modelId); + const registryModel = getRegistryModel(providerId, modelId); + const syncedInputModalities = parseJsonStringArray(synced?.modalities_input); + const syncedOutputModalities = parseJsonStringArray(synced?.modalities_output); + + const syncedContext = isPositiveFiniteNumber(synced?.limit_context) + ? synced.limit_context + : undefined; + const registryContext = isPositiveFiniteNumber(registryModel?.contextLength) + ? registryModel.contextLength + : undefined; + const specContext = isPositiveFiniteNumber(spec?.contextWindow) + ? spec.contextWindow + : undefined; + const contextLength = syncedContext ?? registryContext ?? specContext; + const maxInputTokens = isPositiveFiniteNumber(synced?.limit_input) + ? synced.limit_input + : contextLength; + const maxOutputTokens = isPositiveFiniteNumber(synced?.limit_output) + ? synced.limit_output + : isPositiveFiniteNumber(spec?.maxOutputTokens) + ? spec.maxOutputTokens + : undefined; + + const syncedVision = + typeof synced?.attachment === "boolean" + ? synced.attachment + : syncedInputModalities.length > 0 || syncedOutputModalities.length > 0 + ? [...syncedInputModalities, ...syncedOutputModalities].some((entry) => + entry.toLowerCase().includes("image") + ) + : undefined; + const registryVision = + typeof registryModel?.supportsVision === "boolean" + ? registryModel.supportsVision + : undefined; + const specVision = + typeof spec?.supportsVision === "boolean" ? spec.supportsVision : undefined; + const knownVision = syncedVision ?? registryVision ?? specVision; + + const inputModalities = + syncedInputModalities.length > 0 + ? syncedInputModalities + : knownVision === true + ? ["text", "image"] + : undefined; + const outputModalities = + syncedOutputModalities.length > 0 + ? syncedOutputModalities + : knownVision === true + ? ["text"] + : undefined; + + const capabilities: Record = {}; + if (typeof synced?.tool_call === "boolean") { + capabilities.tool_calling = synced.tool_call; + } else if (typeof registryModel?.toolCalling === "boolean") { + capabilities.tool_calling = registryModel.toolCalling; + } else if (typeof spec?.supportsTools === "boolean") { + capabilities.tool_calling = spec.supportsTools; + } + if (typeof synced?.reasoning === "boolean") { + capabilities.reasoning = synced.reasoning; + } else if (typeof registryModel?.supportsReasoning === "boolean") { + capabilities.reasoning = registryModel.supportsReasoning; + } else if (typeof spec?.supportsThinking === "boolean") { + capabilities.reasoning = spec.supportsThinking; + } + if (typeof knownVision === "boolean") capabilities.vision = knownVision; + if (typeof synced?.attachment === "boolean") capabilities.attachment = synced.attachment; + if (typeof synced?.structured_output === "boolean") { + capabilities.structured_output = synced.structured_output; + } + if (typeof synced?.temperature === "boolean") capabilities.temperature = synced.temperature; + if (typeof synced?.reasoning === "boolean") { + capabilities.thinking = synced.reasoning; + } else if (typeof spec?.supportsThinking === "boolean") { + capabilities.thinking = spec.supportsThinking; + } + + return { + ...(contextLength ? { contextLength } : {}), + ...(maxInputTokens ? { maxInputTokens } : {}), + ...(maxOutputTokens ? { maxOutputTokens } : {}), + ...(inputModalities && inputModalities.length > 0 ? { inputModalities } : {}), + ...(outputModalities && outputModalities.length > 0 ? { outputModalities } : {}), + capabilities, + }; + }; + + const buildComboCatalogMetadata = (combo: Record, allCombos: any[]) => { + const explicitContextLength = isPositiveFiniteNumber(combo.context_length) + ? combo.context_length + : undefined; + + const baseMetadata = explicitContextLength ? { context_length: explicitContextLength } : {}; + const targets = resolveNestedComboTargets(combo, allCombos) as ComboCatalogTarget[]; + if (targets.length === 0) return baseMetadata; + + const targetMetadata = targets.map((target) => getComboTargetCatalogMetadata(target)); + if (targetMetadata.some((metadata) => metadata === null)) return baseMetadata; + + const knownMetadata = targetMetadata as ComboTargetCatalogMetadata[]; + const contextLength = + explicitContextLength ?? + minKnownNumber(knownMetadata.map((metadata) => metadata.contextLength)); + const maxInputTokens = minKnownNumber( + knownMetadata.map((metadata) => metadata.maxInputTokens) + ); + const maxOutputTokens = minKnownNumber( + knownMetadata.map((metadata) => metadata.maxOutputTokens) + ); + + const inputModalities = knownMetadata.every( + (metadata) => Array.isArray(metadata.inputModalities) && metadata.inputModalities.length > 0 + ) + ? intersectStringArrays(knownMetadata.map((metadata) => metadata.inputModalities || [])) + : []; + const outputModalities = knownMetadata.every( + (metadata) => + Array.isArray(metadata.outputModalities) && metadata.outputModalities.length > 0 + ) + ? intersectStringArrays(knownMetadata.map((metadata) => metadata.outputModalities || [])) + : []; + + const capabilities: Record = {}; + for (const key of [ + "tool_calling", + "reasoning", + "vision", + "attachment", + "structured_output", + "temperature", + "thinking", + ]) { + const values = knownMetadata.map((metadata) => metadata.capabilities[key]); + if (values.every((value): value is boolean => typeof value === "boolean")) { + const [first] = values; + if (values.every((value) => value === first)) capabilities[key] = first; + } + } + + return { + ...baseMetadata, + ...(contextLength ? { context_length: contextLength } : {}), + ...(maxInputTokens ? { max_input_tokens: maxInputTokens } : {}), + ...(maxOutputTokens ? { max_output_tokens: maxOutputTokens } : {}), + ...(inputModalities.length > 0 ? { input_modalities: inputModalities } : {}), + ...(outputModalities.length > 0 ? { output_modalities: outputModalities } : {}), + ...(Object.keys(capabilities).length > 0 ? { capabilities } : {}), + }; + }; + // Collect models from active providers (or all if none active) const models = []; const timestamp = Math.floor(Date.now() / 1000); @@ -313,6 +567,7 @@ export async function getUnifiedModelsResponse( // Add combos first (they appear at the top) — only active ones for (const combo of combos) { if (combo.isActive === false || combo.isHidden === true) continue; + const comboMetadata = buildComboCatalogMetadata(combo, combos); models.push({ id: combo.name, object: "model", @@ -321,7 +576,7 @@ export async function getUnifiedModelsResponse( permission: [], root: combo.name, parent: null, - ...(combo.context_length ? { context_length: combo.context_length } : {}), + ...comboMetadata, }); } @@ -817,6 +1072,7 @@ export async function getUnifiedModelsResponse( }; const enrichedModels = finalModels.map((model) => { + if (model.owned_by === "combo") return model; const enriched = enrichCatalogModelEntry(model); const fallbackContextLength = getDefaultContextFallback(enriched); return fallbackContextLength diff --git a/tests/unit/models-catalog-route.test.ts b/tests/unit/models-catalog-route.test.ts index 1ca0db09c8..a81477276b 100644 --- a/tests/unit/models-catalog-route.test.ts +++ b/tests/unit/models-catalog-route.test.ts @@ -37,6 +37,29 @@ async function seedConnection(provider, overrides = {}) { }); } +function capability(overrides = {}) { + return { + tool_call: null, + reasoning: null, + attachment: null, + structured_output: null, + temperature: null, + modalities_input: JSON.stringify([]), + modalities_output: JSON.stringify([]), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: null, + limit_context: null, + limit_input: null, + limit_output: null, + interleaved_field: null, + ...overrides, + }; +} + test.beforeEach(async () => { await resetStorage(); }); @@ -228,6 +251,263 @@ test("v1 models catalog keeps only visible combos when no providers are active", ); }); +test("v1 models catalog derives combo metadata from known targets conservatively", async () => { + try { + modelsDevSync.saveModelsDevCapabilities({ + openai: { + "combo-alpha": capability({ + tool_call: true, + reasoning: true, + attachment: true, + structured_output: true, + temperature: false, + modalities_input: JSON.stringify(["text", "image"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 1000, + limit_input: 900, + limit_output: 120, + }), + }, + gemini: { + "combo-beta": capability({ + tool_call: true, + reasoning: true, + attachment: false, + structured_output: true, + temperature: false, + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 800, + limit_input: 700, + limit_output: 90, + }), + }, + }); + + await combosDb.createCombo({ + name: "metadata-router", + strategy: "priority", + models: ["openai/combo-alpha", "gemini/combo-beta"], + }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const combo = body.data.find((item) => item.id === "metadata-router"); + + assert.equal(response.status, 200); + assert.ok(combo); + assert.equal(combo.context_length, 800); + assert.equal(combo.max_input_tokens, 700); + assert.equal(combo.max_output_tokens, 90); + assert.deepEqual(combo.input_modalities, ["text"]); + assert.deepEqual(combo.output_modalities, ["text"]); + assert.equal(combo.capabilities.structured_output, true); + assert.equal(combo.capabilities.temperature, false); + assert.equal(combo.capabilities.tool_calling, true); + assert.equal(combo.capabilities.reasoning, true); + assert.equal(combo.capabilities.thinking, true); + assert.equal("vision" in combo.capabilities, false); + assert.equal("attachment" in combo.capabilities, false); + assert.equal("architecture" in combo, false); + assert.equal("top_provider" in combo, false); + assert.equal("supported_parameters" in combo, false); + } finally { + modelsDevSync.saveModelsDevCapabilities({}); + } +}); + +test("v1 models catalog lets explicit combo context override derived context", async () => { + try { + modelsDevSync.saveModelsDevCapabilities({ + openai: { + "context-alpha": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 1000, + limit_input: 900, + limit_output: 120, + }), + }, + gemini: { + "context-beta": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 800, + limit_input: 700, + limit_output: 90, + }), + }, + }); + + const combo = await combosDb.createCombo({ + name: "context-router", + strategy: "priority", + models: ["openai/context-alpha", "gemini/context-beta"], + }); + await combosDb.updateCombo((combo as any).id, { context_length: 12345 }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const listed = body.data.find((item) => item.id === "context-router"); + + assert.equal(response.status, 200); + assert.equal(listed.context_length, 12345); + assert.equal(listed.max_input_tokens, 700); + assert.equal(listed.max_output_tokens, 90); + } finally { + modelsDevSync.saveModelsDevCapabilities({}); + } +}); + +test("v1 models catalog keeps unknown combo targets visible without guessed metadata", async () => { + await combosDb.createCombo({ + name: "unknown-router", + strategy: "priority", + models: ["openai/no-known-metadata"], + }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const combo = body.data.find((item) => item.id === "unknown-router"); + + assert.equal(response.status, 200); + assert.ok(combo); + assert.equal("context_length" in combo, false); + assert.equal("max_input_tokens" in combo, false); + assert.equal("max_output_tokens" in combo, false); + assert.equal("input_modalities" in combo, false); + assert.equal("output_modalities" in combo, false); + assert.equal("capabilities" in combo, false); +}); + +test("v1 models catalog aggregates nested combos and keeps hidden child combos unlisted", async () => { + try { + modelsDevSync.saveModelsDevCapabilities({ + openai: { + "nested-alpha": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 1000, + limit_input: 900, + limit_output: 120, + }), + }, + gemini: { + "nested-beta": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 800, + limit_input: 700, + limit_output: 90, + }), + }, + }); + + await combosDb.createCombo({ + name: "hidden-child-router", + strategy: "priority", + models: ["openai/nested-alpha", "gemini/nested-beta"], + isHidden: true, + }); + await combosDb.createCombo({ + name: "parent-router", + strategy: "priority", + models: ["hidden-child-router"], + }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const parent = body.data.find((item) => item.id === "parent-router"); + + assert.equal(response.status, 200); + assert.ok(parent); + assert.equal(parent.context_length, 800); + assert.equal(parent.max_output_tokens, 90); + assert.equal( + body.data.some((item) => item.id === "hidden-child-router"), + false + ); + } finally { + modelsDevSync.saveModelsDevCapabilities({}); + } +}); + +test("v1 models catalog resolves provider aliases without corrupting slashful model ids", async () => { + try { + modelsDevSync.saveModelsDevCapabilities({ + claude: { + "alias-model": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 2000, + limit_input: 1900, + limit_output: 200, + }), + }, + openrouter: { + "Qwen/Qwen3-Coder": capability({ + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + limit_context: 1600, + limit_input: 1500, + limit_output: 150, + }), + }, + }); + + await combosDb.createCombo({ + name: "alias-and-slash-router", + strategy: "priority", + models: [ + { kind: "model", providerId: "claude", model: "cc/alias-model" }, + { kind: "model", providerId: "openrouter", model: "Qwen/Qwen3-Coder" }, + ], + }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const combo = body.data.find((item) => item.id === "alias-and-slash-router"); + + assert.equal(response.status, 200); + assert.ok(combo); + assert.equal(combo.context_length, 1600); + assert.equal(combo.max_input_tokens, 1500); + assert.equal(combo.max_output_tokens, 150); + } finally { + modelsDevSync.saveModelsDevCapabilities({}); + } +}); + +test("v1 models catalog does not final-enrich combo names as real models", async () => { + await combosDb.createCombo({ + name: "gpt-5.5", + strategy: "priority", + models: ["openai/no-known-metadata"], + }); + + const response = await v1ModelsCatalog.getUnifiedModelsResponse( + new Request("http://localhost/api/v1/models") + ); + const body = (await response.json()) as any; + const combo = body.data.find((item) => item.id === "gpt-5.5"); + + assert.equal(response.status, 200); + assert.ok(combo); + assert.equal(combo.owned_by, "combo"); + assert.equal("max_output_tokens" in combo, false); + assert.equal("capabilities" in combo, false); +}); + test("v1 models catalog exposes claude alias and provider-prefixed built-in models with vision metadata", async () => { await seedConnection("claude", { authType: "oauth",