From 11cbd7d4e0ef561e13042eb4aa148eb4595fe245 Mon Sep 17 00:00:00 2001 From: Marcelo Karval <46399382+marcelokarval@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:23:48 -0300 Subject: [PATCH] fix(models): normalize media endpoint metadata (#11397) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validado em lote combinado (batch-0824g) contra o tip de release/v3.8.50: typecheck:core limpo, gates estáticos OK, 62/62 testes focados passando (endpoint/parser/schema/static-model + catálogo). Canonicaliza metadados de endpoint legados (video/audio) para IDs específicos por operação, mantendo compatibilidade retroativa via `normalizeModelSupportedEndpoints` (valores antigos `audio`/`video` continuam válidos como entrada e são normalizados na escrita). Obrigado pela contribuição, primeira PR bem-vinda! --- .../[id]/components/CustomModelsSection.tsx | 71 +++++++++++++------ src/app/api/v1/models/catalog.ts | 18 ++--- src/lib/db/models.ts | 3 +- src/lib/providerModels/geminiModelsParser.ts | 20 ++++-- src/lib/providers/staticModels.ts | 4 +- .../constants/modelSupportedEndpoints.ts | 54 ++++++++++++++ src/shared/validation/schemas/provider.ts | 20 ++---- tests/unit/gemini-models-parser.test.ts | 4 +- tests/unit/model-supported-endpoints.test.ts | 58 +++++++++++++++ .../provider-model-endpoint-schema.test.ts | 16 +++++ .../static-model-operation-endpoints.test.ts | 20 ++++++ 11 files changed, 233 insertions(+), 55 deletions(-) create mode 100644 src/shared/constants/modelSupportedEndpoints.ts create mode 100644 tests/unit/model-supported-endpoints.test.ts create mode 100644 tests/unit/provider-model-endpoint-schema.test.ts create mode 100644 tests/unit/static-model-operation-endpoints.test.ts diff --git a/src/app/(dashboard)/dashboard/providers/[id]/components/CustomModelsSection.tsx b/src/app/(dashboard)/dashboard/providers/[id]/components/CustomModelsSection.tsx index abb840e51b..18801dcb0e 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/components/CustomModelsSection.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/components/CustomModelsSection.tsx @@ -11,6 +11,10 @@ import React, { useState, useEffect, useCallback, useMemo } from "react"; import { useTranslations } from "next-intl"; import { Button } from "@/shared/components"; +import { + normalizeModelSupportedEndpoints, + type ModelSupportedEndpoint, +} from "@/shared/constants/modelSupportedEndpoints"; import { useNotificationStore } from "@/store/notificationStore"; import { buildCompatMap, @@ -51,6 +55,29 @@ function targetFormatLabel(value: string, t: (key: string) => string): string { return key ? t(key) : value; } +const MODEL_ENDPOINT_OPTIONS: ModelSupportedEndpoint[] = [ + "chat", + "embeddings", + "rerank", + "images", + "videos", + "audio-speech", + "audio-transcriptions", +]; + +function endpointLabel(endpoint: ModelSupportedEndpoint, t: (key: string) => string): string { + const labels: Partial> = { + chat: `💬 ${t("supportedEndpointChat")}`, + embeddings: `📐 ${t("supportedEndpointEmbeddings")}`, + rerank: providerText(t, "rerankEndpoint", "Rerank"), + images: `🖼️ ${t("supportedEndpointImages")}`, + videos: "🎬 Video", + "audio-speech": `🔊 ${t("audioSpeech")}`, + "audio-transcriptions": `🎙️ ${t("audioTranscriptions")}`, + }; + return labels[endpoint] || endpoint; +} + /** * #4125: parse the free-text "Context Window Override" field. Blank → no override * (`value: null`, not an error). A non-empty value must be a positive whole number of @@ -232,7 +259,7 @@ export default function CustomModelsSection({ setEditingApiFormat(model.apiFormat || "chat-completions"); setEditingEndpoints( Array.isArray(model.supportedEndpoints) && model.supportedEndpoints.length - ? model.supportedEndpoints + ? normalizeModelSupportedEndpoints(model.supportedEndpoints) : ["chat"] ); setEditingTargetFormat(model.targetFormat || ""); @@ -428,6 +455,7 @@ export default function CustomModelsSection({ +
@@ -454,7 +482,7 @@ export default function CustomModelsSection({ {t("supportedEndpointsLabel")}
- {["chat", "embeddings", "rerank", "images", "audio"].map((ep) => ( + {MODEL_ENDPOINT_OPTIONS.map((ep) => ( ))}
@@ -594,6 +614,22 @@ export default function CustomModelsSection({ {`🔊 ${t("audioShortLabel")}`} )} + {(model.supportedEndpoints?.includes("videos") || + model.supportedEndpoints?.includes("video")) && ( + + 🎬 Video + + )} + {model.supportedEndpoints?.includes("audio-speech") && ( + + {`🔊 ${t("audioSpeech")}`} + + )} + {model.supportedEndpoints?.includes("audio-transcriptions") && ( + + {`🎙️ ${t("audioTranscriptions")}`} + + )} {anyNormalizeCompatBadge(model.id!, customMap, overrideMap) && ( {t("audioTranscriptions")} +
@@ -697,7 +734,7 @@ export default function CustomModelsSection({ {t("supportedEndpointsLabel")}
- {["chat", "embeddings", "rerank", "images", "audio"].map((ep) => ( + {MODEL_ENDPOINT_OPTIONS.map((ep) => ( ))}
diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 04bd7bcc49..d70a275e3d 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -69,6 +69,7 @@ import { import { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; import { getModelsDevPricing, getSyncedCapability } from "@/lib/modelsDevSync"; import { getModelSpec } from "@/shared/constants/modelSpecs"; +import { classifyModelSupportedEndpoints } from "@/shared/constants/modelSupportedEndpoints"; import { getModelsCatalogPrefixMode } from "@/shared/utils/featureFlags"; import { buildReservedPrefixes, selectCompatibleNodeForPrefix } from "@/lib/providerNodePrefixes"; import { applyCatalogPostFilters, finalizeCatalogResponse } from "./catalogResponse"; @@ -1152,18 +1153,15 @@ async function buildUnifiedModelsResponseCore( const aliasId = `${alias}/${displayModelId}`; const endpoints = Array.isArray(sm.supportedEndpoints) ? sm.supportedEndpoints : ["chat"]; const apiFormat = typeof sm.apiFormat === "string" ? sm.apiFormat : "chat-completions"; - let modelType: string | undefined; - if (endpoints.includes("embeddings")) modelType = "embedding"; - else if (endpoints.includes("rerank")) modelType = "rerank"; - else if (endpoints.includes("images")) modelType = "image"; - else if (endpoints.includes("audio")) modelType = "audio"; + const classification = classifyModelSupportedEndpoints(endpoints); + const modelType = classification.type; // Same owned_by the alias/canonical entries below will carry — computed once // so the effort_tiers exclusion (codex/glm/kimi) and the entries agree. const syncedOwnedBy = resolvePublicOwnerId(providerId, canonicalProviderId); const syncedFields = { ...(modelType ? { type: modelType } : {}), ...(apiFormat !== "chat-completions" ? { api_format: apiFormat } : {}), - ...(modelType === "audio" ? { subtype: "transcription" } : {}), + ...(classification.subtype ? { subtype: classification.subtype } : {}), ...(sm.inputTokenLimit ? { context_length: sm.inputTokenLimit } : {}), ...(typeof sm.outputTokenLimit === "number" ? { max_output_tokens: sm.outputTokenLimit } @@ -1604,11 +1602,8 @@ async function buildUnifiedModelsResponseCore( : ["chat"]; const apiFormat = typeof model.apiFormat === "string" ? model.apiFormat : "chat-completions"; - let modelType: string | undefined; - if (endpoints.includes("embeddings")) modelType = "embedding"; - else if (endpoints.includes("rerank")) modelType = "rerank"; - else if (endpoints.includes("images")) modelType = "image"; - else if (endpoints.includes("audio")) modelType = "audio"; + const classification = classifyModelSupportedEndpoints(endpoints); + const modelType = classification.type; if ( modelType && hasEquivalentSpecialtyModel(canonicalProviderId, modelId, modelType, aliasId) @@ -1631,6 +1626,7 @@ async function buildUnifiedModelsResponseCore( parent: null, custom: true, ...(modelType ? { type: modelType } : {}), + ...(classification.subtype ? { subtype: classification.subtype } : {}), ...(apiFormat !== "chat-completions" ? { api_format: apiFormat } : {}), ...(endpoints.length > 1 || !endpoints.includes("chat") ? { supported_endpoints: endpoints } diff --git a/src/lib/db/models.ts b/src/lib/db/models.ts index 56e5c20f29..24b7707730 100644 --- a/src/lib/db/models.ts +++ b/src/lib/db/models.ts @@ -190,7 +190,8 @@ export async function addCustomModel( | "rerank" | "audio-transcriptions" | "audio-speech" - | "images-generations" = "chat-completions", + | "images-generations" + | "video" = "chat-completions", supportedEndpoints: string[] = ["chat"], // #2905: optional per-model wire format override (e.g. "claude" for an // opencode-go custom model). When unset, routing falls back to the provider diff --git a/src/lib/providerModels/geminiModelsParser.ts b/src/lib/providerModels/geminiModelsParser.ts index 26c0d8f4cf..e4fd1bfdd7 100644 --- a/src/lib/providerModels/geminiModelsParser.ts +++ b/src/lib/providerModels/geminiModelsParser.ts @@ -3,11 +3,15 @@ * * Each model's `supportedGenerationMethods` is mapped to OmniRoute endpoints: * - generateContent / generateAnswer → "chat" - * - predictLongRunning → "video" (Veo video generation) + * - predict → "images" (Imagen image generation) + * - predictLongRunning → "videos" (Veo video generation) * - embedContent → "embeddings" * - bidiGenerateContent → "audio" (Live real-time audio) * - * Model-id heuristics ensure Veo models remain in the video bucket. + * Model-id heuristics refine the long-running bucket because Google exposes both + * Imagen and Veo via long-running methods on the same endpoint: + * - id contains "veo" → ensure "videos" + * - id contains "imagen" → force "images" (never "videos") * * Note: `gemini-*-image` models (e.g. gemini-3-pro-image) generate images via the * regular `generateContent` path, so they stay "chat" (image output is a chat @@ -21,7 +25,8 @@ const METHOD_TO_ENDPOINT: Record = { generateContent: "chat", embedContent: "embeddings", - predictLongRunning: "video", + predict: "images", + predictLongRunning: "videos", bidiGenerateContent: "audio", generateAnswer: "chat", }; @@ -63,9 +68,14 @@ export function parseGeminiModelsList(data: any): GeminiDiscoveryModel[] { const id = ((m.name as string) || (m.id as string) || "").replace(/^models\//, ""); const lowerId = id.toLowerCase(); - // Keep Veo models in the video bucket even when the method list is incomplete. + // Google exposes Imagen (image) and Veo (video) via long-running methods; the + // method alone can't always distinguish them, so refine by model id. if (lowerId.includes("veo")) { - endpoints.add("video"); + endpoints.add("videos"); + } + if (lowerId.includes("imagen")) { + endpoints.delete("videos"); + endpoints.add("images"); } if (endpoints.size === 0) endpoints.add("chat"); diff --git a/src/lib/providers/staticModels.ts b/src/lib/providers/staticModels.ts index a62c90b110..9c86c8cc6f 100644 --- a/src/lib/providers/staticModels.ts +++ b/src/lib/providers/staticModels.ts @@ -221,7 +221,7 @@ export function getStaticModelsForProvider(provider: string): LocalCatalogModel[ if (speechProvider) { appendModels(speechProvider.models, { apiFormat: "audio", - supportedEndpoints: ["audio"], + supportedEndpoints: ["audio-speech"], }); } @@ -229,7 +229,7 @@ export function getStaticModelsForProvider(provider: string): LocalCatalogModel[ if (transcriptionProvider) { appendModels(transcriptionProvider.models, { apiFormat: "audio", - supportedEndpoints: ["audio"], + supportedEndpoints: ["audio-transcriptions"], }); } diff --git a/src/shared/constants/modelSupportedEndpoints.ts b/src/shared/constants/modelSupportedEndpoints.ts new file mode 100644 index 0000000000..6201c79b48 --- /dev/null +++ b/src/shared/constants/modelSupportedEndpoints.ts @@ -0,0 +1,54 @@ +export const MODEL_SUPPORTED_ENDPOINT_VALUES = [ + "chat", + "embeddings", + "rerank", + "images", + "videos", + "audio-speech", + "audio-transcriptions", + "images-generations", + // Persisted legacy values remain valid input and normalize on write/edit. + "video", + "audio", +] as const; + +export type ModelSupportedEndpoint = (typeof MODEL_SUPPORTED_ENDPOINT_VALUES)[number]; + +export function normalizeModelSupportedEndpoints(endpoints: readonly string[]): string[] { + const normalized: string[] = []; + const add = (endpoint: string) => { + if (!normalized.includes(endpoint)) normalized.push(endpoint); + }; + + for (const endpoint of endpoints) { + if (endpoint === "video") { + add("videos"); + } else if (endpoint === "audio") { + add("audio-speech"); + add("audio-transcriptions"); + } else { + add(endpoint); + } + } + return normalized; +} + +export function classifyModelSupportedEndpoints(endpoints: readonly string[]): { + type?: "embedding" | "rerank" | "image" | "video" | "audio"; + subtype?: "speech" | "transcription"; +} { + if (endpoints.includes("embeddings")) return { type: "embedding" }; + if (endpoints.includes("rerank")) return { type: "rerank" }; + if (endpoints.includes("images")) return { type: "image" }; + if (endpoints.includes("videos") || endpoints.includes("video")) return { type: "video" }; + + const supportsSpeech = endpoints.includes("audio-speech"); + const supportsTranscription = + endpoints.includes("audio-transcriptions") || endpoints.includes("audio"); + if (!supportsSpeech && !supportsTranscription) return {}; + if (supportsSpeech && !supportsTranscription) return { type: "audio", subtype: "speech" }; + if (supportsTranscription && !supportsSpeech) { + return { type: "audio", subtype: "transcription" }; + } + return { type: "audio" }; +} diff --git a/src/shared/validation/schemas/provider.ts b/src/shared/validation/schemas/provider.ts index 7e6cd052b4..a99ba7c947 100644 --- a/src/shared/validation/schemas/provider.ts +++ b/src/shared/validation/schemas/provider.ts @@ -6,6 +6,10 @@ import { import { SUPPORTED_BATCH_ENDPOINTS } from "@/shared/constants/batchEndpoints"; import { MAX_REQUEST_BODY_LIMIT_MB, MIN_REQUEST_BODY_LIMIT_MB } from "@/shared/constants/bodySize"; import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode"; +import { + MODEL_SUPPORTED_ENDPOINT_VALUES, + normalizeModelSupportedEndpoints, +} from "@/shared/constants/modelSupportedEndpoints"; import { providerAllowsOptionalApiKey } from "@/shared/constants/providers"; import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility"; import { @@ -238,22 +242,12 @@ export const providerModelMutationSchema = z.object({ "audio-transcriptions", "audio-speech", "images-generations", + "video", ]) .default("chat-completions"), supportedEndpoints: z - .array( - z.enum([ - "chat", - "embeddings", - "rerank", - "images", - "audio", - "audio-transcriptions", - "audio-speech", - "images-generations", - "videos", - ]) - ) + .array(z.enum(MODEL_SUPPORTED_ENDPOINT_VALUES)) + .transform(normalizeModelSupportedEndpoints) .default(["chat"]), // #2905: optional per-model wire format override for custom models (e.g. a // custom opencode-go model that must use the Anthropic Messages shape). diff --git a/tests/unit/gemini-models-parser.test.ts b/tests/unit/gemini-models-parser.test.ts index 0e07a6f7a1..8c9da3dac4 100644 --- a/tests/unit/gemini-models-parser.test.ts +++ b/tests/unit/gemini-models-parser.test.ts @@ -82,11 +82,11 @@ test("parseGeminiModelsList maps embedContent and bidiGenerateContent", () => { ]); }); -test("parseGeminiModelsList maps Veo predictLongRunning models to the video endpoint", () => { +test("parseGeminiModelsList maps Veo predictLongRunning models to the videos endpoint", () => { const models = parseGeminiModelsList(SAMPLE); const veo = models.find((m) => m.id === "veo-3.0-generate-001"); assert.ok(veo, "veo-3.0-generate-001 should be present"); - assert.deepEqual(veo!.supportedEndpoints, ["video"]); + assert.deepEqual(veo!.supportedEndpoints, ["videos"]); }); test("parseGeminiModelsList defaults to chat and tolerates empty/missing input", () => { diff --git a/tests/unit/model-supported-endpoints.test.ts b/tests/unit/model-supported-endpoints.test.ts new file mode 100644 index 0000000000..30d93f67fc --- /dev/null +++ b/tests/unit/model-supported-endpoints.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + classifyModelSupportedEndpoints, + MODEL_SUPPORTED_ENDPOINT_VALUES, + normalizeModelSupportedEndpoints, +} from "../../src/shared/constants/modelSupportedEndpoints.ts"; + +test("normalizes legacy video and audio metadata to operation-specific endpoint ids", () => { + assert.deepEqual(normalizeModelSupportedEndpoints(["chat", "video", "audio"]), [ + "chat", + "videos", + "audio-speech", + "audio-transcriptions", + ]); +}); + +test("deduplicates canonical endpoint ids while preserving order", () => { + assert.deepEqual( + normalizeModelSupportedEndpoints([ + "videos", + "video", + "audio-speech", + "audio", + "audio-transcriptions", + ]), + ["videos", "audio-speech", "audio-transcriptions"] + ); +}); + +test("exports operation-specific values accepted by model metadata", () => { + assert.ok(MODEL_SUPPORTED_ENDPOINT_VALUES.includes("videos")); + assert.ok(MODEL_SUPPORTED_ENDPOINT_VALUES.includes("audio-speech")); + assert.ok(MODEL_SUPPORTED_ENDPOINT_VALUES.includes("audio-transcriptions")); +}); + +test("preserves endpoint ids introduced by external discovery", () => { + assert.deepEqual(normalizeModelSupportedEndpoints(["responses", "video"]), [ + "responses", + "videos", + ]); +}); + +test("classifies operation-specific media endpoints for the model catalog", () => { + assert.deepEqual(classifyModelSupportedEndpoints(["videos"]), { type: "video" }); + assert.deepEqual(classifyModelSupportedEndpoints(["audio-speech"]), { + type: "audio", + subtype: "speech", + }); + assert.deepEqual(classifyModelSupportedEndpoints(["audio-transcriptions"]), { + type: "audio", + subtype: "transcription", + }); + assert.deepEqual(classifyModelSupportedEndpoints(["audio-speech", "audio-transcriptions"]), { + type: "audio", + }); +}); diff --git a/tests/unit/provider-model-endpoint-schema.test.ts b/tests/unit/provider-model-endpoint-schema.test.ts new file mode 100644 index 0000000000..145ab449c4 --- /dev/null +++ b/tests/unit/provider-model-endpoint-schema.test.ts @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { providerModelMutationSchema } from "../../src/shared/validation/schemas/provider.ts"; + +test("provider model mutations accept video and persist canonical operation endpoints", () => { + const parsed = providerModelMutationSchema.parse({ + provider: "example", + modelId: "media-model", + apiFormat: "video", + supportedEndpoints: ["video", "audio"], + }); + + assert.equal(parsed.apiFormat, "video"); + assert.deepEqual(parsed.supportedEndpoints, ["videos", "audio-speech", "audio-transcriptions"]); +}); diff --git a/tests/unit/static-model-operation-endpoints.test.ts b/tests/unit/static-model-operation-endpoints.test.ts new file mode 100644 index 0000000000..be9605afa4 --- /dev/null +++ b/tests/unit/static-model-operation-endpoints.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { getStaticModelsForProvider } from "../../src/lib/providers/staticModels.ts"; + +test("speech-only static models advertise the speech operation", () => { + const models = getStaticModelsForProvider("elevenlabs") || []; + + assert.ok(models.length > 0); + assert.ok(models.every((model) => model.supportedEndpoints?.includes("audio-speech"))); + assert.ok(models.every((model) => !model.supportedEndpoints?.includes("audio"))); +}); + +test("transcription-only static models advertise the transcription operation", () => { + const models = getStaticModelsForProvider("gladia") || []; + + assert.ok(models.length > 0); + assert.ok(models.every((model) => model.supportedEndpoints?.includes("audio-transcriptions"))); + assert.ok(models.every((model) => !model.supportedEndpoints?.includes("audio"))); +});