From 02f1ff413540cd761e72251baa467e6c2576a25a Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 15 Aug 2026 01:13:21 -0300 Subject: [PATCH] feat(video-bridge): add media capability contracts --- open-sse/config/providerPluginManifest.ts | 22 +++----- open-sse/config/providers/shared.ts | 1 + open-sse/utils/mediaParts.ts | 49 +++++++++++++++- src/lib/modelCapabilities.ts | 34 +++++------ src/lib/modelCapabilityModalities.ts | 47 ++++++++++++++++ .../constants/modalityBridgeDefaults.ts | 38 +++++++++++++ src/shared/constants/modelSpecs.ts | 1 + src/shared/validation/settingsSchemas.ts | 5 ++ .../video-bridge-media-capabilities.test.ts | 55 ++++++++++++++++++ tests/unit/video-bridge-settings.test.ts | 56 +++++++++++++++++++ 10 files changed, 273 insertions(+), 35 deletions(-) create mode 100644 src/lib/modelCapabilityModalities.ts create mode 100644 tests/unit/video-bridge-media-capabilities.test.ts create mode 100644 tests/unit/video-bridge-settings.test.ts diff --git a/open-sse/config/providerPluginManifest.ts b/open-sse/config/providerPluginManifest.ts index 6221b245a8..b0925b126f 100644 --- a/open-sse/config/providerPluginManifest.ts +++ b/open-sse/config/providerPluginManifest.ts @@ -1,12 +1,7 @@ import type { RegistryEntry, RegistryModel } from "./providers/shared.ts"; export type ProviderPluginCapability = - | "apikey" - | "custom-executor" - | "oauth" - | "passthrough-models" - | "responses" - | "sidecar-candidate"; + "apikey" | "custom-executor" | "oauth" | "passthrough-models" | "responses" | "sidecar-candidate"; export interface ProviderPluginModel { id: string; @@ -16,6 +11,7 @@ export interface ProviderPluginModel { toolCalling?: boolean; supportsReasoning?: boolean; supportsVision?: boolean; + supportsVideo?: boolean; unsupportedParams?: readonly string[]; targetFormat?: string; } @@ -58,7 +54,7 @@ const SIDECAR_COMPATIBLE_EXECUTORS = new Set(["default"]); function compactObject>(value: T): Partial { return Object.fromEntries( - Object.entries(value).filter(([, entryValue]) => entryValue !== undefined), + Object.entries(value).filter(([, entryValue]) => entryValue !== undefined) ) as Partial; } @@ -71,6 +67,7 @@ function mapModel(model: RegistryModel): ProviderPluginModel { toolCalling: model.toolCalling, supportsReasoning: model.supportsReasoning, supportsVision: model.supportsVision, + supportsVideo: model.supportsVideo, unsupportedParams: model.unsupportedParams, targetFormat: model.targetFormat, }) as ProviderPluginModel; @@ -130,7 +127,7 @@ function capabilitiesFor(entry: RegistryEntry, eligible: boolean): ProviderPlugi } export function createProviderPluginManifestEntry( - entry: RegistryEntry, + entry: RegistryEntry ): ProviderPluginManifestEntry { const sidecar = sidecarEligibility(entry); @@ -163,7 +160,7 @@ export function createProviderPluginManifestEntry( } export function generateProviderPluginManifestFromRegistry( - registry: Record, + registry: Record ): ProviderPluginManifest { return { schemaVersion: 1, @@ -191,7 +188,7 @@ export function createServiceBackendManifestEntry( template: Pick< ProviderPluginManifestEntry, "format" | "executor" | "auth" | "endpoints" | "capabilities" | "passthroughModels" | "sidecar" - >, + > ): ProviderPluginManifestEntry { return { id: pluginId, @@ -202,11 +199,10 @@ export function createServiceBackendManifestEntry( export function getProviderPluginManifestEntryFromRegistry( registry: Record, - provider: string, + provider: string ): ProviderPluginManifestEntry | null { const entry = - registry[provider] || - Object.values(registry).find((candidate) => candidate.alias === provider); + registry[provider] || Object.values(registry).find((candidate) => candidate.alias === provider); return entry ? createProviderPluginManifestEntry(entry) : null; } diff --git a/open-sse/config/providers/shared.ts b/open-sse/config/providers/shared.ts index 1198862b18..5696ecf08f 100644 --- a/open-sse/config/providers/shared.ts +++ b/open-sse/config/providers/shared.ts @@ -51,6 +51,7 @@ export interface RegistryModel { supportedThinkingEfforts?: readonly string[]; supportsVision?: boolean; supportsAudio?: boolean; + supportsVideo?: boolean; supportsXHighEffort?: boolean; maxOutputTokens?: number; targetFormat?: string; diff --git a/open-sse/utils/mediaParts.ts b/open-sse/utils/mediaParts.ts index 1a52ef003d..99c3f6d68e 100644 --- a/open-sse/utils/mediaParts.ts +++ b/open-sse/utils/mediaParts.ts @@ -4,7 +4,7 @@ * and the combo compatibility filter (open-sse/) — the two previously kept * divergent copies (guardrail missed input_image; combo saw it). */ -export type MediaKind = "image" | "audio"; +export type MediaKind = "image" | "audio" | "video"; export interface MediaPart { kind: MediaKind; @@ -36,6 +36,9 @@ export interface MediaPart { | "audio_url" /** Audio detected via `source.media_type: audio/*` (no explicit type). */ | "audio_source" + | "input_video" + | "video_url" + | "video_source" /** * Combo-parity indicator: the value looks like an image part (image-ish * `type` in any casing, a bare `image_url`/`input_image` key, or a @@ -149,6 +152,48 @@ function inspectAudioShapes( return false; } +/** Strict video shapes with an extractable URL, data URI, or base64 ref. */ +function inspectVideoShapes( + obj: Record, + type: string | undefined, + mediaType: unknown, + ctx: DetectCtx, + depth: number +): boolean { + if (type === "input_video") { + const ref = urlFrom(obj.video_url ?? obj.input_video ?? obj.url); + if (ref) { + pushPart(ctx, "video", ref, "input_video", depth); + return true; + } + } + if (type === "video_url") { + const ref = urlFrom(obj.video_url); + if (ref) { + pushPart(ctx, "video", ref, "video_url", depth); + return true; + } + } + const source = obj.source as Record | undefined; + if ( + (type === "video_source" || + (typeof mediaType === "string" && mediaType.toLowerCase().startsWith("video/"))) && + source + ) { + if (typeof source.data === "string") { + const mime = typeof mediaType === "string" ? mediaType : "video/mp4"; + pushPart(ctx, "video", `data:${mime};base64,${source.data}`, "video_source", depth); + return true; + } + const ref = urlFrom(source.url); + if (ref) { + pushPart(ctx, "video", ref, "video_source", depth); + return true; + } + } + return false; +} + /** * Combo-parity image indicators: the legacy valueContainsImagePart * (comboStructure) matched image-ish `type` names case-insensitively, bare @@ -182,6 +227,7 @@ function inspect(value: unknown, ctx: DetectCtx, depth: number): void { if (ctx.found || depth > MAX_DEPTH || value == null) return; if (typeof value === "string") { if (value.startsWith("data:image/")) pushPart(ctx, "image", value, "data_uri_string", depth); + if (value.startsWith("data:video/")) pushPart(ctx, "video", value, "data_uri_string", depth); return; } if (Array.isArray(value)) { @@ -203,6 +249,7 @@ function inspect(value: unknown, ctx: DetectCtx, depth: number): void { // matched) or nest image parts inside its payload. inspectAudioShapes(obj, type, mediaType, ctx, depth); if (ctx.found) return; + if (inspectVideoShapes(obj, type, mediaType, ctx, depth)) return; if (inspectImageIndicators(obj, type, mediaType, ctx, depth)) return; for (const nested of Object.values(obj)) { inspect(nested, ctx, depth + 1); diff --git a/src/lib/modelCapabilities.ts b/src/lib/modelCapabilities.ts index a4416925b0..38acf56476 100644 --- a/src/lib/modelCapabilities.ts +++ b/src/lib/modelCapabilities.ts @@ -17,9 +17,11 @@ import { getModelCapabilityOverride } from "@/lib/db/modelCapabilityOverrides"; import { getDbInstance } from "@/lib/db/core"; import { getKeyValue } from "@/lib/db/models/shared"; import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; +import { resolveAudioCapability, resolveVideoCapability } from "@/lib/modelCapabilityModalities"; export type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; export { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot"; +export { resolveAudioCapability, resolveVideoCapability } from "@/lib/modelCapabilityModalities"; import { isVisionModelId } from "@/shared/constants/visionModels"; import { getUnsupportedParams } from "@omniroute/open-sse/config/providerRegistry.ts"; import { @@ -126,6 +128,7 @@ export interface ResolvedModelCapabilities { supportsTools: boolean | null; supportsVision: boolean | null; supportsAudio: boolean | null; + supportsVideo: boolean | null; supportsMaxTokens: boolean; attachment: boolean | null; structuredOutput: boolean | null; @@ -548,25 +551,6 @@ function resolveVisionCapability( return null; } -/** - * Resolve whether a chat model accepts audio input. - * - * Explicit catalog metadata wins. Synced input modalities are authoritative - * only when they contain at least one declared modality; an empty list means - * that no source knows the answer and remains `null` so Audio Bridge can act - * conservatively. - */ -export function resolveAudioCapability( - spec: Pick | undefined, - registryModel: { supportsAudio?: boolean } | null, - modalitiesInput: readonly string[] -): boolean | null { - if (typeof registryModel?.supportsAudio === "boolean") return registryModel.supportsAudio; - if (typeof spec?.supportsAudio === "boolean") return spec.supportsAudio; - if (modalitiesInput.length === 0) return null; - return modalitiesInput.some((entry) => String(entry).toLowerCase().includes("audio")); -} - /** * Issue #6524: an operator-set `max_output_tokens` capability override (see * `src/lib/db/modelCapabilityOverrides.ts`) is the manual escape hatch for a @@ -768,7 +752,9 @@ export function getResolvedModelCapabilities( // reflects the real *total* window and wins over every static/synced source. // `maxInputTokens` still follows its own precedence chain; only when that // chain has no narrower source does it naturally fall back to this window. - const persistedContextWindow = usePersistedOverrides ? getContextOverride(resolved, snapshot) : null; + const persistedContextWindow = usePersistedOverrides + ? getContextOverride(resolved, snapshot) + : null; const contextWindow = persistedContextWindow ?? authoritativeContextWindow ?? @@ -809,6 +795,7 @@ export function getResolvedModelCapabilities( customVisionOverride ); const supportsAudio = resolveAudioCapability(spec, registryModel, modalitiesInput); + const supportsVideo = resolveVideoCapability(spec, registryModel, modalitiesInput); // #8250: when resolve promoted vision over a contradictory attachment=false, // expose attachment=true so catalog / Vision Bridge / clients see one verdict. @@ -827,6 +814,7 @@ export function getResolvedModelCapabilities( supportsTools, supportsVision, supportsAudio, + supportsVideo, supportsMaxTokens: heuristicMaxTokens(lookupKey), attachment, structuredOutput: synced?.structured_output ?? null, @@ -991,7 +979,11 @@ export function getModelContextLimit( ): number | null { const resolved = typeof providerOrInput === "string" && modelId !== undefined - ? getResolvedModelCapabilities({ provider: providerOrInput, model: modelId }, undefined, snapshot) + ? getResolvedModelCapabilities( + { provider: providerOrInput, model: modelId }, + undefined, + snapshot + ) : getResolvedModelCapabilities(providerOrInput, undefined, snapshot); // Feature 5004: a persisted override (operator-set or auto-discovered) wins over the // static catalog / models.dev sync. `getResolvedModelCapabilities` stays override-free diff --git a/src/lib/modelCapabilityModalities.ts b/src/lib/modelCapabilityModalities.ts new file mode 100644 index 0000000000..8cb528b674 --- /dev/null +++ b/src/lib/modelCapabilityModalities.ts @@ -0,0 +1,47 @@ +import type { ModelSpec } from "@/shared/constants/modelSpecs"; + +type ModalityName = "audio" | "video"; +type ModalityCapabilityKey = "supportsAudio" | "supportsVideo"; + +function resolveInputModalityCapability( + modality: ModalityName, + capabilityKey: ModalityCapabilityKey, + spec: Pick | undefined, + registryModel: Partial> | null, + modalitiesInput: readonly string[] +): boolean | null { + const registryValue = registryModel?.[capabilityKey]; + if (typeof registryValue === "boolean") return registryValue; + const specValue = spec?.[capabilityKey]; + if (typeof specValue === "boolean") return specValue; + if (modalitiesInput.length === 0) return null; + return modalitiesInput.some((entry) => String(entry).toLowerCase().includes(modality)); +} + +export function resolveAudioCapability( + spec: Pick | undefined, + registryModel: { supportsAudio?: boolean } | null, + modalitiesInput: readonly string[] +): boolean | null { + return resolveInputModalityCapability( + "audio", + "supportsAudio", + spec, + registryModel, + modalitiesInput + ); +} + +export function resolveVideoCapability( + spec: Pick | undefined, + registryModel: { supportsVideo?: boolean } | null, + modalitiesInput: readonly string[] +): boolean | null { + return resolveInputModalityCapability( + "video", + "supportsVideo", + spec, + registryModel, + modalitiesInput + ); +} diff --git a/src/shared/constants/modalityBridgeDefaults.ts b/src/shared/constants/modalityBridgeDefaults.ts index e0dcc0aab8..0cd3f15ced 100644 --- a/src/shared/constants/modalityBridgeDefaults.ts +++ b/src/shared/constants/modalityBridgeDefaults.ts @@ -21,6 +21,11 @@ export const MODALITY_BRIDGE_DEFAULTS = { audioModel: "", audioTimeoutMs: 60000, audioMaxClips: 3, + videoEnabled: false, + videoModel: "", + videoFrameCount: 8, + videoMaxVideos: 1, + videoTimeoutMs: 120000, } as const; export interface VisionBridgeRuntimeSettings { @@ -47,6 +52,17 @@ export interface AudioBridgeRuntimeSettings { cacheMaxEntries: number; } +export interface VideoBridgeRuntimeSettings { + enabled: boolean; + model: string; + frameCount: number; + maxVideos: number; + timeoutMs: number; + cacheEnabled: boolean; + cacheTtlMinutes: number; + cacheMaxEntries: number; +} + // Typed candidate pickers: a stored value of the wrong type (e.g. the string // "off" in a boolean field) is skipped so the next candidate/default wins. function pickBoolean(...values: unknown[]): boolean | undefined { @@ -116,3 +132,25 @@ export function resolveAudioBridgeRuntimeSettings( pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries, }; } + +/** Resolve persisted Video Bridge settings with safe, bounded defaults. */ +export function resolveVideoBridgeRuntimeSettings( + settings: Record | null | undefined +): VideoBridgeRuntimeSettings { + const s = settings ?? {}; + return { + enabled: pickBoolean(s.modalityBridgeVideoEnabled) ?? MODALITY_BRIDGE_DEFAULTS.videoEnabled, + model: pickString(s.modalityBridgeVideoModel) ?? MODALITY_BRIDGE_DEFAULTS.videoModel, + frameCount: + pickNumber(s.modalityBridgeVideoFrameCount) ?? MODALITY_BRIDGE_DEFAULTS.videoFrameCount, + maxVideos: + pickNumber(s.modalityBridgeVideoMaxVideos) ?? MODALITY_BRIDGE_DEFAULTS.videoMaxVideos, + timeoutMs: pickNumber(s.modalityBridgeVideoTimeout) ?? MODALITY_BRIDGE_DEFAULTS.videoTimeoutMs, + cacheEnabled: + pickBoolean(s.modalityBridgeCacheEnabled) ?? MODALITY_BRIDGE_DEFAULTS.cacheEnabled, + cacheTtlMinutes: + pickNumber(s.modalityBridgeCacheTtlMinutes) ?? MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes, + cacheMaxEntries: + pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries, + }; +} diff --git a/src/shared/constants/modelSpecs.ts b/src/shared/constants/modelSpecs.ts index d15fe50425..ebf1f2ca1d 100644 --- a/src/shared/constants/modelSpecs.ts +++ b/src/shared/constants/modelSpecs.ts @@ -16,6 +16,7 @@ export interface ModelSpec { supportsTools?: boolean; supportsVision?: boolean; supportsAudio?: boolean; + supportsVideo?: boolean; // Model defaults to adaptive thinking and REJECTS an explicit `thinking.type:"disabled"` // (upstream returns 400). Used to normalize the request when a combo/route substitutes // this model after the client already chose `disabled`. See issue #3554. diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index ac548aa7d5..94240f5aed 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -356,6 +356,11 @@ export const updateSettingsSchema = z.object({ modalityBridgeAudioModel: z.string().max(200).optional(), modalityBridgeAudioTimeout: z.number().int().min(1000).max(300000).optional(), modalityBridgeAudioMaxClips: z.number().int().min(1).max(10).optional(), + modalityBridgeVideoEnabled: z.boolean().optional(), + modalityBridgeVideoModel: z.string().max(200).optional(), + modalityBridgeVideoFrameCount: z.number().int().min(1).max(16).optional(), + modalityBridgeVideoMaxVideos: z.number().int().min(1).max(4).optional(), + modalityBridgeVideoTimeout: z.number().int().min(1000).max(300000).optional(), modalityBridgeCacheEnabled: z.boolean().optional(), modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(), modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(), diff --git a/tests/unit/video-bridge-media-capabilities.test.ts b/tests/unit/video-bridge-media-capabilities.test.ts new file mode 100644 index 0000000000..a0a4ff3613 --- /dev/null +++ b/tests/unit/video-bridge-media-capabilities.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { containsMediaKind, detectMediaParts } from "../../open-sse/utils/mediaParts.ts"; + +const messages = (content: unknown) => [{ role: "user", content }]; + +test("detects supported Chat and Responses video blocks without treating plain text as video", () => { + const detected = detectMediaParts( + messages([ + { type: "input_video", video_url: "https://media.example/input.mp4" }, + { type: "video_url", video_url: { url: "https://media.example/url.webm" } }, + { + type: "video_source", + source: { type: "base64", media_type: "video/mp4", data: "QUJD" }, + }, + { type: "input_text", text: "Please inspect demo.mp4 but do not fetch it." }, + ]) + ); + + assert.deepEqual( + detected.map(({ kind, ref, shape, nested }) => ({ kind, ref, shape, nested })), + [ + { + kind: "video", + ref: "https://media.example/input.mp4", + shape: "input_video", + nested: false, + }, + { + kind: "video", + ref: "https://media.example/url.webm", + shape: "video_url", + nested: false, + }, + { + kind: "video", + ref: "data:video/mp4;base64,QUJD", + shape: "video_source", + nested: false, + }, + ] + ); + assert.equal( + containsMediaKind(messages([{ type: "input_text", text: "demo.mp4" }]), "video"), + false + ); + assert.equal( + containsMediaKind( + messages([{ type: "input_video", video_url: "data:video/mp4;base64,QUJD" }]), + "video" + ), + true + ); +}); diff --git a/tests/unit/video-bridge-settings.test.ts b/tests/unit/video-bridge-settings.test.ts new file mode 100644 index 0000000000..089660945f --- /dev/null +++ b/tests/unit/video-bridge-settings.test.ts @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + resolveAudioCapability, + resolveVideoCapability, +} from "../../src/lib/modelCapabilityModalities.ts"; +import { + MODALITY_BRIDGE_DEFAULTS, + resolveVideoBridgeRuntimeSettings, +} from "../../src/shared/constants/modalityBridgeDefaults.ts"; +import { updateSettingsSchema } from "../../src/shared/validation/settingsSchemas.ts"; + +test("audio and video capability resolution trusts explicit catalog data before modalities", () => { + assert.equal(resolveAudioCapability({ supportsAudio: false }, null, ["audio"]), false); + assert.equal(resolveVideoCapability(undefined, { supportsVideo: true }, ["text"]), true); + assert.equal(resolveVideoCapability(undefined, null, ["text", "video"]), true); + assert.equal(resolveVideoCapability(undefined, null, ["text"]), false); + assert.equal(resolveVideoCapability(undefined, null, []), null); +}); + +test("Video Bridge settings default to a bounded disabled runtime and accept valid overrides", () => { + assert.deepEqual(resolveVideoBridgeRuntimeSettings({}), { + enabled: false, + model: "", + frameCount: 8, + maxVideos: 1, + timeoutMs: 120_000, + cacheEnabled: MODALITY_BRIDGE_DEFAULTS.cacheEnabled, + cacheTtlMinutes: MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes, + cacheMaxEntries: MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries, + }); + + const valid = updateSettingsSchema.safeParse({ + modalityBridgeVideoEnabled: true, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + modalityBridgeVideoFrameCount: 16, + modalityBridgeVideoMaxVideos: 4, + modalityBridgeVideoTimeout: 120_000, + }); + assert.equal(valid.success, true); +}); + +test("Video Bridge settings schema rejects values outside extraction bounds", () => { + for (const [field, value] of Object.entries({ + modalityBridgeVideoFrameCount: 17, + modalityBridgeVideoMaxVideos: 0, + modalityBridgeVideoTimeout: 300_001, + })) { + assert.equal( + updateSettingsSchema.safeParse({ [field]: value }).success, + false, + `${field}=${value} should be rejected` + ); + } +});