feat(video-bridge): add media capability contracts

open-sse/config/providerPluginManifest.ts

@@ -1,12 +1,7 @@
 import type { RegistryEntry, RegistryModel } from "./providers/shared.ts";
 
 export type ProviderPluginCapability =
-  | "apikey"
-  | "custom-executor"
-  | "oauth"
-  | "passthrough-models"
-  | "responses"
-  | "sidecar-candidate";
+  "apikey" | "custom-executor" | "oauth" | "passthrough-models" | "responses" | "sidecar-candidate";
 
 export interface ProviderPluginModel {
   id: string;
@@ -16,6 +11,7 @@ export interface ProviderPluginModel {
   toolCalling?: boolean;
   supportsReasoning?: boolean;
   supportsVision?: boolean;
+  supportsVideo?: boolean;
   unsupportedParams?: readonly string[];
   targetFormat?: string;
 }
@@ -58,7 +54,7 @@ const SIDECAR_COMPATIBLE_EXECUTORS = new Set(["default"]);
 
 function compactObject<T extends Record<string, unknown>>(value: T): Partial<T> {
   return Object.fromEntries(
-    Object.entries(value).filter(([, entryValue]) => entryValue !== undefined),
+    Object.entries(value).filter(([, entryValue]) => entryValue !== undefined)
   ) as Partial<T>;
 }
 
@@ -71,6 +67,7 @@ function mapModel(model: RegistryModel): ProviderPluginModel {
     toolCalling: model.toolCalling,
     supportsReasoning: model.supportsReasoning,
     supportsVision: model.supportsVision,
+    supportsVideo: model.supportsVideo,
     unsupportedParams: model.unsupportedParams,
     targetFormat: model.targetFormat,
   }) as ProviderPluginModel;
@@ -130,7 +127,7 @@ function capabilitiesFor(entry: RegistryEntry, eligible: boolean): ProviderPlugi
 }
 
 export function createProviderPluginManifestEntry(
-  entry: RegistryEntry,
+  entry: RegistryEntry
 ): ProviderPluginManifestEntry {
   const sidecar = sidecarEligibility(entry);
 
@@ -163,7 +160,7 @@ export function createProviderPluginManifestEntry(
 }
 
 export function generateProviderPluginManifestFromRegistry(
-  registry: Record<string, RegistryEntry>,
+  registry: Record<string, RegistryEntry>
 ): ProviderPluginManifest {
   return {
     schemaVersion: 1,
@@ -191,7 +188,7 @@ export function createServiceBackendManifestEntry(
   template: Pick<
     ProviderPluginManifestEntry,
     "format" | "executor" | "auth" | "endpoints" | "capabilities" | "passthroughModels" | "sidecar"
-  >,
+  >
 ): ProviderPluginManifestEntry {
   return {
     id: pluginId,
@@ -202,11 +199,10 @@ export function createServiceBackendManifestEntry(
 
 export function getProviderPluginManifestEntryFromRegistry(
   registry: Record<string, RegistryEntry>,
-  provider: string,
+  provider: string
 ): ProviderPluginManifestEntry | null {
   const entry =
-    registry[provider] ||
-    Object.values(registry).find((candidate) => candidate.alias === provider);
+    registry[provider] || Object.values(registry).find((candidate) => candidate.alias === provider);
 
   return entry ? createProviderPluginManifestEntry(entry) : null;
 }

open-sse/config/providers/shared.ts

@@ -51,6 +51,7 @@ export interface RegistryModel {
   supportedThinkingEfforts?: readonly string[];
   supportsVision?: boolean;
   supportsAudio?: boolean;
+  supportsVideo?: boolean;
   supportsXHighEffort?: boolean;
   maxOutputTokens?: number;
   targetFormat?: string;

open-sse/utils/mediaParts.ts

@@ -4,7 +4,7 @@
  * and the combo compatibility filter (open-sse/) — the two previously kept
  * divergent copies (guardrail missed input_image; combo saw it).
  */
-export type MediaKind = "image" | "audio";
+export type MediaKind = "image" | "audio" | "video";
 
 export interface MediaPart {
   kind: MediaKind;
@@ -36,6 +36,9 @@ export interface MediaPart {
     | "audio_url"
     /** Audio detected via `source.media_type: audio/*` (no explicit type). */
     | "audio_source"
+    | "input_video"
+    | "video_url"
+    | "video_source"
     /**
      * Combo-parity indicator: the value looks like an image part (image-ish
      * `type` in any casing, a bare `image_url`/`input_image` key, or a
@@ -149,6 +152,48 @@ function inspectAudioShapes(
   return false;
 }
 
+/** Strict video shapes with an extractable URL, data URI, or base64 ref. */
+function inspectVideoShapes(
+  obj: Record<string, unknown>,
+  type: string | undefined,
+  mediaType: unknown,
+  ctx: DetectCtx,
+  depth: number
+): boolean {
+  if (type === "input_video") {
+    const ref = urlFrom(obj.video_url ?? obj.input_video ?? obj.url);
+    if (ref) {
+      pushPart(ctx, "video", ref, "input_video", depth);
+      return true;
+    }
+  }
+  if (type === "video_url") {
+    const ref = urlFrom(obj.video_url);
+    if (ref) {
+      pushPart(ctx, "video", ref, "video_url", depth);
+      return true;
+    }
+  }
+  const source = obj.source as Record<string, unknown> | undefined;
+  if (
+    (type === "video_source" ||
+      (typeof mediaType === "string" && mediaType.toLowerCase().startsWith("video/"))) &&
+    source
+  ) {
+    if (typeof source.data === "string") {
+      const mime = typeof mediaType === "string" ? mediaType : "video/mp4";
+      pushPart(ctx, "video", `data:${mime};base64,${source.data}`, "video_source", depth);
+      return true;
+    }
+    const ref = urlFrom(source.url);
+    if (ref) {
+      pushPart(ctx, "video", ref, "video_source", depth);
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Combo-parity image indicators: the legacy valueContainsImagePart
  * (comboStructure) matched image-ish `type` names case-insensitively, bare
@@ -182,6 +227,7 @@ function inspect(value: unknown, ctx: DetectCtx, depth: number): void {
   if (ctx.found || depth > MAX_DEPTH || value == null) return;
   if (typeof value === "string") {
     if (value.startsWith("data:image/")) pushPart(ctx, "image", value, "data_uri_string", depth);
+    if (value.startsWith("data:video/")) pushPart(ctx, "video", value, "data_uri_string", depth);
     return;
   }
   if (Array.isArray(value)) {
@@ -203,6 +249,7 @@ function inspect(value: unknown, ctx: DetectCtx, depth: number): void {
   // matched) or nest image parts inside its payload.
   inspectAudioShapes(obj, type, mediaType, ctx, depth);
   if (ctx.found) return;
+  if (inspectVideoShapes(obj, type, mediaType, ctx, depth)) return;
   if (inspectImageIndicators(obj, type, mediaType, ctx, depth)) return;
   for (const nested of Object.values(obj)) {
     inspect(nested, ctx, depth + 1);

src/lib/modelCapabilities.ts

@@ -17,9 +17,11 @@ import { getModelCapabilityOverride } from "@/lib/db/modelCapabilityOverrides";
 import { getDbInstance } from "@/lib/db/core";
 import { getKeyValue } from "@/lib/db/models/shared";
 import type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
+import { resolveAudioCapability, resolveVideoCapability } from "@/lib/modelCapabilityModalities";
 
 export type { ModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
 export { createModelCapabilityResolutionSnapshot } from "@/lib/modelCapabilityResolutionSnapshot";
+export { resolveAudioCapability, resolveVideoCapability } from "@/lib/modelCapabilityModalities";
 import { isVisionModelId } from "@/shared/constants/visionModels";
 import { getUnsupportedParams } from "@omniroute/open-sse/config/providerRegistry.ts";
 import {
@@ -126,6 +128,7 @@ export interface ResolvedModelCapabilities {
   supportsTools: boolean | null;
   supportsVision: boolean | null;
   supportsAudio: boolean | null;
+  supportsVideo: boolean | null;
   supportsMaxTokens: boolean;
   attachment: boolean | null;
   structuredOutput: boolean | null;
@@ -548,25 +551,6 @@ function resolveVisionCapability(
   return null;
 }
 
-/**
- * Resolve whether a chat model accepts audio input.
- *
- * Explicit catalog metadata wins. Synced input modalities are authoritative
- * only when they contain at least one declared modality; an empty list means
- * that no source knows the answer and remains `null` so Audio Bridge can act
- * conservatively.
- */
-export function resolveAudioCapability(
-  spec: Pick<ModelSpec, "supportsAudio"> | undefined,
-  registryModel: { supportsAudio?: boolean } | null,
-  modalitiesInput: readonly string[]
-): boolean | null {
-  if (typeof registryModel?.supportsAudio === "boolean") return registryModel.supportsAudio;
-  if (typeof spec?.supportsAudio === "boolean") return spec.supportsAudio;
-  if (modalitiesInput.length === 0) return null;
-  return modalitiesInput.some((entry) => String(entry).toLowerCase().includes("audio"));
-}
-
 /**
  * Issue #6524: an operator-set `max_output_tokens` capability override (see
  * `src/lib/db/modelCapabilityOverrides.ts`) is the manual escape hatch for a
@@ -768,7 +752,9 @@ export function getResolvedModelCapabilities(
   // reflects the real *total* window and wins over every static/synced source.
   // `maxInputTokens` still follows its own precedence chain; only when that
   // chain has no narrower source does it naturally fall back to this window.
-  const persistedContextWindow = usePersistedOverrides ? getContextOverride(resolved, snapshot) : null;
+  const persistedContextWindow = usePersistedOverrides
+    ? getContextOverride(resolved, snapshot)
+    : null;
   const contextWindow =
     persistedContextWindow ??
     authoritativeContextWindow ??
@@ -809,6 +795,7 @@ export function getResolvedModelCapabilities(
     customVisionOverride
   );
   const supportsAudio = resolveAudioCapability(spec, registryModel, modalitiesInput);
+  const supportsVideo = resolveVideoCapability(spec, registryModel, modalitiesInput);
 
   // #8250: when resolve promoted vision over a contradictory attachment=false,
   // expose attachment=true so catalog / Vision Bridge / clients see one verdict.
@@ -827,6 +814,7 @@ export function getResolvedModelCapabilities(
     supportsTools,
     supportsVision,
     supportsAudio,
+    supportsVideo,
     supportsMaxTokens: heuristicMaxTokens(lookupKey),
     attachment,
     structuredOutput: synced?.structured_output ?? null,
@@ -991,7 +979,11 @@ export function getModelContextLimit(
 ): number | null {
   const resolved =
     typeof providerOrInput === "string" && modelId !== undefined
-      ? getResolvedModelCapabilities({ provider: providerOrInput, model: modelId }, undefined, snapshot)
+      ? getResolvedModelCapabilities(
+          { provider: providerOrInput, model: modelId },
+          undefined,
+          snapshot
+        )
       : getResolvedModelCapabilities(providerOrInput, undefined, snapshot);
   // Feature 5004: a persisted override (operator-set or auto-discovered) wins over the
   // static catalog / models.dev sync. `getResolvedModelCapabilities` stays override-free

src/lib/modelCapabilityModalities.ts

@@ -0,0 +1,47 @@
+import type { ModelSpec } from "@/shared/constants/modelSpecs";
+
+type ModalityName = "audio" | "video";
+type ModalityCapabilityKey = "supportsAudio" | "supportsVideo";
+
+function resolveInputModalityCapability(
+  modality: ModalityName,
+  capabilityKey: ModalityCapabilityKey,
+  spec: Pick<ModelSpec, ModalityCapabilityKey> | undefined,
+  registryModel: Partial<Record<ModalityCapabilityKey, boolean>> | null,
+  modalitiesInput: readonly string[]
+): boolean | null {
+  const registryValue = registryModel?.[capabilityKey];
+  if (typeof registryValue === "boolean") return registryValue;
+  const specValue = spec?.[capabilityKey];
+  if (typeof specValue === "boolean") return specValue;
+  if (modalitiesInput.length === 0) return null;
+  return modalitiesInput.some((entry) => String(entry).toLowerCase().includes(modality));
+}
+
+export function resolveAudioCapability(
+  spec: Pick<ModelSpec, "supportsAudio"> | undefined,
+  registryModel: { supportsAudio?: boolean } | null,
+  modalitiesInput: readonly string[]
+): boolean | null {
+  return resolveInputModalityCapability(
+    "audio",
+    "supportsAudio",
+    spec,
+    registryModel,
+    modalitiesInput
+  );
+}
+
+export function resolveVideoCapability(
+  spec: Pick<ModelSpec, "supportsVideo"> | undefined,
+  registryModel: { supportsVideo?: boolean } | null,
+  modalitiesInput: readonly string[]
+): boolean | null {
+  return resolveInputModalityCapability(
+    "video",
+    "supportsVideo",
+    spec,
+    registryModel,
+    modalitiesInput
+  );
+}

src/shared/constants/modalityBridgeDefaults.ts

@@ -21,6 +21,11 @@ export const MODALITY_BRIDGE_DEFAULTS = {
   audioModel: "",
   audioTimeoutMs: 60000,
   audioMaxClips: 3,
+  videoEnabled: false,
+  videoModel: "",
+  videoFrameCount: 8,
+  videoMaxVideos: 1,
+  videoTimeoutMs: 120000,
 } as const;
 
 export interface VisionBridgeRuntimeSettings {
@@ -47,6 +52,17 @@ export interface AudioBridgeRuntimeSettings {
   cacheMaxEntries: number;
 }
 
+export interface VideoBridgeRuntimeSettings {
+  enabled: boolean;
+  model: string;
+  frameCount: number;
+  maxVideos: number;
+  timeoutMs: number;
+  cacheEnabled: boolean;
+  cacheTtlMinutes: number;
+  cacheMaxEntries: number;
+}
+
 // Typed candidate pickers: a stored value of the wrong type (e.g. the string
 // "off" in a boolean field) is skipped so the next candidate/default wins.
 function pickBoolean(...values: unknown[]): boolean | undefined {
@@ -116,3 +132,25 @@ export function resolveAudioBridgeRuntimeSettings(
       pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries,
   };
 }
+
+/** Resolve persisted Video Bridge settings with safe, bounded defaults. */
+export function resolveVideoBridgeRuntimeSettings(
+  settings: Record<string, unknown> | null | undefined
+): VideoBridgeRuntimeSettings {
+  const s = settings ?? {};
+  return {
+    enabled: pickBoolean(s.modalityBridgeVideoEnabled) ?? MODALITY_BRIDGE_DEFAULTS.videoEnabled,
+    model: pickString(s.modalityBridgeVideoModel) ?? MODALITY_BRIDGE_DEFAULTS.videoModel,
+    frameCount:
+      pickNumber(s.modalityBridgeVideoFrameCount) ?? MODALITY_BRIDGE_DEFAULTS.videoFrameCount,
+    maxVideos:
+      pickNumber(s.modalityBridgeVideoMaxVideos) ?? MODALITY_BRIDGE_DEFAULTS.videoMaxVideos,
+    timeoutMs: pickNumber(s.modalityBridgeVideoTimeout) ?? MODALITY_BRIDGE_DEFAULTS.videoTimeoutMs,
+    cacheEnabled:
+      pickBoolean(s.modalityBridgeCacheEnabled) ?? MODALITY_BRIDGE_DEFAULTS.cacheEnabled,
+    cacheTtlMinutes:
+      pickNumber(s.modalityBridgeCacheTtlMinutes) ?? MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes,
+    cacheMaxEntries:
+      pickNumber(s.modalityBridgeCacheMaxEntries) ?? MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries,
+  };
+}

src/shared/constants/modelSpecs.ts

@@ -16,6 +16,7 @@ export interface ModelSpec {
   supportsTools?: boolean;
   supportsVision?: boolean;
   supportsAudio?: boolean;
+  supportsVideo?: boolean;
   // Model defaults to adaptive thinking and REJECTS an explicit `thinking.type:"disabled"`
   // (upstream returns 400). Used to normalize the request when a combo/route substitutes
   // this model after the client already chose `disabled`. See issue #3554.

src/shared/validation/settingsSchemas.ts

@@ -356,6 +356,11 @@ export const updateSettingsSchema = z.object({
   modalityBridgeAudioModel: z.string().max(200).optional(),
   modalityBridgeAudioTimeout: z.number().int().min(1000).max(300000).optional(),
   modalityBridgeAudioMaxClips: z.number().int().min(1).max(10).optional(),
+  modalityBridgeVideoEnabled: z.boolean().optional(),
+  modalityBridgeVideoModel: z.string().max(200).optional(),
+  modalityBridgeVideoFrameCount: z.number().int().min(1).max(16).optional(),
+  modalityBridgeVideoMaxVideos: z.number().int().min(1).max(4).optional(),
+  modalityBridgeVideoTimeout: z.number().int().min(1000).max(300000).optional(),
   modalityBridgeCacheEnabled: z.boolean().optional(),
   modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(),
   modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(),

tests/unit/video-bridge-media-capabilities.test.ts

@@ -0,0 +1,55 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { containsMediaKind, detectMediaParts } from "../../open-sse/utils/mediaParts.ts";
+
+const messages = (content: unknown) => [{ role: "user", content }];
+
+test("detects supported Chat and Responses video blocks without treating plain text as video", () => {
+  const detected = detectMediaParts(
+    messages([
+      { type: "input_video", video_url: "https://media.example/input.mp4" },
+      { type: "video_url", video_url: { url: "https://media.example/url.webm" } },
+      {
+        type: "video_source",
+        source: { type: "base64", media_type: "video/mp4", data: "QUJD" },
+      },
+      { type: "input_text", text: "Please inspect demo.mp4 but do not fetch it." },
+    ])
+  );
+
+  assert.deepEqual(
+    detected.map(({ kind, ref, shape, nested }) => ({ kind, ref, shape, nested })),
+    [
+      {
+        kind: "video",
+        ref: "https://media.example/input.mp4",
+        shape: "input_video",
+        nested: false,
+      },
+      {
+        kind: "video",
+        ref: "https://media.example/url.webm",
+        shape: "video_url",
+        nested: false,
+      },
+      {
+        kind: "video",
+        ref: "data:video/mp4;base64,QUJD",
+        shape: "video_source",
+        nested: false,
+      },
+    ]
+  );
+  assert.equal(
+    containsMediaKind(messages([{ type: "input_text", text: "demo.mp4" }]), "video"),
+    false
+  );
+  assert.equal(
+    containsMediaKind(
+      messages([{ type: "input_video", video_url: "data:video/mp4;base64,QUJD" }]),
+      "video"
+    ),
+    true
+  );
+});

tests/unit/video-bridge-settings.test.ts

@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  resolveAudioCapability,
+  resolveVideoCapability,
+} from "../../src/lib/modelCapabilityModalities.ts";
+import {
+  MODALITY_BRIDGE_DEFAULTS,
+  resolveVideoBridgeRuntimeSettings,
+} from "../../src/shared/constants/modalityBridgeDefaults.ts";
+import { updateSettingsSchema } from "../../src/shared/validation/settingsSchemas.ts";
+
+test("audio and video capability resolution trusts explicit catalog data before modalities", () => {
+  assert.equal(resolveAudioCapability({ supportsAudio: false }, null, ["audio"]), false);
+  assert.equal(resolveVideoCapability(undefined, { supportsVideo: true }, ["text"]), true);
+  assert.equal(resolveVideoCapability(undefined, null, ["text", "video"]), true);
+  assert.equal(resolveVideoCapability(undefined, null, ["text"]), false);
+  assert.equal(resolveVideoCapability(undefined, null, []), null);
+});
+
+test("Video Bridge settings default to a bounded disabled runtime and accept valid overrides", () => {
+  assert.deepEqual(resolveVideoBridgeRuntimeSettings({}), {
+    enabled: false,
+    model: "",
+    frameCount: 8,
+    maxVideos: 1,
+    timeoutMs: 120_000,
+    cacheEnabled: MODALITY_BRIDGE_DEFAULTS.cacheEnabled,
+    cacheTtlMinutes: MODALITY_BRIDGE_DEFAULTS.cacheTtlMinutes,
+    cacheMaxEntries: MODALITY_BRIDGE_DEFAULTS.cacheMaxEntries,
+  });
+
+  const valid = updateSettingsSchema.safeParse({
+    modalityBridgeVideoEnabled: true,
+    modalityBridgeVideoModel: "openai/gpt-4o-mini",
+    modalityBridgeVideoFrameCount: 16,
+    modalityBridgeVideoMaxVideos: 4,
+    modalityBridgeVideoTimeout: 120_000,
+  });
+  assert.equal(valid.success, true);
+});
+
+test("Video Bridge settings schema rejects values outside extraction bounds", () => {
+  for (const [field, value] of Object.entries({
+    modalityBridgeVideoFrameCount: 17,
+    modalityBridgeVideoMaxVideos: 0,
+    modalityBridgeVideoTimeout: 300_001,
+  })) {
+    assert.equal(
+      updateSettingsSchema.safeParse({ [field]: value }).success,
+      false,
+      `${field}=${value} should be rejected`
+    );
+  }
+});