feat(video-bridge): caption frames for text-only models

src/lib/guardrails/modalityBridge/bridgeStats.ts

@@ -1,10 +1,10 @@
 /**
  * Modality Bridge stats + response transparency header (PR-1 Task 9).
  *
- * In-memory, process-global counters for bridge activity ("vision" today,
- * "audio" reserved for PR-3) plus the builder for the
+ * In-memory, process-global counters for vision, audio, and video bridge
+ * activity plus the builder for the
  * `x-omniroute-modality-bridge` response header, which tells clients that
- * their request payload was transparently transformed (image→text describe).
+ * their request payload was transparently transformed into text.
  * Reroutes do NOT get a header — the payload was untouched, only the model
  * changed, and that is already visible in the response body's `model` field.
  *
@@ -18,13 +18,16 @@ export interface BridgeModalityStats {
   lastUsedAt: string | null;
 }
 
-const stats: Record<"vision" | "audio", BridgeModalityStats> = {
+export type BridgeModality = "vision" | "audio" | "video";
+
+const stats: Record<BridgeModality, BridgeModalityStats> = {
   vision: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null },
   audio: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null },
+  video: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null },
 };
 
 export function recordBridgeUse(
-  kind: "vision" | "audio",
+  kind: BridgeModality,
   opts: { cacheHit?: boolean; failure?: boolean } = {}
 ): void {
   const s = stats[kind];
@@ -34,7 +37,7 @@ export function recordBridgeUse(
   s.lastUsedAt = new Date().toISOString();
 }
 
-export function getBridgeStats(): Record<"vision" | "audio", BridgeModalityStats> {
+export function getBridgeStats(): Record<BridgeModality, BridgeModalityStats> {
   return structuredClone(stats);
 }
 
@@ -77,6 +80,15 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string
         `audio->text;model=${headerModelToken(meta.sttModel)};parts=${meta.clipsProcessed}`
       );
     }
+    if (
+      r.guardrail === "video-bridge" &&
+      typeof meta.videosProcessed === "number" &&
+      !meta.rerouted
+    ) {
+      segments.push(
+        `video->text;model=${headerModelToken(meta.videoModel)};parts=${meta.videosProcessed}`
+      );
+    }
   }
   return segments.length ? segments.join(", ") : null;
 }

src/lib/guardrails/registry.ts

@@ -8,6 +8,7 @@ import { PIIMaskerGuardrail } from "./piiMasker";
 import { PromptInjectionGuardrail } from "./promptInjection";
 import { VisionBridgeGuardrail } from "./visionBridge";
 import { AudioBridgeGuardrail } from "./audioBridge";
+import { VideoBridgeGuardrail } from "./videoBridge";
 import { CredentialMaskerGuardrail } from "./credentialMasker";
 
 /**
@@ -288,6 +289,7 @@ export function registerDefaultGuardrails() {
 
   guardrailRegistry.register(new VisionBridgeGuardrail());
   guardrailRegistry.register(new AudioBridgeGuardrail());
+  guardrailRegistry.register(new VideoBridgeGuardrail());
   guardrailRegistry.register(new PIIMaskerGuardrail());
   guardrailRegistry.register(new CredentialMaskerGuardrail());
   guardrailRegistry.register(new PromptInjectionGuardrail());

src/lib/guardrails/videoBridge.ts

@@ -0,0 +1,174 @@
+import { getSettings as defaultGetSettings } from "@/lib/db/settings";
+import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
+import {
+  resolveVideoBridgeRuntimeSettings,
+  resolveVisionBridgeRuntimeSettings,
+} from "@/shared/constants/modalityBridgeDefaults";
+
+import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";
+import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache";
+import { recordBridgeUse } from "./modalityBridge/bridgeStats";
+import {
+  describeVideoPart as defaultDescribeVideoPart,
+  extractVideoParts,
+  formatVideoTimestamp,
+  replaceVideoParts,
+  type DescribedVideo,
+  type VideoPart,
+} from "./videoBridgeHelpers";
+import {
+  callVisionModel as defaultCallVisionModel,
+  type VisionModelConfig,
+} from "./visionBridgeHelpers";
+
+type VideoBridgeBody = {
+  model?: string;
+  messages?: Array<{ role?: string; content?: unknown }>;
+  input?: Array<{ role?: string; content?: unknown }>;
+  [key: string]: unknown;
+};
+
+export interface VideoBridgeDependencies {
+  getSettings?: () => Promise<Record<string, unknown>>;
+  getCapabilities?: (model: string) => { supportsVideo: boolean | null };
+  describePart?: (part: VideoPart) => Promise<DescribedVideo>;
+  callVisionModel?: (
+    imageDataUri: string,
+    config: VisionModelConfig,
+    apiKey?: string
+  ) => Promise<string>;
+}
+
+export class VideoBridgeGuardrail extends BaseGuardrail {
+  name = "video-bridge";
+  priority = 7;
+
+  private readonly deps: VideoBridgeDependencies;
+
+  constructor(options?: { enabled?: boolean; deps?: VideoBridgeDependencies }) {
+    super("video-bridge", { priority: 7, enabled: options?.enabled });
+    this.deps = options?.deps ?? {};
+  }
+
+  async preCall(payload: unknown, context: GuardrailContext): Promise<GuardrailResult<unknown>> {
+    if (!this.enabled || context.disabledGuardrails?.includes("video-bridge")) {
+      return { block: false };
+    }
+
+    const body = payload as VideoBridgeBody;
+    const model = context.model || body.model;
+    if (!model) return { block: false };
+
+    const getSettings = this.deps.getSettings ?? defaultGetSettings;
+    let persisted: Record<string, unknown> = {};
+    try {
+      persisted = await getSettings();
+    } catch {
+      // Early boot can run before the settings database is ready; defaults are safe.
+    }
+    const runtime = resolveVideoBridgeRuntimeSettings(persisted);
+    if (!runtime.enabled) return { block: false };
+
+    const parts = extractVideoParts(body).slice(0, runtime.maxVideos);
+    if (parts.length === 0) return { block: false };
+
+    const capabilities = (this.deps.getCapabilities ?? getResolvedModelCapabilities)(model);
+    if (capabilities.supportsVideo === true) return { block: false };
+
+    const visionRuntime = resolveVisionBridgeRuntimeSettings(persisted);
+    const videoModel = runtime.model.trim() || visionRuntime.model.trim();
+    const startedAt = Date.now();
+    const descriptions: Array<string | null> = [];
+    let totalFramesRequested = 0;
+    let totalFramesUsed = 0;
+    let totalDurationSeconds = 0;
+    let totalCacheHits = 0;
+    let failures = 0;
+
+    for (let index = 0; index < parts.length; index++) {
+      const part = parts[index];
+      try {
+        if (!videoModel) throw new Error("Video Bridge vision model is not configured");
+        const described = this.deps.describePart
+          ? await this.deps.describePart(part)
+          : await this.describeWithVisionModel(part, runtime, visionRuntime, videoModel);
+        const videoCacheHits = described.cacheHits ?? 0;
+        descriptions.push(described.description);
+        totalFramesRequested += described.framesRequested;
+        totalFramesUsed += described.framesUsed;
+        totalDurationSeconds += described.durationSeconds;
+        totalCacheHits += videoCacheHits;
+        recordBridgeUse("video", { cacheHit: videoCacheHits > 0 });
+      } catch {
+        failures += 1;
+        recordBridgeUse("video", { failure: true });
+        context.log?.warn?.(
+          "VIDEO_BRIDGE",
+          `Failed to describe video ${index + 1}; preserving or stubbing it according to capability policy`
+        );
+        descriptions.push(
+          capabilities.supportsVideo === false
+            ? `[Video ${index + 1}]: (unavailable — video could not be described)`
+            : null
+        );
+      }
+    }
+
+    const videosProcessed = descriptions.filter((description) => description !== null).length;
+    if (videosProcessed === 0) return { block: false };
+
+    return {
+      block: false,
+      modifiedPayload: replaceVideoParts(body, parts, descriptions),
+      meta: {
+        cacheHits: totalCacheHits,
+        durationSeconds: totalDurationSeconds,
+        failures,
+        framesRequested: totalFramesRequested,
+        framesUsed: totalFramesUsed,
+        processingTimeMs: Date.now() - startedAt,
+        videoModel: videoModel || "unavailable",
+        videosProcessed,
+      },
+    };
+  }
+
+  private async describeWithVisionModel(
+    part: VideoPart,
+    runtime: ReturnType<typeof resolveVideoBridgeRuntimeSettings>,
+    visionRuntime: ReturnType<typeof resolveVisionBridgeRuntimeSettings>,
+    videoModel: string
+  ): Promise<DescribedVideo> {
+    const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null;
+    const callVisionModel = this.deps.callVisionModel ?? defaultCallVisionModel;
+    let cacheHits = 0;
+    const described = await defaultDescribeVideoPart(
+      part,
+      {
+        frameCount: runtime.frameCount,
+        timeoutMs: runtime.timeoutMs,
+      },
+      async (frameDataUri, timestampSeconds, signal) => {
+        const prompt = `${visionRuntime.prompt}\n\nThis frame is from a video at ${formatVideoTimestamp(timestampSeconds)}. Describe only observable details relevant to the video.`;
+        const key = cache
+          ? bridgeCacheKey(frameDataUri, `${prompt}@${timestampSeconds.toFixed(3)}`, videoModel)
+          : null;
+        const cached = key && cache ? cache.get(key) : undefined;
+        if (cached !== undefined) {
+          cacheHits += 1;
+          return cached;
+        }
+        const caption = await callVisionModel(frameDataUri, {
+          maxImages: 1,
+          model: videoModel,
+          prompt,
+          signal,
+          timeoutMs: runtime.timeoutMs,
+        });
+        if (key && cache) cache.set(key, caption);
+        return caption;
+      }
+    );
+    return { ...described, cacheHits };
+  }
+}

tests/unit/guardrails/videoBridge.test.ts

@@ -0,0 +1,169 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts";
+import {
+  buildModalityBridgeHeader,
+  getBridgeStats,
+} from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts";
+import {
+  registerDefaultGuardrails,
+  resetGuardrailsForTests,
+} from "../../../src/lib/guardrails/registry.ts";
+
+const payload = () => ({
+  model: "example/text-only",
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "input_video", video_url: "data:video/mp4;base64,QUJD" },
+        { type: "text", text: "What happens?" },
+      ],
+    },
+  ],
+});
+
+function guardrail(options: { capability?: boolean | null; fail?: boolean } = {}) {
+  return new VideoBridgeGuardrail({
+    deps: {
+      getSettings: async () => ({
+        modalityBridgeVideoEnabled: true,
+        modalityBridgeVideoModel: "openai/gpt-4o-mini",
+        modalityBridgeCacheEnabled: false,
+      }),
+      getCapabilities: () => ({
+        supportsVideo: options.capability === undefined ? false : options.capability,
+      }),
+      describePart: async () => {
+        if (options.fail) throw new Error("private ffmpeg failure");
+        return {
+          description: "[Video description: frame@t=00:01.000 a person waves]",
+          durationSeconds: 2,
+          framesRequested: 1,
+          framesUsed: 1,
+        };
+      },
+    },
+  });
+}
+
+test("VideoBridgeGuardrail has priority 7 and native video targets bypass conversion", async () => {
+  let calls = 0;
+  const native = new VideoBridgeGuardrail({
+    deps: {
+      getSettings: async () => ({ modalityBridgeVideoEnabled: true }),
+      getCapabilities: () => ({ supportsVideo: true }),
+      describePart: async () => {
+        calls += 1;
+        throw new Error("should not run");
+      },
+    },
+  });
+  assert.equal(native.name, "video-bridge");
+  assert.equal(native.priority, 7);
+  assert.equal((await native.preCall(payload(), {})).modifiedPayload, undefined);
+  assert.equal(calls, 0);
+});
+
+test("converts Chat video to timestamped text and emits telemetry/header metadata", async () => {
+  const before = getBridgeStats().video;
+  const result = await guardrail().preCall(payload(), {});
+  const modified = result.modifiedPayload as ReturnType<typeof payload>;
+  assert.deepEqual(modified.messages[0].content[0], {
+    type: "text",
+    text: "[Video description: frame@t=00:01.000 a person waves]",
+  });
+  assert.equal(result.meta?.videosProcessed, 1);
+  assert.equal(result.meta?.framesUsed, 1);
+  assert.equal(result.meta?.videoModel, "openai/gpt-4o-mini");
+  assert.equal(
+    buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]),
+    "video->text;model=openai/gpt-4o-mini;parts=1"
+  );
+  assert.ok(getBridgeStats().video.bridged >= before.bridged + 1);
+});
+
+test("converts Responses input using input_text while preserving sibling order", async () => {
+  const body = {
+    model: "example/text-only",
+    input: [
+      {
+        role: "user",
+        content: [
+          { type: "input_text", text: "before" },
+          { type: "video_url", video_url: { url: "https://example.test/video.mp4" } },
+          { type: "input_text", text: "after" },
+        ],
+      },
+    ],
+  };
+  const result = await guardrail().preCall(body, {});
+  assert.deepEqual((result.modifiedPayload as typeof body).input[0].content, [
+    { type: "input_text", text: "before" },
+    { type: "input_text", text: "[Video description: frame@t=00:01.000 a person waves]" },
+    { type: "input_text", text: "after" },
+  ]);
+});
+
+test("preserves unknown-capability video on total failure but stubs proven text-only input", async () => {
+  const original = payload();
+  const snapshot = structuredClone(original);
+  const unknown = await guardrail({ capability: null, fail: true }).preCall(original, {});
+  assert.equal(unknown.modifiedPayload, undefined);
+  assert.deepEqual(original, snapshot);
+
+  const knownFalse = await guardrail({ capability: false, fail: true }).preCall(payload(), {});
+  const modified = knownFalse.modifiedPayload as ReturnType<typeof payload>;
+  assert.deepEqual(modified.messages[0].content[0], {
+    type: "text",
+    text: "[Video 1]: (unavailable — video could not be described)",
+  });
+  assert.equal(String(knownFalse.meta?.failures).includes("private"), false);
+});
+
+test("reports cache hits per converted video without carrying a previous hit forward", async () => {
+  const body = payload();
+  body.messages[0].content.splice(1, 0, {
+    type: "video_url",
+    video_url: "data:video/mp4;base64,REVG",
+  });
+  const before = getBridgeStats().video;
+  let described = 0;
+  const bridge = new VideoBridgeGuardrail({
+    deps: {
+      getSettings: async () => ({
+        modalityBridgeVideoEnabled: true,
+        modalityBridgeVideoMaxVideos: 2,
+        modalityBridgeVideoModel: "openai/gpt-4o-mini",
+      }),
+      getCapabilities: () => ({ supportsVideo: false }),
+      describePart: async () => {
+        described += 1;
+        return {
+          cacheHits: described === 1 ? 1 : 0,
+          description: `[Video description: frame@t=00:0${described}.000 frame ${described}]`,
+          durationSeconds: 2,
+          framesRequested: 1,
+          framesUsed: 1,
+        };
+      },
+    },
+  });
+
+  const result = await bridge.preCall(body, {});
+  const after = getBridgeStats().video;
+  assert.equal(result.meta?.cacheHits, 1);
+  assert.equal(after.bridged - before.bridged, 2);
+  assert.equal(after.cacheHits - before.cacheHits, 1);
+});
+
+test("default registry includes Video Bridge after Vision and Audio", () => {
+  resetGuardrailsForTests({ registerDefaults: false });
+  const names = registerDefaultGuardrails()
+    .list()
+    .filter((entry) => entry.name.endsWith("-bridge"))
+    .map((entry) => `${entry.priority}:${entry.name}`);
+  assert.deepEqual(names, ["5:vision-bridge", "6:audio-bridge", "7:video-bridge"]);
+  resetGuardrailsForTests();
+});

tests/unit/video-bridge-header-stats.test.ts

@@ -0,0 +1,42 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  buildModalityBridgeHeader,
+  getBridgeStats,
+  recordBridgeUse,
+} from "../../src/lib/guardrails/modalityBridge/bridgeStats.ts";
+
+test("composes vision, audio, and video bridge header segments in deterministic order", () => {
+  assert.equal(
+    buildModalityBridgeHeader([
+      {
+        guardrail: "vision-bridge",
+        meta: { imagesProcessed: 2, visionModel: "openai/gpt-4o-mini" },
+      },
+      {
+        guardrail: "audio-bridge",
+        meta: { clipsProcessed: 1, sttModel: "deepgram/nova-3" },
+      },
+      {
+        guardrail: "video-bridge",
+        meta: { videoModel: "openai/gpt-4o-mini", videosProcessed: 3 },
+      },
+    ]),
+    "image->text;model=openai/gpt-4o-mini;parts=2, " +
+      "audio->text;model=deepgram/nova-3;parts=1, " +
+      "video->text;model=openai/gpt-4o-mini;parts=3"
+  );
+});
+
+test("tracks video bridge totals, failures, and cache hits independently", () => {
+  const before = getBridgeStats().video;
+  recordBridgeUse("video", { cacheHit: true });
+  recordBridgeUse("video", { failure: true });
+  const after = getBridgeStats().video;
+
+  assert.equal(after.bridged - before.bridged, 2);
+  assert.equal(after.cacheHits - before.cacheHits, 1);
+  assert.equal(after.failures - before.failures, 1);
+  assert.match(after.lastUsedAt ?? "", /^\d{4}-\d{2}-\d{2}T/);
+});