From 2a72090502c2a398961fa8b3d454d4b551ef3c42 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 15 Aug 2026 01:13:37 -0300 Subject: [PATCH] feat(video-bridge): caption frames for text-only models --- .../guardrails/modalityBridge/bridgeStats.ts | 24 ++- src/lib/guardrails/registry.ts | 2 + src/lib/guardrails/videoBridge.ts | 174 ++++++++++++++++++ tests/unit/guardrails/videoBridge.test.ts | 169 +++++++++++++++++ tests/unit/video-bridge-header-stats.test.ts | 42 +++++ 5 files changed, 405 insertions(+), 6 deletions(-) create mode 100644 src/lib/guardrails/videoBridge.ts create mode 100644 tests/unit/guardrails/videoBridge.test.ts create mode 100644 tests/unit/video-bridge-header-stats.test.ts diff --git a/src/lib/guardrails/modalityBridge/bridgeStats.ts b/src/lib/guardrails/modalityBridge/bridgeStats.ts index b511ced37d..3bd13ce38b 100644 --- a/src/lib/guardrails/modalityBridge/bridgeStats.ts +++ b/src/lib/guardrails/modalityBridge/bridgeStats.ts @@ -1,10 +1,10 @@ /** * Modality Bridge stats + response transparency header (PR-1 Task 9). * - * In-memory, process-global counters for bridge activity ("vision" today, - * "audio" reserved for PR-3) plus the builder for the + * In-memory, process-global counters for vision, audio, and video bridge + * activity plus the builder for the * `x-omniroute-modality-bridge` response header, which tells clients that - * their request payload was transparently transformed (image→text describe). + * their request payload was transparently transformed into text. * Reroutes do NOT get a header — the payload was untouched, only the model * changed, and that is already visible in the response body's `model` field. * @@ -18,13 +18,16 @@ export interface BridgeModalityStats { lastUsedAt: string | null; } -const stats: Record<"vision" | "audio", BridgeModalityStats> = { +export type BridgeModality = "vision" | "audio" | "video"; + +const stats: Record = { vision: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null }, audio: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null }, + video: { bridged: 0, cacheHits: 0, failures: 0, lastUsedAt: null }, }; export function recordBridgeUse( - kind: "vision" | "audio", + kind: BridgeModality, opts: { cacheHit?: boolean; failure?: boolean } = {} ): void { const s = stats[kind]; @@ -34,7 +37,7 @@ export function recordBridgeUse( s.lastUsedAt = new Date().toISOString(); } -export function getBridgeStats(): Record<"vision" | "audio", BridgeModalityStats> { +export function getBridgeStats(): Record { return structuredClone(stats); } @@ -77,6 +80,15 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string `audio->text;model=${headerModelToken(meta.sttModel)};parts=${meta.clipsProcessed}` ); } + if ( + r.guardrail === "video-bridge" && + typeof meta.videosProcessed === "number" && + !meta.rerouted + ) { + segments.push( + `video->text;model=${headerModelToken(meta.videoModel)};parts=${meta.videosProcessed}` + ); + } } return segments.length ? segments.join(", ") : null; } diff --git a/src/lib/guardrails/registry.ts b/src/lib/guardrails/registry.ts index 774e47014a..7a8f7f8bf6 100644 --- a/src/lib/guardrails/registry.ts +++ b/src/lib/guardrails/registry.ts @@ -8,6 +8,7 @@ import { PIIMaskerGuardrail } from "./piiMasker"; import { PromptInjectionGuardrail } from "./promptInjection"; import { VisionBridgeGuardrail } from "./visionBridge"; import { AudioBridgeGuardrail } from "./audioBridge"; +import { VideoBridgeGuardrail } from "./videoBridge"; import { CredentialMaskerGuardrail } from "./credentialMasker"; /** @@ -288,6 +289,7 @@ export function registerDefaultGuardrails() { guardrailRegistry.register(new VisionBridgeGuardrail()); guardrailRegistry.register(new AudioBridgeGuardrail()); + guardrailRegistry.register(new VideoBridgeGuardrail()); guardrailRegistry.register(new PIIMaskerGuardrail()); guardrailRegistry.register(new CredentialMaskerGuardrail()); guardrailRegistry.register(new PromptInjectionGuardrail()); diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts new file mode 100644 index 0000000000..0baff23881 --- /dev/null +++ b/src/lib/guardrails/videoBridge.ts @@ -0,0 +1,174 @@ +import { getSettings as defaultGetSettings } from "@/lib/db/settings"; +import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; +import { + resolveVideoBridgeRuntimeSettings, + resolveVisionBridgeRuntimeSettings, +} from "@/shared/constants/modalityBridgeDefaults"; + +import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base"; +import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache"; +import { recordBridgeUse } from "./modalityBridge/bridgeStats"; +import { + describeVideoPart as defaultDescribeVideoPart, + extractVideoParts, + formatVideoTimestamp, + replaceVideoParts, + type DescribedVideo, + type VideoPart, +} from "./videoBridgeHelpers"; +import { + callVisionModel as defaultCallVisionModel, + type VisionModelConfig, +} from "./visionBridgeHelpers"; + +type VideoBridgeBody = { + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + input?: Array<{ role?: string; content?: unknown }>; + [key: string]: unknown; +}; + +export interface VideoBridgeDependencies { + getSettings?: () => Promise>; + getCapabilities?: (model: string) => { supportsVideo: boolean | null }; + describePart?: (part: VideoPart) => Promise; + callVisionModel?: ( + imageDataUri: string, + config: VisionModelConfig, + apiKey?: string + ) => Promise; +} + +export class VideoBridgeGuardrail extends BaseGuardrail { + name = "video-bridge"; + priority = 7; + + private readonly deps: VideoBridgeDependencies; + + constructor(options?: { enabled?: boolean; deps?: VideoBridgeDependencies }) { + super("video-bridge", { priority: 7, enabled: options?.enabled }); + this.deps = options?.deps ?? {}; + } + + async preCall(payload: unknown, context: GuardrailContext): Promise> { + if (!this.enabled || context.disabledGuardrails?.includes("video-bridge")) { + return { block: false }; + } + + const body = payload as VideoBridgeBody; + const model = context.model || body.model; + if (!model) return { block: false }; + + const getSettings = this.deps.getSettings ?? defaultGetSettings; + let persisted: Record = {}; + try { + persisted = await getSettings(); + } catch { + // Early boot can run before the settings database is ready; defaults are safe. + } + const runtime = resolveVideoBridgeRuntimeSettings(persisted); + if (!runtime.enabled) return { block: false }; + + const parts = extractVideoParts(body).slice(0, runtime.maxVideos); + if (parts.length === 0) return { block: false }; + + const capabilities = (this.deps.getCapabilities ?? getResolvedModelCapabilities)(model); + if (capabilities.supportsVideo === true) return { block: false }; + + const visionRuntime = resolveVisionBridgeRuntimeSettings(persisted); + const videoModel = runtime.model.trim() || visionRuntime.model.trim(); + const startedAt = Date.now(); + const descriptions: Array = []; + let totalFramesRequested = 0; + let totalFramesUsed = 0; + let totalDurationSeconds = 0; + let totalCacheHits = 0; + let failures = 0; + + for (let index = 0; index < parts.length; index++) { + const part = parts[index]; + try { + if (!videoModel) throw new Error("Video Bridge vision model is not configured"); + const described = this.deps.describePart + ? await this.deps.describePart(part) + : await this.describeWithVisionModel(part, runtime, visionRuntime, videoModel); + const videoCacheHits = described.cacheHits ?? 0; + descriptions.push(described.description); + totalFramesRequested += described.framesRequested; + totalFramesUsed += described.framesUsed; + totalDurationSeconds += described.durationSeconds; + totalCacheHits += videoCacheHits; + recordBridgeUse("video", { cacheHit: videoCacheHits > 0 }); + } catch { + failures += 1; + recordBridgeUse("video", { failure: true }); + context.log?.warn?.( + "VIDEO_BRIDGE", + `Failed to describe video ${index + 1}; preserving or stubbing it according to capability policy` + ); + descriptions.push( + capabilities.supportsVideo === false + ? `[Video ${index + 1}]: (unavailable — video could not be described)` + : null + ); + } + } + + const videosProcessed = descriptions.filter((description) => description !== null).length; + if (videosProcessed === 0) return { block: false }; + + return { + block: false, + modifiedPayload: replaceVideoParts(body, parts, descriptions), + meta: { + cacheHits: totalCacheHits, + durationSeconds: totalDurationSeconds, + failures, + framesRequested: totalFramesRequested, + framesUsed: totalFramesUsed, + processingTimeMs: Date.now() - startedAt, + videoModel: videoModel || "unavailable", + videosProcessed, + }, + }; + } + + private async describeWithVisionModel( + part: VideoPart, + runtime: ReturnType, + visionRuntime: ReturnType, + videoModel: string + ): Promise { + const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null; + const callVisionModel = this.deps.callVisionModel ?? defaultCallVisionModel; + let cacheHits = 0; + const described = await defaultDescribeVideoPart( + part, + { + frameCount: runtime.frameCount, + timeoutMs: runtime.timeoutMs, + }, + async (frameDataUri, timestampSeconds, signal) => { + const prompt = `${visionRuntime.prompt}\n\nThis frame is from a video at ${formatVideoTimestamp(timestampSeconds)}. Describe only observable details relevant to the video.`; + const key = cache + ? bridgeCacheKey(frameDataUri, `${prompt}@${timestampSeconds.toFixed(3)}`, videoModel) + : null; + const cached = key && cache ? cache.get(key) : undefined; + if (cached !== undefined) { + cacheHits += 1; + return cached; + } + const caption = await callVisionModel(frameDataUri, { + maxImages: 1, + model: videoModel, + prompt, + signal, + timeoutMs: runtime.timeoutMs, + }); + if (key && cache) cache.set(key, caption); + return caption; + } + ); + return { ...described, cacheHits }; + } +} diff --git a/tests/unit/guardrails/videoBridge.test.ts b/tests/unit/guardrails/videoBridge.test.ts new file mode 100644 index 0000000000..ae4321cb41 --- /dev/null +++ b/tests/unit/guardrails/videoBridge.test.ts @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts"; +import { + buildModalityBridgeHeader, + getBridgeStats, +} from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts"; +import { + registerDefaultGuardrails, + resetGuardrailsForTests, +} from "../../../src/lib/guardrails/registry.ts"; + +const payload = () => ({ + model: "example/text-only", + messages: [ + { + role: "user", + content: [ + { type: "input_video", video_url: "data:video/mp4;base64,QUJD" }, + { type: "text", text: "What happens?" }, + ], + }, + ], +}); + +function guardrail(options: { capability?: boolean | null; fail?: boolean } = {}) { + return new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVideoEnabled: true, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + modalityBridgeCacheEnabled: false, + }), + getCapabilities: () => ({ + supportsVideo: options.capability === undefined ? false : options.capability, + }), + describePart: async () => { + if (options.fail) throw new Error("private ffmpeg failure"); + return { + description: "[Video description: frame@t=00:01.000 a person waves]", + durationSeconds: 2, + framesRequested: 1, + framesUsed: 1, + }; + }, + }, + }); +} + +test("VideoBridgeGuardrail has priority 7 and native video targets bypass conversion", async () => { + let calls = 0; + const native = new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ modalityBridgeVideoEnabled: true }), + getCapabilities: () => ({ supportsVideo: true }), + describePart: async () => { + calls += 1; + throw new Error("should not run"); + }, + }, + }); + assert.equal(native.name, "video-bridge"); + assert.equal(native.priority, 7); + assert.equal((await native.preCall(payload(), {})).modifiedPayload, undefined); + assert.equal(calls, 0); +}); + +test("converts Chat video to timestamped text and emits telemetry/header metadata", async () => { + const before = getBridgeStats().video; + const result = await guardrail().preCall(payload(), {}); + const modified = result.modifiedPayload as ReturnType; + assert.deepEqual(modified.messages[0].content[0], { + type: "text", + text: "[Video description: frame@t=00:01.000 a person waves]", + }); + assert.equal(result.meta?.videosProcessed, 1); + assert.equal(result.meta?.framesUsed, 1); + assert.equal(result.meta?.videoModel, "openai/gpt-4o-mini"); + assert.equal( + buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]), + "video->text;model=openai/gpt-4o-mini;parts=1" + ); + assert.ok(getBridgeStats().video.bridged >= before.bridged + 1); +}); + +test("converts Responses input using input_text while preserving sibling order", async () => { + const body = { + model: "example/text-only", + input: [ + { + role: "user", + content: [ + { type: "input_text", text: "before" }, + { type: "video_url", video_url: { url: "https://example.test/video.mp4" } }, + { type: "input_text", text: "after" }, + ], + }, + ], + }; + const result = await guardrail().preCall(body, {}); + assert.deepEqual((result.modifiedPayload as typeof body).input[0].content, [ + { type: "input_text", text: "before" }, + { type: "input_text", text: "[Video description: frame@t=00:01.000 a person waves]" }, + { type: "input_text", text: "after" }, + ]); +}); + +test("preserves unknown-capability video on total failure but stubs proven text-only input", async () => { + const original = payload(); + const snapshot = structuredClone(original); + const unknown = await guardrail({ capability: null, fail: true }).preCall(original, {}); + assert.equal(unknown.modifiedPayload, undefined); + assert.deepEqual(original, snapshot); + + const knownFalse = await guardrail({ capability: false, fail: true }).preCall(payload(), {}); + const modified = knownFalse.modifiedPayload as ReturnType; + assert.deepEqual(modified.messages[0].content[0], { + type: "text", + text: "[Video 1]: (unavailable — video could not be described)", + }); + assert.equal(String(knownFalse.meta?.failures).includes("private"), false); +}); + +test("reports cache hits per converted video without carrying a previous hit forward", async () => { + const body = payload(); + body.messages[0].content.splice(1, 0, { + type: "video_url", + video_url: "data:video/mp4;base64,REVG", + }); + const before = getBridgeStats().video; + let described = 0; + const bridge = new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeVideoEnabled: true, + modalityBridgeVideoMaxVideos: 2, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + }), + getCapabilities: () => ({ supportsVideo: false }), + describePart: async () => { + described += 1; + return { + cacheHits: described === 1 ? 1 : 0, + description: `[Video description: frame@t=00:0${described}.000 frame ${described}]`, + durationSeconds: 2, + framesRequested: 1, + framesUsed: 1, + }; + }, + }, + }); + + const result = await bridge.preCall(body, {}); + const after = getBridgeStats().video; + assert.equal(result.meta?.cacheHits, 1); + assert.equal(after.bridged - before.bridged, 2); + assert.equal(after.cacheHits - before.cacheHits, 1); +}); + +test("default registry includes Video Bridge after Vision and Audio", () => { + resetGuardrailsForTests({ registerDefaults: false }); + const names = registerDefaultGuardrails() + .list() + .filter((entry) => entry.name.endsWith("-bridge")) + .map((entry) => `${entry.priority}:${entry.name}`); + assert.deepEqual(names, ["5:vision-bridge", "6:audio-bridge", "7:video-bridge"]); + resetGuardrailsForTests(); +}); diff --git a/tests/unit/video-bridge-header-stats.test.ts b/tests/unit/video-bridge-header-stats.test.ts new file mode 100644 index 0000000000..834d181828 --- /dev/null +++ b/tests/unit/video-bridge-header-stats.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildModalityBridgeHeader, + getBridgeStats, + recordBridgeUse, +} from "../../src/lib/guardrails/modalityBridge/bridgeStats.ts"; + +test("composes vision, audio, and video bridge header segments in deterministic order", () => { + assert.equal( + buildModalityBridgeHeader([ + { + guardrail: "vision-bridge", + meta: { imagesProcessed: 2, visionModel: "openai/gpt-4o-mini" }, + }, + { + guardrail: "audio-bridge", + meta: { clipsProcessed: 1, sttModel: "deepgram/nova-3" }, + }, + { + guardrail: "video-bridge", + meta: { videoModel: "openai/gpt-4o-mini", videosProcessed: 3 }, + }, + ]), + "image->text;model=openai/gpt-4o-mini;parts=2, " + + "audio->text;model=deepgram/nova-3;parts=1, " + + "video->text;model=openai/gpt-4o-mini;parts=3" + ); +}); + +test("tracks video bridge totals, failures, and cache hits independently", () => { + const before = getBridgeStats().video; + recordBridgeUse("video", { cacheHit: true }); + recordBridgeUse("video", { failure: true }); + const after = getBridgeStats().video; + + assert.equal(after.bridged - before.bridged, 2); + assert.equal(after.cacheHits - before.cacheHits, 1); + assert.equal(after.failures - before.failures, 1); + assert.match(after.lastUsedAt ?? "", /^\d{4}-\d{2}-\d{2}T/); +});