From 68b3fe715ac7bd813dbc2660ac1b0930e36d1059 Mon Sep 17 00:00:00 2001 From: Xiangzhe Date: Tue, 18 Aug 2026 02:07:22 -0300 Subject: [PATCH] feat(video): add timestamped contact sheets --- .../guardrails/modalityBridge/bridgeCache.ts | 2 + src/lib/guardrails/videoBridge.ts | 10 +- src/lib/guardrails/videoBridgeContactSheet.ts | 110 ++++++++++++++++++ src/lib/guardrails/videoBridgeHelpers.ts | 28 ++++- .../videoBridgeContactSheet.test.ts | 73 ++++++++++++ 5 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 src/lib/guardrails/videoBridgeContactSheet.ts create mode 100644 tests/unit/guardrails/videoBridgeContactSheet.test.ts diff --git a/src/lib/guardrails/modalityBridge/bridgeCache.ts b/src/lib/guardrails/modalityBridge/bridgeCache.ts index 68bf44fdad..5b1c235b19 100644 --- a/src/lib/guardrails/modalityBridge/bridgeCache.ts +++ b/src/lib/guardrails/modalityBridge/bridgeCache.ts @@ -16,6 +16,7 @@ export interface BridgeCacheKeyOptions { strategy?: string; frameCount?: number; maxVideos?: number; + contactSheet?: boolean; transcript?: string; version?: string; } @@ -39,6 +40,7 @@ export function bridgeCacheKey( strategy: options.strategy, frameCount: options.frameCount, maxVideos: options.maxVideos, + contactSheet: options.contactSheet, transcript: options.transcript, version: options.version, }; diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index d80b2a71e6..adbe53b9a1 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -71,6 +71,7 @@ interface VideoResultCacheMetadata { samplingPolicyEffective?: "uniform" | "scene_aware" | "segment_aware"; samplingPolicyRequested?: "uniform" | "scene_aware" | "segment_aware"; transcriptCuesApplied?: number; + contactSheetUsed?: boolean; cacheBytes: number; modelUsed: string; } @@ -119,7 +120,8 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe record.samplingPolicyRequested === "scene_aware" || record.samplingPolicyRequested === "segment_aware") && (record.transcriptCuesApplied === undefined || - (typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0)) + (typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0)) && + (record.contactSheetUsed === undefined || typeof record.contactSheetUsed === "boolean") ); } @@ -187,6 +189,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { let totalDedupDropped = 0; let focusWindowsApplied = 0; let transcriptCuesApplied = 0; + let contactSheetsUsed = 0; let samplingPolicyEffective: "uniform" | "scene_aware" | "segment_aware" = "uniform"; let failures = 0; @@ -209,6 +212,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { focusEndSeconds: part.focusWindow?.endSeconds ?? null, focusStartSeconds: part.focusWindow?.startSeconds ?? null, transcript: safeTranscriptFingerprint(part.transcript), + contactSheet: part.contactSheet ?? false, version: VIDEO_BRIDGE_RESULT_CACHE_VERSION, }) : null; @@ -240,6 +244,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { totalDurationSeconds += meta.durationSeconds; totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0; transcriptCuesApplied += meta.transcriptCuesApplied ?? 0; + if (meta.contactSheetUsed) contactSheetsUsed += 1; if (meta.samplingPolicyEffective && meta.samplingPolicyEffective !== "uniform") { samplingPolicyEffective = meta.samplingPolicyEffective; } @@ -282,6 +287,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { totalDedupDropped += described.dedupDropped ?? 0; if (described.focusWindow) focusWindowsApplied += 1; transcriptCuesApplied += described.transcriptCues?.length ?? 0; + if (described.contactSheetUsed) contactSheetsUsed += 1; totalDurationSeconds += described.durationSeconds; totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0; if ( @@ -320,6 +326,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { samplingPolicyRequested: described.sampling?.policyRequested ?? runtime.samplingPolicy, transcriptCuesApplied: described.transcriptCues?.length ?? 0, + contactSheetUsed: described.contactSheetUsed ?? false, }, }); recordBridgeUse("video", { @@ -386,6 +393,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { dedupDropped: totalDedupDropped, focusWindowsApplied, transcriptCuesApplied, + contactSheetsUsed, samplingCandidateCount: totalSamplingCandidateCount, samplingPolicyEffective, samplingPolicyRequested: runtime.samplingPolicy, diff --git a/src/lib/guardrails/videoBridgeContactSheet.ts b/src/lib/guardrails/videoBridgeContactSheet.ts new file mode 100644 index 0000000000..fc17627553 --- /dev/null +++ b/src/lib/guardrails/videoBridgeContactSheet.ts @@ -0,0 +1,110 @@ +export interface ContactSheetFrame { + dataUri: string; + timestampSeconds: number; +} + +export interface ContactSheetOptions { + columns?: number; + signal?: AbortSignal; + timeoutMs?: number; +} + +export interface VideoContactSheetResult { + dataUri?: string; + fallbackReason?: "CONTACT_SHEET_UNAVAILABLE"; + frames: ContactSheetFrame[]; + height?: number; + timestamps: number[]; + used: boolean; + width?: number; +} + +const MAX_FRAMES = 16; +const MAX_SHEET_BYTES = 32 * 1024 * 1024; +const TILE_SIZE = 512; + +function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult { + return { + fallbackReason: "CONTACT_SHEET_UNAVAILABLE", + frames: frames.map((frame) => ({ ...frame })), + timestamps: frames.map((frame) => frame.timestampSeconds), + used: false, + }; +} + +function decodeFrame(dataUri: string): Buffer { + const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri); + if (!match) throw new Error("Contact sheet requires JPEG data URIs"); + return Buffer.from(match[1], "base64"); +} + +/** Build an optional bounded JPEG grid; every failure except abort is fail-safe to individual frames. */ +export async function buildVideoContactSheet( + frames: readonly ContactSheetFrame[], + options: ContactSheetOptions = {} +): Promise { + if (options.signal?.aborted) throw new Error("Video contact sheet was aborted"); + if (frames.length < 1 || frames.length > MAX_FRAMES) return fallback(frames); + if ( + frames.some( + (frame) => + !Number.isFinite(frame.timestampSeconds) || frame.timestampSeconds < 0 || !frame.dataUri + ) + ) { + return fallback(frames); + } + const columns = Math.min(4, Math.max(1, Math.floor(options.columns ?? 2)), frames.length); + const rows = Math.ceil(frames.length / columns); + const controller = new AbortController(); + const timeout = options.timeoutMs + ? setTimeout(() => controller.abort(), options.timeoutMs) + : null; + const signal = options.signal + ? AbortSignal.any([options.signal, controller.signal]) + : controller.signal; + try { + const { default: sharp } = await import("sharp"); + if (signal.aborted) throw new Error("Video contact sheet was aborted"); + const tiles = await Promise.all( + frames.map(async (frame) => + sharp(decodeFrame(frame.dataUri)) + .resize(TILE_SIZE, TILE_SIZE, { fit: "contain", background: "#000000" }) + .jpeg({ quality: 80 }) + .toBuffer() + ) + ); + if (signal.aborted) throw new Error("Video contact sheet was aborted"); + const output = await sharp({ + create: { + background: "#000000", + channels: 3, + height: rows * TILE_SIZE, + width: columns * TILE_SIZE, + }, + }) + .composite( + tiles.map((input, index) => ({ + input, + left: (index % columns) * TILE_SIZE, + top: Math.floor(index / columns) * TILE_SIZE, + })) + ) + .jpeg({ quality: 80 }) + .toBuffer(); + if (signal.aborted) throw new Error("Video contact sheet was aborted"); + if (output.byteLength > MAX_SHEET_BYTES) return fallback(frames); + return { + dataUri: `data:image/jpeg;base64,${output.toString("base64")}`, + frames: frames.map((frame) => ({ ...frame })), + height: rows * TILE_SIZE, + timestamps: frames.map((frame) => frame.timestampSeconds), + used: true, + width: columns * TILE_SIZE, + }; + } catch (error) { + if (signal.aborted) throw new Error("Video contact sheet was aborted"); + return fallback(frames); + } finally { + if (timeout) clearTimeout(timeout); + } +} diff --git a/src/lib/guardrails/videoBridgeHelpers.ts b/src/lib/guardrails/videoBridgeHelpers.ts index f447d4369a..9af6aba355 100644 --- a/src/lib/guardrails/videoBridgeHelpers.ts +++ b/src/lib/guardrails/videoBridgeHelpers.ts @@ -3,6 +3,7 @@ import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/medi import { fetchRemoteMedia, type RemoteMediaFetchResult } from "@/shared/network/remoteImageFetch"; import { fuseVideoAndAudio } from "./videoAudioFusion"; +import { buildVideoContactSheet } from "./videoBridgeContactSheet"; import { extractVideoFramesViaBroker, type BrokerExtractionOptions, @@ -38,6 +39,7 @@ export interface VideoPart { focusWindow?: { endSeconds?: number; startSeconds?: number }; transcript?: unknown; audioTranscript?: unknown; + contactSheet?: boolean; } export type VideoTranscriptSource = "audio-bridge" | "client" | "embedded"; @@ -176,6 +178,9 @@ export function extractVideoParts(body: VideoRequestBody): VideoPart[] { const audioTranscript = objects.find( (object) => object.audioTranscript !== undefined )?.audioTranscript; + const contactSheet = objects.find( + (object) => object.contactSheet !== undefined + )?.contactSheet; return { container, ...(startSeconds === undefined && endSeconds === undefined @@ -187,6 +192,7 @@ export function extractVideoParts(body: VideoRequestBody): VideoPart[] { shape: part.shape as VideoPart["shape"], ...(transcript === undefined ? {} : { transcript }), ...(audioTranscript === undefined ? {} : { audioTranscript }), + ...(contactSheet === undefined ? {} : { contactSheet: contactSheet === true }), }; }); } @@ -244,6 +250,7 @@ export interface DescribedVideo { dedupDropped?: number; focusWindow?: VideoFocusWindow; transcriptCues?: VideoTranscriptCue[]; + contactSheetUsed?: boolean; } export interface VideoCaptionFrame { @@ -436,17 +443,33 @@ export async function describeVideoPart( }); const deduplicated = await deduplicateVideoFrames(extracted.frames); + const contactSheet = part.contactSheet + ? await buildVideoContactSheet(deduplicated.frames, { + signal, + timeoutMs: options.timeoutMs, + }) + : null; + const framesToCaption = + contactSheet?.used && contactSheet.dataUri + ? [{ dataUri: contactSheet.dataUri, timestampSeconds: 0 }] + : deduplicated.frames; const focusWindow = options.focusWindow ? resolveVideoFocusWindow(extracted.durationSeconds, options.focusWindow) : null; let transcriptCues = normalizeVideoTranscript(part.transcript, extracted.durationSeconds); const descriptions: string[] = []; - for (const frame of deduplicated.frames) { + for (const frame of framesToCaption) { if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted"); try { const caption = (await captionFrame(frame.dataUri, frame.timestampSeconds, signal)).trim(); if (caption) { - descriptions.push(`frame@t=${formatVideoTimestamp(frame.timestampSeconds)} ${caption}`); + descriptions.push( + `${ + contactSheet?.used + ? `contact-sheet[timestamps=${contactSheet.timestamps.map(formatVideoTimestamp).join(",")}]` + : `frame@t=${formatVideoTimestamp(frame.timestampSeconds)}` + } ${caption}` + ); } } catch { if (signal.aborted) { @@ -505,6 +528,7 @@ export async function describeVideoPart( focusWindow: focusWindow ?? undefined, sampling: extracted.sampling, transcriptCues: transcriptCues.length > 0 ? transcriptCues : undefined, + contactSheetUsed: contactSheet?.used || undefined, }; } catch (error) { if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted"); diff --git a/tests/unit/guardrails/videoBridgeContactSheet.test.ts b/tests/unit/guardrails/videoBridgeContactSheet.test.ts new file mode 100644 index 0000000000..578baccefb --- /dev/null +++ b/tests/unit/guardrails/videoBridgeContactSheet.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import sharp from "sharp"; + +import { describeVideoPart } from "../../../src/lib/guardrails/videoBridgeHelpers"; +import { buildVideoContactSheet } from "../../../src/lib/guardrails/videoBridgeContactSheet"; + +async function frame(color: string, timestampSeconds: number) { + const bytes = await sharp({ + create: { background: color, channels: 3, height: 24, width: 32 }, + }) + .jpeg() + .toBuffer(); + return { dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, timestampSeconds }; +} + +test("builds a bounded contact sheet and preserves timestamp labels", async () => { + const result = await buildVideoContactSheet([ + await frame("red", 1), + await frame("green", 5), + await frame("blue", 9), + ]); + + assert.equal(result.used, true); + assert.match(result.dataUri ?? "", /^data:image\/jpeg;base64,/); + assert.deepEqual(result.timestamps, [1, 5, 9]); + assert.equal(result.frames.length, 3); +}); + +test("contact sheet falls back to individual frames when decoding fails", async () => { + const frames = [{ dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 2 }]; + const result = await buildVideoContactSheet(frames); + assert.equal(result.used, false); + assert.equal(result.fallbackReason, "CONTACT_SHEET_UNAVAILABLE"); + assert.deepEqual(result.frames, frames); +}); + +test("contact sheet respects the parent abort signal", async () => { + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + buildVideoContactSheet([await frame("red", 1)], { signal: controller.signal }), + /aborted/i + ); +}); + +test("Video Bridge uses the sheet only when explicitly requested", async () => { + const sourceFrames = [await frame("red", 1), await frame("blue", 5)]; + let captionCalls = 0; + const result = await describeVideoPart( + { + container: "messages", + contactSheet: true, + messageIndex: 0, + partIndex: 0, + ref: "data:video/mp4;base64,AA==", + shape: "data_uri_string", + }, + { frameCount: 2, timeoutMs: 5000 }, + async () => { + captionCalls += 1; + return "combined scene"; + }, + { + extractFrames: async () => ({ durationSeconds: 6, frames: sourceFrames }), + } + ); + + assert.equal(captionCalls, 1); + assert.equal(result.contactSheetUsed, true); + assert.match(result.description, /contact-sheet\[timestamps=00:01\.000,00:05\.000\]/); +});