feat(video): add timestamped contact sheets

src/lib/guardrails/modalityBridge/bridgeCache.ts

@@ -16,6 +16,7 @@ export interface BridgeCacheKeyOptions {
   strategy?: string;
   frameCount?: number;
   maxVideos?: number;
+  contactSheet?: boolean;
   transcript?: string;
   version?: string;
 }
@@ -39,6 +40,7 @@ export function bridgeCacheKey(
     strategy: options.strategy,
     frameCount: options.frameCount,
     maxVideos: options.maxVideos,
+    contactSheet: options.contactSheet,
     transcript: options.transcript,
     version: options.version,
   };

src/lib/guardrails/videoBridge.ts

@@ -71,6 +71,7 @@ interface VideoResultCacheMetadata {
   samplingPolicyEffective?: "uniform" | "scene_aware" | "segment_aware";
   samplingPolicyRequested?: "uniform" | "scene_aware" | "segment_aware";
   transcriptCuesApplied?: number;
+  contactSheetUsed?: boolean;
   cacheBytes: number;
   modelUsed: string;
 }
@@ -119,7 +120,8 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe
       record.samplingPolicyRequested === "scene_aware" ||
       record.samplingPolicyRequested === "segment_aware") &&
     (record.transcriptCuesApplied === undefined ||
-      (typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0))
+      (typeof record.transcriptCuesApplied === "number" && record.transcriptCuesApplied >= 0)) &&
+    (record.contactSheetUsed === undefined || typeof record.contactSheetUsed === "boolean")
   );
 }
 
@@ -187,6 +189,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
     let totalDedupDropped = 0;
     let focusWindowsApplied = 0;
     let transcriptCuesApplied = 0;
+    let contactSheetsUsed = 0;
     let samplingPolicyEffective: "uniform" | "scene_aware" | "segment_aware" = "uniform";
     let failures = 0;
 
@@ -209,6 +212,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
                 focusEndSeconds: part.focusWindow?.endSeconds ?? null,
                 focusStartSeconds: part.focusWindow?.startSeconds ?? null,
                 transcript: safeTranscriptFingerprint(part.transcript),
+                contactSheet: part.contactSheet ?? false,
                 version: VIDEO_BRIDGE_RESULT_CACHE_VERSION,
               })
             : null;
@@ -240,6 +244,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
             totalDurationSeconds += meta.durationSeconds;
             totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0;
             transcriptCuesApplied += meta.transcriptCuesApplied ?? 0;
+            if (meta.contactSheetUsed) contactSheetsUsed += 1;
             if (meta.samplingPolicyEffective && meta.samplingPolicyEffective !== "uniform") {
               samplingPolicyEffective = meta.samplingPolicyEffective;
             }
@@ -282,6 +287,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         totalDedupDropped += described.dedupDropped ?? 0;
         if (described.focusWindow) focusWindowsApplied += 1;
         transcriptCuesApplied += described.transcriptCues?.length ?? 0;
+        if (described.contactSheetUsed) contactSheetsUsed += 1;
         totalDurationSeconds += described.durationSeconds;
         totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0;
         if (
@@ -320,6 +326,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
               samplingPolicyRequested:
                 described.sampling?.policyRequested ?? runtime.samplingPolicy,
               transcriptCuesApplied: described.transcriptCues?.length ?? 0,
+              contactSheetUsed: described.contactSheetUsed ?? false,
             },
           });
           recordBridgeUse("video", {
@@ -386,6 +393,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         dedupDropped: totalDedupDropped,
         focusWindowsApplied,
         transcriptCuesApplied,
+        contactSheetsUsed,
         samplingCandidateCount: totalSamplingCandidateCount,
         samplingPolicyEffective,
         samplingPolicyRequested: runtime.samplingPolicy,

src/lib/guardrails/videoBridgeContactSheet.ts

@@ -0,0 +1,110 @@
+export interface ContactSheetFrame {
+  dataUri: string;
+  timestampSeconds: number;
+}
+
+export interface ContactSheetOptions {
+  columns?: number;
+  signal?: AbortSignal;
+  timeoutMs?: number;
+}
+
+export interface VideoContactSheetResult {
+  dataUri?: string;
+  fallbackReason?: "CONTACT_SHEET_UNAVAILABLE";
+  frames: ContactSheetFrame[];
+  height?: number;
+  timestamps: number[];
+  used: boolean;
+  width?: number;
+}
+
+const MAX_FRAMES = 16;
+const MAX_SHEET_BYTES = 32 * 1024 * 1024;
+const TILE_SIZE = 512;
+
+function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult {
+  return {
+    fallbackReason: "CONTACT_SHEET_UNAVAILABLE",
+    frames: frames.map((frame) => ({ ...frame })),
+    timestamps: frames.map((frame) => frame.timestampSeconds),
+    used: false,
+  };
+}
+
+function decodeFrame(dataUri: string): Buffer {
+  const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri);
+  if (!match) throw new Error("Contact sheet requires JPEG data URIs");
+  return Buffer.from(match[1], "base64");
+}
+
+/** Build an optional bounded JPEG grid; every failure except abort is fail-safe to individual frames. */
+export async function buildVideoContactSheet(
+  frames: readonly ContactSheetFrame[],
+  options: ContactSheetOptions = {}
+): Promise<VideoContactSheetResult> {
+  if (options.signal?.aborted) throw new Error("Video contact sheet was aborted");
+  if (frames.length < 1 || frames.length > MAX_FRAMES) return fallback(frames);
+  if (
+    frames.some(
+      (frame) =>
+        !Number.isFinite(frame.timestampSeconds) || frame.timestampSeconds < 0 || !frame.dataUri
+    )
+  ) {
+    return fallback(frames);
+  }
+  const columns = Math.min(4, Math.max(1, Math.floor(options.columns ?? 2)), frames.length);
+  const rows = Math.ceil(frames.length / columns);
+  const controller = new AbortController();
+  const timeout = options.timeoutMs
+    ? setTimeout(() => controller.abort(), options.timeoutMs)
+    : null;
+  const signal = options.signal
+    ? AbortSignal.any([options.signal, controller.signal])
+    : controller.signal;
+  try {
+    const { default: sharp } = await import("sharp");
+    if (signal.aborted) throw new Error("Video contact sheet was aborted");
+    const tiles = await Promise.all(
+      frames.map(async (frame) =>
+        sharp(decodeFrame(frame.dataUri))
+          .resize(TILE_SIZE, TILE_SIZE, { fit: "contain", background: "#000000" })
+          .jpeg({ quality: 80 })
+          .toBuffer()
+      )
+    );
+    if (signal.aborted) throw new Error("Video contact sheet was aborted");
+    const output = await sharp({
+      create: {
+        background: "#000000",
+        channels: 3,
+        height: rows * TILE_SIZE,
+        width: columns * TILE_SIZE,
+      },
+    })
+      .composite(
+        tiles.map((input, index) => ({
+          input,
+          left: (index % columns) * TILE_SIZE,
+          top: Math.floor(index / columns) * TILE_SIZE,
+        }))
+      )
+      .jpeg({ quality: 80 })
+      .toBuffer();
+    if (signal.aborted) throw new Error("Video contact sheet was aborted");
+    if (output.byteLength > MAX_SHEET_BYTES) return fallback(frames);
+    return {
+      dataUri: `data:image/jpeg;base64,${output.toString("base64")}`,
+      frames: frames.map((frame) => ({ ...frame })),
+      height: rows * TILE_SIZE,
+      timestamps: frames.map((frame) => frame.timestampSeconds),
+      used: true,
+      width: columns * TILE_SIZE,
+    };
+  } catch (error) {
+    if (signal.aborted) throw new Error("Video contact sheet was aborted");
+    return fallback(frames);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}

src/lib/guardrails/videoBridgeHelpers.ts

@@ -3,6 +3,7 @@ import { detectMediaParts, type MediaPart } from "@omniroute/open-sse/utils/medi
 import { fetchRemoteMedia, type RemoteMediaFetchResult } from "@/shared/network/remoteImageFetch";
 
 import { fuseVideoAndAudio } from "./videoAudioFusion";
+import { buildVideoContactSheet } from "./videoBridgeContactSheet";
 import {
   extractVideoFramesViaBroker,
   type BrokerExtractionOptions,
@@ -38,6 +39,7 @@ export interface VideoPart {
   focusWindow?: { endSeconds?: number; startSeconds?: number };
   transcript?: unknown;
   audioTranscript?: unknown;
+  contactSheet?: boolean;
 }
 
 export type VideoTranscriptSource = "audio-bridge" | "client" | "embedded";
@@ -176,6 +178,9 @@ export function extractVideoParts(body: VideoRequestBody): VideoPart[] {
       const audioTranscript = objects.find(
         (object) => object.audioTranscript !== undefined
       )?.audioTranscript;
+      const contactSheet = objects.find(
+        (object) => object.contactSheet !== undefined
+      )?.contactSheet;
       return {
         container,
         ...(startSeconds === undefined && endSeconds === undefined
@@ -187,6 +192,7 @@ export function extractVideoParts(body: VideoRequestBody): VideoPart[] {
         shape: part.shape as VideoPart["shape"],
         ...(transcript === undefined ? {} : { transcript }),
         ...(audioTranscript === undefined ? {} : { audioTranscript }),
+        ...(contactSheet === undefined ? {} : { contactSheet: contactSheet === true }),
       };
     });
 }
@@ -244,6 +250,7 @@ export interface DescribedVideo {
   dedupDropped?: number;
   focusWindow?: VideoFocusWindow;
   transcriptCues?: VideoTranscriptCue[];
+  contactSheetUsed?: boolean;
 }
 
 export interface VideoCaptionFrame {
@@ -436,17 +443,33 @@ export async function describeVideoPart(
     });
 
     const deduplicated = await deduplicateVideoFrames(extracted.frames);
+    const contactSheet = part.contactSheet
+      ? await buildVideoContactSheet(deduplicated.frames, {
+          signal,
+          timeoutMs: options.timeoutMs,
+        })
+      : null;
+    const framesToCaption =
+      contactSheet?.used && contactSheet.dataUri
+        ? [{ dataUri: contactSheet.dataUri, timestampSeconds: 0 }]
+        : deduplicated.frames;
     const focusWindow = options.focusWindow
       ? resolveVideoFocusWindow(extracted.durationSeconds, options.focusWindow)
       : null;
     let transcriptCues = normalizeVideoTranscript(part.transcript, extracted.durationSeconds);
     const descriptions: string[] = [];
-    for (const frame of deduplicated.frames) {
+    for (const frame of framesToCaption) {
       if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted");
       try {
         const caption = (await captionFrame(frame.dataUri, frame.timestampSeconds, signal)).trim();
         if (caption) {
-          descriptions.push(`frame@t=${formatVideoTimestamp(frame.timestampSeconds)} ${caption}`);
+          descriptions.push(
+            `${
+              contactSheet?.used
+                ? `contact-sheet[timestamps=${contactSheet.timestamps.map(formatVideoTimestamp).join(",")}]`
+                : `frame@t=${formatVideoTimestamp(frame.timestampSeconds)}`
+            } ${caption}`
+          );
         }
       } catch {
         if (signal.aborted) {
@@ -505,6 +528,7 @@ export async function describeVideoPart(
       focusWindow: focusWindow ?? undefined,
       sampling: extracted.sampling,
       transcriptCues: transcriptCues.length > 0 ? transcriptCues : undefined,
+      contactSheetUsed: contactSheet?.used || undefined,
     };
   } catch (error) {
     if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted");

tests/unit/guardrails/videoBridgeContactSheet.test.ts

@@ -0,0 +1,73 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import sharp from "sharp";
+
+import { describeVideoPart } from "../../../src/lib/guardrails/videoBridgeHelpers";
+import { buildVideoContactSheet } from "../../../src/lib/guardrails/videoBridgeContactSheet";
+
+async function frame(color: string, timestampSeconds: number) {
+  const bytes = await sharp({
+    create: { background: color, channels: 3, height: 24, width: 32 },
+  })
+    .jpeg()
+    .toBuffer();
+  return { dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, timestampSeconds };
+}
+
+test("builds a bounded contact sheet and preserves timestamp labels", async () => {
+  const result = await buildVideoContactSheet([
+    await frame("red", 1),
+    await frame("green", 5),
+    await frame("blue", 9),
+  ]);
+
+  assert.equal(result.used, true);
+  assert.match(result.dataUri ?? "", /^data:image\/jpeg;base64,/);
+  assert.deepEqual(result.timestamps, [1, 5, 9]);
+  assert.equal(result.frames.length, 3);
+});
+
+test("contact sheet falls back to individual frames when decoding fails", async () => {
+  const frames = [{ dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 2 }];
+  const result = await buildVideoContactSheet(frames);
+  assert.equal(result.used, false);
+  assert.equal(result.fallbackReason, "CONTACT_SHEET_UNAVAILABLE");
+  assert.deepEqual(result.frames, frames);
+});
+
+test("contact sheet respects the parent abort signal", async () => {
+  const controller = new AbortController();
+  controller.abort();
+  await assert.rejects(
+    buildVideoContactSheet([await frame("red", 1)], { signal: controller.signal }),
+    /aborted/i
+  );
+});
+
+test("Video Bridge uses the sheet only when explicitly requested", async () => {
+  const sourceFrames = [await frame("red", 1), await frame("blue", 5)];
+  let captionCalls = 0;
+  const result = await describeVideoPart(
+    {
+      container: "messages",
+      contactSheet: true,
+      messageIndex: 0,
+      partIndex: 0,
+      ref: "data:video/mp4;base64,AA==",
+      shape: "data_uri_string",
+    },
+    { frameCount: 2, timeoutMs: 5000 },
+    async () => {
+      captionCalls += 1;
+      return "combined scene";
+    },
+    {
+      extractFrames: async () => ({ durationSeconds: 6, frames: sourceFrames }),
+    }
+  );
+
+  assert.equal(captionCalls, 1);
+  assert.equal(result.contactSheetUsed, true);
+  assert.match(result.description, /contact-sheet\[timestamps=00:01\.000,00:05\.000\]/);
+});