feat(video): add conservative frame deduplication

docs/security/GUARDRAILS.md

@@ -317,7 +317,11 @@ applied after selection in every policy.
 Each frame is limited to 4 MiB, all raw frames together to 23 MiB, and the
 serialized broker response to 32 MiB. A private temporary directory is removed
 in `finally`. OmniRoute does not bundle FFmpeg and does not accept a custom
-executable path.
+executable path. Before captioning, the bridge applies a conservative visual
+deduplication pass: each JPEG is reduced to a 16×16 grayscale buffer and is
+compared only with the last frame retained. The first and final timeline frames
+are always retained; comparator or decoder errors fail open and keep coverage.
+The output metadata reports how many frames were dropped.
 
 Frames are captioned sequentially with the configured Video model. An empty
 Video override inherits the Vision setting; if both are empty, the Vision

src/lib/guardrails/videoBridge.ts

@@ -55,6 +55,7 @@ interface VideoResultCacheMetadata {
   framesRequested: number;
   framesExtracted: number;
   framesUsed: number;
+  dedupDropped?: number;
   samplingCandidateCount?: number;
   samplingPolicyEffective?: "uniform" | "scene_aware";
   samplingPolicyRequested?: "uniform" | "scene_aware";
@@ -91,6 +92,8 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe
     typeof record.framesRequested === "number" &&
     typeof record.framesExtracted === "number" &&
     typeof record.framesUsed === "number" &&
+    (record.dedupDropped === undefined ||
+      (typeof record.dedupDropped === "number" && record.dedupDropped >= 0)) &&
     typeof record.cacheBytes === "number" &&
     typeof record.modelUsed === "string" &&
     (record.samplingCandidateCount === undefined ||
@@ -165,6 +168,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
     let totalDurationSeconds = 0;
     let totalCacheHits = 0;
     let totalSamplingCandidateCount = 0;
+    let totalDedupDropped = 0;
     let samplingPolicyEffective: "uniform" | "scene_aware" = "uniform";
     let failures = 0;
 
@@ -205,6 +209,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
             totalFramesRequested += meta.framesRequested;
             totalFramesExtracted += meta.framesExtracted;
             totalFramesUsed += meta.framesUsed;
+            totalDedupDropped += meta.dedupDropped ?? 0;
             totalDurationSeconds += meta.durationSeconds;
             totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0;
             if (meta.samplingPolicyEffective === "scene_aware") {
@@ -246,6 +251,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         totalFramesRequested += described.framesRequested;
         totalFramesExtracted += described.framesExtracted ?? described.framesUsed;
         totalFramesUsed += described.framesUsed;
+        totalDedupDropped += described.dedupDropped ?? 0;
         totalDurationSeconds += described.durationSeconds;
         totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0;
         if (described.sampling?.policyEffective === "scene_aware") {
@@ -271,6 +277,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
               framesRequested: described.framesRequested,
               framesExtracted: described.framesExtracted ?? described.framesUsed,
               framesUsed: described.framesUsed,
+              dedupDropped: described.dedupDropped ?? 0,
               cacheBytes: resultCacheBytes,
               modelUsed: described.modelUsed ?? selectedModel,
               samplingCandidateCount: described.sampling?.candidateCount ?? 0,
@@ -340,6 +347,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         framesExtracted: totalFramesExtracted,
         framesRequested: totalFramesRequested,
         framesUsed: totalFramesUsed,
+        dedupDropped: totalDedupDropped,
         samplingCandidateCount: totalSamplingCandidateCount,
         samplingPolicyEffective,
         samplingPolicyRequested: runtime.samplingPolicy,

src/lib/guardrails/videoBridgeHelpers.ts

@@ -111,6 +111,81 @@ export interface DescribedVideo {
   framesUsed: number;
   modelUsed?: string;
   sampling?: VideoSamplingMetadata;
+  dedupDropped?: number;
+}
+
+export interface VideoCaptionFrame {
+  dataUri: string;
+  timestampSeconds: number;
+}
+
+export interface VideoFrameDeduplicationResult {
+  dropped: number;
+  frames: VideoCaptionFrame[];
+}
+
+type VideoFrameComparator = (
+  previous: VideoCaptionFrame,
+  current: VideoCaptionFrame
+) => Promise<number>;
+
+const VIDEO_DEDUP_THRESHOLD = 0.04;
+
+async function compareVideoFramesByGrayscale(
+  previous: VideoCaptionFrame,
+  current: VideoCaptionFrame
+): Promise<number> {
+  const decode = (dataUri: string): Buffer => {
+    const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri);
+    if (!match) throw new Error("Video frame is not a JPEG data URI");
+    return Buffer.from(match[1], "base64");
+  };
+  const { default: sharp } = await import("sharp");
+  const [left, right] = await Promise.all(
+    [previous, current].map((frame) =>
+      sharp(decode(frame.dataUri)).resize(16, 16, { fit: "fill" }).greyscale().raw().toBuffer()
+    )
+  );
+  if (left.length !== right.length || left.length === 0) {
+    throw new Error("Video frame comparison returned invalid dimensions");
+  }
+  let difference = 0;
+  for (let index = 0; index < left.length; index++) {
+    difference += Math.abs(left[index] - right[index]) / 255;
+  }
+  return difference / left.length;
+}
+
+export async function deduplicateVideoFrames(
+  frames: readonly VideoCaptionFrame[],
+  options: { compare?: VideoFrameComparator; threshold?: number } = {}
+): Promise<VideoFrameDeduplicationResult> {
+  if (frames.length < 2) return { dropped: 0, frames: [...frames] };
+  const compare = options.compare ?? compareVideoFramesByGrayscale;
+  const threshold =
+    typeof options.threshold === "number" && Number.isFinite(options.threshold)
+      ? Math.max(0, Math.min(1, options.threshold))
+      : VIDEO_DEDUP_THRESHOLD;
+  const kept: VideoCaptionFrame[] = [frames[0]];
+  let dropped = 0;
+  for (let index = 1; index < frames.length; index++) {
+    const current = frames[index];
+    if (index === frames.length - 1) {
+      kept.push(current);
+      continue;
+    }
+    try {
+      const distance = await compare(kept[kept.length - 1], current);
+      if (Number.isFinite(distance) && distance <= threshold) {
+        dropped += 1;
+        continue;
+      }
+    } catch {
+      // A malformed or unsupported frame must never reduce visual coverage.
+    }
+    kept.push(current);
+  }
+  return { dropped, frames: kept };
 }
 
 function normalizeBase64(base64: string): string {
@@ -223,8 +298,9 @@ export async function describeVideoPart(
       timeoutMs: options.timeoutMs,
     });
 
+    const deduplicated = await deduplicateVideoFrames(extracted.frames);
     const descriptions: string[] = [];
-    for (const frame of extracted.frames) {
+    for (const frame of deduplicated.frames) {
       if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted");
       try {
         const caption = (await captionFrame(frame.dataUri, frame.timestampSeconds, signal)).trim();
@@ -247,6 +323,7 @@ export async function describeVideoPart(
       framesExtracted: extracted.frames.length,
       framesRequested: options.frameCount,
       framesUsed: descriptions.length,
+      dedupDropped: deduplicated.dropped,
       sampling: extracted.sampling,
     };
   } catch (error) {

tests/unit/guardrails/videoBridge.test.ts

@@ -100,6 +100,7 @@ test("preserves scene-aware sampler metadata in guardrail meta and the transpare
         framesRequested: 4,
         framesExtracted: 4,
         framesUsed: 4,
+        dedupDropped: 1,
         sampling: {
           candidateCount: 3,
           policyEffective: "scene_aware",
@@ -113,6 +114,7 @@ test("preserves scene-aware sampler metadata in guardrail meta and the transpare
   assert.equal(result.meta?.samplingPolicyRequested, "scene_aware");
   assert.equal(result.meta?.samplingPolicyEffective, "scene_aware");
   assert.equal(result.meta?.samplingCandidateCount, 3);
+  assert.equal(result.meta?.dedupDropped, 1);
   assert.equal(
     buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]),
     "video->text;model=openai/gpt-4o-mini;parts=1;sampling=scene_aware;candidates=3"

tests/unit/guardrails/videoBridgeDedup.test.ts

@@ -0,0 +1,49 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  deduplicateVideoFrames,
+  type VideoCaptionFrame,
+} from "../../../src/lib/guardrails/videoBridgeHelpers.ts";
+
+const frame = (
+  timestampSeconds: number,
+  dataUri = "data:image/jpeg;base64,QQ=="
+): VideoCaptionFrame => ({
+  dataUri,
+  timestampSeconds,
+});
+
+test("deduplication keeps the first frame and the final frame while dropping redundant middle frames", async () => {
+  const result = await deduplicateVideoFrames([frame(1), frame(2), frame(3), frame(4)], {
+    compare: async () => 0.01,
+    threshold: 0.05,
+  });
+
+  assert.deepEqual(
+    result.frames.map((item) => item.timestampSeconds),
+    [1, 4]
+  );
+  assert.equal(result.dropped, 2);
+});
+
+test("deduplication keeps visually distinct frames", async () => {
+  const result = await deduplicateVideoFrames([frame(1), frame(2), frame(3)], {
+    compare: async () => 0.2,
+    threshold: 0.05,
+  });
+
+  assert.equal(result.frames.length, 3);
+  assert.equal(result.dropped, 0);
+});
+
+test("deduplication fails open when the visual comparator errors", async () => {
+  const result = await deduplicateVideoFrames([frame(1), frame(2)], {
+    compare: async () => {
+      throw new Error("invalid JPEG");
+    },
+  });
+
+  assert.equal(result.frames.length, 2);
+  assert.equal(result.dropped, 0);
+});