feat(video): add conservative frame deduplication

src/lib/guardrails/videoBridge.ts

@@ -55,6 +55,7 @@ interface VideoResultCacheMetadata {
   framesRequested: number;
   framesExtracted: number;
   framesUsed: number;
+  dedupDropped?: number;
   samplingCandidateCount?: number;
   samplingPolicyEffective?: "uniform" | "scene_aware";
   samplingPolicyRequested?: "uniform" | "scene_aware";
@@ -91,6 +92,8 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe
     typeof record.framesRequested === "number" &&
     typeof record.framesExtracted === "number" &&
     typeof record.framesUsed === "number" &&
+    (record.dedupDropped === undefined ||
+      (typeof record.dedupDropped === "number" && record.dedupDropped >= 0)) &&
     typeof record.cacheBytes === "number" &&
     typeof record.modelUsed === "string" &&
     (record.samplingCandidateCount === undefined ||
@@ -165,6 +168,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
     let totalDurationSeconds = 0;
     let totalCacheHits = 0;
     let totalSamplingCandidateCount = 0;
+    let totalDedupDropped = 0;
     let samplingPolicyEffective: "uniform" | "scene_aware" = "uniform";
     let failures = 0;
 
@@ -205,6 +209,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
             totalFramesRequested += meta.framesRequested;
             totalFramesExtracted += meta.framesExtracted;
             totalFramesUsed += meta.framesUsed;
+            totalDedupDropped += meta.dedupDropped ?? 0;
             totalDurationSeconds += meta.durationSeconds;
             totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0;
             if (meta.samplingPolicyEffective === "scene_aware") {
@@ -246,6 +251,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         totalFramesRequested += described.framesRequested;
         totalFramesExtracted += described.framesExtracted ?? described.framesUsed;
         totalFramesUsed += described.framesUsed;
+        totalDedupDropped += described.dedupDropped ?? 0;
         totalDurationSeconds += described.durationSeconds;
         totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0;
         if (described.sampling?.policyEffective === "scene_aware") {
@@ -271,6 +277,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
               framesRequested: described.framesRequested,
               framesExtracted: described.framesExtracted ?? described.framesUsed,
               framesUsed: described.framesUsed,
+              dedupDropped: described.dedupDropped ?? 0,
               cacheBytes: resultCacheBytes,
               modelUsed: described.modelUsed ?? selectedModel,
               samplingCandidateCount: described.sampling?.candidateCount ?? 0,
@@ -340,6 +347,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail {
         framesExtracted: totalFramesExtracted,
         framesRequested: totalFramesRequested,
         framesUsed: totalFramesUsed,
+        dedupDropped: totalDedupDropped,
         samplingCandidateCount: totalSamplingCandidateCount,
         samplingPolicyEffective,
         samplingPolicyRequested: runtime.samplingPolicy,

src/lib/guardrails/videoBridgeHelpers.ts

@@ -111,6 +111,81 @@ export interface DescribedVideo {
   framesUsed: number;
   modelUsed?: string;
   sampling?: VideoSamplingMetadata;
+  dedupDropped?: number;
+}
+
+export interface VideoCaptionFrame {
+  dataUri: string;
+  timestampSeconds: number;
+}
+
+export interface VideoFrameDeduplicationResult {
+  dropped: number;
+  frames: VideoCaptionFrame[];
+}
+
+type VideoFrameComparator = (
+  previous: VideoCaptionFrame,
+  current: VideoCaptionFrame
+) => Promise<number>;
+
+const VIDEO_DEDUP_THRESHOLD = 0.04;
+
+async function compareVideoFramesByGrayscale(
+  previous: VideoCaptionFrame,
+  current: VideoCaptionFrame
+): Promise<number> {
+  const decode = (dataUri: string): Buffer => {
+    const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri);
+    if (!match) throw new Error("Video frame is not a JPEG data URI");
+    return Buffer.from(match[1], "base64");
+  };
+  const { default: sharp } = await import("sharp");
+  const [left, right] = await Promise.all(
+    [previous, current].map((frame) =>
+      sharp(decode(frame.dataUri)).resize(16, 16, { fit: "fill" }).greyscale().raw().toBuffer()
+    )
+  );
+  if (left.length !== right.length || left.length === 0) {
+    throw new Error("Video frame comparison returned invalid dimensions");
+  }
+  let difference = 0;
+  for (let index = 0; index < left.length; index++) {
+    difference += Math.abs(left[index] - right[index]) / 255;
+  }
+  return difference / left.length;
+}
+
+export async function deduplicateVideoFrames(
+  frames: readonly VideoCaptionFrame[],
+  options: { compare?: VideoFrameComparator; threshold?: number } = {}
+): Promise<VideoFrameDeduplicationResult> {
+  if (frames.length < 2) return { dropped: 0, frames: [...frames] };
+  const compare = options.compare ?? compareVideoFramesByGrayscale;
+  const threshold =
+    typeof options.threshold === "number" && Number.isFinite(options.threshold)
+      ? Math.max(0, Math.min(1, options.threshold))
+      : VIDEO_DEDUP_THRESHOLD;
+  const kept: VideoCaptionFrame[] = [frames[0]];
+  let dropped = 0;
+  for (let index = 1; index < frames.length; index++) {
+    const current = frames[index];
+    if (index === frames.length - 1) {
+      kept.push(current);
+      continue;
+    }
+    try {
+      const distance = await compare(kept[kept.length - 1], current);
+      if (Number.isFinite(distance) && distance <= threshold) {
+        dropped += 1;
+        continue;
+      }
+    } catch {
+      // A malformed or unsupported frame must never reduce visual coverage.
+    }
+    kept.push(current);
+  }
+  return { dropped, frames: kept };
 }
 
 function normalizeBase64(base64: string): string {
@@ -223,8 +298,9 @@ export async function describeVideoPart(
       timeoutMs: options.timeoutMs,
     });
 
+    const deduplicated = await deduplicateVideoFrames(extracted.frames);
     const descriptions: string[] = [];
-    for (const frame of extracted.frames) {
+    for (const frame of deduplicated.frames) {
       if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted");
       try {
         const caption = (await captionFrame(frame.dataUri, frame.timestampSeconds, signal)).trim();
@@ -247,6 +323,7 @@ export async function describeVideoPart(
       framesExtracted: extracted.frames.length,
       framesRequested: options.frameCount,
       framesUsed: descriptions.length,
+      dedupDropped: deduplicated.dropped,
       sampling: extracted.sampling,
     };
   } catch (error) {