diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index c4bfb8b700..c1743e4ba8 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -317,7 +317,11 @@ applied after selection in every policy. Each frame is limited to 4 MiB, all raw frames together to 23 MiB, and the serialized broker response to 32 MiB. A private temporary directory is removed in `finally`. OmniRoute does not bundle FFmpeg and does not accept a custom -executable path. +executable path. Before captioning, the bridge applies a conservative visual +deduplication pass: each JPEG is reduced to a 16×16 grayscale buffer and is +compared only with the last frame retained. The first and final timeline frames +are always retained; comparator or decoder errors fail open and keep coverage. +The output metadata reports how many frames were dropped. Frames are captioned sequentially with the configured Video model. An empty Video override inherits the Vision setting; if both are empty, the Vision diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 2af84a06d9..1b2341f58a 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -55,6 +55,7 @@ interface VideoResultCacheMetadata { framesRequested: number; framesExtracted: number; framesUsed: number; + dedupDropped?: number; samplingCandidateCount?: number; samplingPolicyEffective?: "uniform" | "scene_aware"; samplingPolicyRequested?: "uniform" | "scene_aware"; @@ -91,6 +92,8 @@ function isVideoResultCacheMetadata(value: unknown): value is VideoResultCacheMe typeof record.framesRequested === "number" && typeof record.framesExtracted === "number" && typeof record.framesUsed === "number" && + (record.dedupDropped === undefined || + (typeof record.dedupDropped === "number" && record.dedupDropped >= 0)) && typeof record.cacheBytes === "number" && typeof record.modelUsed === "string" && (record.samplingCandidateCount === undefined || @@ -165,6 +168,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { let totalDurationSeconds = 0; let totalCacheHits = 0; let totalSamplingCandidateCount = 0; + let totalDedupDropped = 0; let samplingPolicyEffective: "uniform" | "scene_aware" = "uniform"; let failures = 0; @@ -205,6 +209,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { totalFramesRequested += meta.framesRequested; totalFramesExtracted += meta.framesExtracted; totalFramesUsed += meta.framesUsed; + totalDedupDropped += meta.dedupDropped ?? 0; totalDurationSeconds += meta.durationSeconds; totalSamplingCandidateCount += meta.samplingCandidateCount ?? 0; if (meta.samplingPolicyEffective === "scene_aware") { @@ -246,6 +251,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { totalFramesRequested += described.framesRequested; totalFramesExtracted += described.framesExtracted ?? described.framesUsed; totalFramesUsed += described.framesUsed; + totalDedupDropped += described.dedupDropped ?? 0; totalDurationSeconds += described.durationSeconds; totalSamplingCandidateCount += described.sampling?.candidateCount ?? 0; if (described.sampling?.policyEffective === "scene_aware") { @@ -271,6 +277,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { framesRequested: described.framesRequested, framesExtracted: described.framesExtracted ?? described.framesUsed, framesUsed: described.framesUsed, + dedupDropped: described.dedupDropped ?? 0, cacheBytes: resultCacheBytes, modelUsed: described.modelUsed ?? selectedModel, samplingCandidateCount: described.sampling?.candidateCount ?? 0, @@ -340,6 +347,7 @@ export class VideoBridgeGuardrail extends BaseGuardrail { framesExtracted: totalFramesExtracted, framesRequested: totalFramesRequested, framesUsed: totalFramesUsed, + dedupDropped: totalDedupDropped, samplingCandidateCount: totalSamplingCandidateCount, samplingPolicyEffective, samplingPolicyRequested: runtime.samplingPolicy, diff --git a/src/lib/guardrails/videoBridgeHelpers.ts b/src/lib/guardrails/videoBridgeHelpers.ts index 57b953a4ba..e3ac810bbb 100644 --- a/src/lib/guardrails/videoBridgeHelpers.ts +++ b/src/lib/guardrails/videoBridgeHelpers.ts @@ -111,6 +111,81 @@ export interface DescribedVideo { framesUsed: number; modelUsed?: string; sampling?: VideoSamplingMetadata; + dedupDropped?: number; +} + +export interface VideoCaptionFrame { + dataUri: string; + timestampSeconds: number; +} + +export interface VideoFrameDeduplicationResult { + dropped: number; + frames: VideoCaptionFrame[]; +} + +type VideoFrameComparator = ( + previous: VideoCaptionFrame, + current: VideoCaptionFrame +) => Promise; + +const VIDEO_DEDUP_THRESHOLD = 0.04; + +async function compareVideoFramesByGrayscale( + previous: VideoCaptionFrame, + current: VideoCaptionFrame +): Promise { + const decode = (dataUri: string): Buffer => { + const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri); + if (!match) throw new Error("Video frame is not a JPEG data URI"); + return Buffer.from(match[1], "base64"); + }; + const { default: sharp } = await import("sharp"); + const [left, right] = await Promise.all( + [previous, current].map((frame) => + sharp(decode(frame.dataUri)).resize(16, 16, { fit: "fill" }).greyscale().raw().toBuffer() + ) + ); + if (left.length !== right.length || left.length === 0) { + throw new Error("Video frame comparison returned invalid dimensions"); + } + let difference = 0; + for (let index = 0; index < left.length; index++) { + difference += Math.abs(left[index] - right[index]) / 255; + } + return difference / left.length; +} + +export async function deduplicateVideoFrames( + frames: readonly VideoCaptionFrame[], + options: { compare?: VideoFrameComparator; threshold?: number } = {} +): Promise { + if (frames.length < 2) return { dropped: 0, frames: [...frames] }; + const compare = options.compare ?? compareVideoFramesByGrayscale; + const threshold = + typeof options.threshold === "number" && Number.isFinite(options.threshold) + ? Math.max(0, Math.min(1, options.threshold)) + : VIDEO_DEDUP_THRESHOLD; + const kept: VideoCaptionFrame[] = [frames[0]]; + let dropped = 0; + for (let index = 1; index < frames.length; index++) { + const current = frames[index]; + if (index === frames.length - 1) { + kept.push(current); + continue; + } + try { + const distance = await compare(kept[kept.length - 1], current); + if (Number.isFinite(distance) && distance <= threshold) { + dropped += 1; + continue; + } + } catch { + // A malformed or unsupported frame must never reduce visual coverage. + } + kept.push(current); + } + return { dropped, frames: kept }; } function normalizeBase64(base64: string): string { @@ -223,8 +298,9 @@ export async function describeVideoPart( timeoutMs: options.timeoutMs, }); + const deduplicated = await deduplicateVideoFrames(extracted.frames); const descriptions: string[] = []; - for (const frame of extracted.frames) { + for (const frame of deduplicated.frames) { if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted"); try { const caption = (await captionFrame(frame.dataUri, frame.timestampSeconds, signal)).trim(); @@ -247,6 +323,7 @@ export async function describeVideoPart( framesExtracted: extracted.frames.length, framesRequested: options.frameCount, framesUsed: descriptions.length, + dedupDropped: deduplicated.dropped, sampling: extracted.sampling, }; } catch (error) { diff --git a/tests/unit/guardrails/videoBridge.test.ts b/tests/unit/guardrails/videoBridge.test.ts index 200f804aff..c3c10e08a3 100644 --- a/tests/unit/guardrails/videoBridge.test.ts +++ b/tests/unit/guardrails/videoBridge.test.ts @@ -100,6 +100,7 @@ test("preserves scene-aware sampler metadata in guardrail meta and the transpare framesRequested: 4, framesExtracted: 4, framesUsed: 4, + dedupDropped: 1, sampling: { candidateCount: 3, policyEffective: "scene_aware", @@ -113,6 +114,7 @@ test("preserves scene-aware sampler metadata in guardrail meta and the transpare assert.equal(result.meta?.samplingPolicyRequested, "scene_aware"); assert.equal(result.meta?.samplingPolicyEffective, "scene_aware"); assert.equal(result.meta?.samplingCandidateCount, 3); + assert.equal(result.meta?.dedupDropped, 1); assert.equal( buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]), "video->text;model=openai/gpt-4o-mini;parts=1;sampling=scene_aware;candidates=3" diff --git a/tests/unit/guardrails/videoBridgeDedup.test.ts b/tests/unit/guardrails/videoBridgeDedup.test.ts new file mode 100644 index 0000000000..bbadb0bc3e --- /dev/null +++ b/tests/unit/guardrails/videoBridgeDedup.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + deduplicateVideoFrames, + type VideoCaptionFrame, +} from "../../../src/lib/guardrails/videoBridgeHelpers.ts"; + +const frame = ( + timestampSeconds: number, + dataUri = "data:image/jpeg;base64,QQ==" +): VideoCaptionFrame => ({ + dataUri, + timestampSeconds, +}); + +test("deduplication keeps the first frame and the final frame while dropping redundant middle frames", async () => { + const result = await deduplicateVideoFrames([frame(1), frame(2), frame(3), frame(4)], { + compare: async () => 0.01, + threshold: 0.05, + }); + + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 4] + ); + assert.equal(result.dropped, 2); +}); + +test("deduplication keeps visually distinct frames", async () => { + const result = await deduplicateVideoFrames([frame(1), frame(2), frame(3)], { + compare: async () => 0.2, + threshold: 0.05, + }); + + assert.equal(result.frames.length, 3); + assert.equal(result.dropped, 0); +}); + +test("deduplication fails open when the visual comparator errors", async () => { + const result = await deduplicateVideoFrames([frame(1), frame(2)], { + compare: async () => { + throw new Error("invalid JPEG"); + }, + }); + + assert.equal(result.frames.length, 2); + assert.equal(result.dropped, 0); +});