diff --git a/changelog.d/fixes/pending-video-bridge-dedup-policy.md b/changelog.d/fixes/pending-video-bridge-dedup-policy.md new file mode 100644 index 0000000000..58e0332a36 --- /dev/null +++ b/changelog.d/fixes/pending-video-bridge-dedup-policy.md @@ -0,0 +1 @@ +- **fix(video):** apply the caption-frame cap after bounded visual deduplication, preserve first/final candidates plus small high-contrast motion and text changes, and version the dedup policy in result-cache identity diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index f20cb80527..28cd0a4e38 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -1,13 +1,13 @@ --- title: "Guardrails" version: 3.8.50 -lastUpdated: 2026-08-14 +lastUpdated: 2026-08-24 --- # Guardrails > **Source of truth:** `src/lib/guardrails/` -> **Last updated:** 2026-08-15 — v3.8.50 (Video Bridge broker confinement) +> **Last updated:** 2026-08-24 — v3.8.50 (Video Bridge visual dedup hardening) Guardrails enforce safety, policy, and content transformations at the boundary between OmniRoute and upstream providers. Each guardrail can inspect (and @@ -340,11 +340,20 @@ serialized broker response to 32 MiB. A private temporary directory is removed in `finally`. OmniRoute does not bundle FFmpeg and does not accept a custom executable path. Before captioning, the bridge applies a conservative visual deduplication pass: each JPEG is reduced to a 16×16 grayscale buffer and is -compared only with the last frame retained, using a fixed similarity threshold -of 0.04 — a deliberate constant chosen for predictability, not a runtime -setting. The first and final timeline frames -are always retained; comparator or decoder errors fail open and keep coverage. -The output metadata reports how many frames were dropped. +compared only with the last frame retained. For a requested caption budget +above one frame, extraction supplies a +bounded candidate pool of up to twice that budget and never more than 16 frames. +The requested cap is applied only after deduplication, with the first and final +selected candidates preserved during final thinning when the budget is at least +two. The versioned +`grayscale-16x16-mean-cells-v2` policy uses the larger of mean luma delta and +the ratio of thumbnail cells whose normalized delta is at least 0.05. The +duplicate threshold is the constant 0.04, chosen for predictability rather than +exposed as a runtime setting. This secondary +high-contrast signal preserves small motion and visible-text changes that a +mean-only comparison can hide. Comparator or decoder errors fail open and keep +coverage. Output metadata separates extracted candidates, successfully used +frames, and visual duplicates dropped. An explicitly marked video part may request a timestamped contact sheet. The bridge builds at most a 4-column, 16-frame JPEG grid and labels the resulting @@ -401,7 +410,10 @@ instead of relabeling it as the requested routing plan. The whole-video result cache is keyed on every input that changes the output — prompt, effective model, sampling policy, frame count, focus window, `transcript`, `audioTranscript`, and the contact-sheet flag — so changing any of those -dimensions is a cache miss, never a stale reuse. +dimensions is a cache miss, never a stale reuse. The visual dedup policy +version, threshold, and bounded candidate-frame count are also explicit in the +result-cache key and metadata; a policy change therefore cannot reuse a stale +whole-video description. The guardrail extracts every supported video part but describes no more than `modalityBridgeVideoMaxVideos`. For a target proven to have diff --git a/scripts/perf/video-bridge-bench.ts b/scripts/perf/video-bridge-bench.ts index 6e9a18337b..66c5fd7391 100644 --- a/scripts/perf/video-bridge-bench.ts +++ b/scripts/perf/video-bridge-bench.ts @@ -1,24 +1,78 @@ /** - * Video Bridge benchmarks (VB-FU-07 sampler overhead + VB-FU-09 contact sheet A/B). + * Video Bridge benchmarks (VB-FU-03 dedup comparator, VB-FU-07 sampler overhead, + * and VB-FU-09 contact sheet A/B). * * Run: node --import tsx/esm scripts/perf/video-bridge-bench.ts * - * 1. Sampler: measures the pure timestamp-selection cost of uniform vs + * 1. Dedup: measures bounded CPU and process-memory observations for the + * production 16x16 grayscale comparator over the hard 16-frame candidate cap. + * 2. Sampler: measures the pure timestamp-selection cost of uniform vs * scene_aware vs segment_aware for growing scene-candidate counts. The * ffmpeg scene-detection pass is shared by both aware policies and is * I/O-bound, so the incremental policy cost is exactly this selection step. - * 2. Contact sheet: composes synthetic JPEG frames into the timestamped grid + * 3. Contact sheet: composes synthetic JPEG frames into the timestamped grid * and compares payload bytes + model calls against individual frames. */ import { performance } from "node:perf_hooks"; import { buildVideoContactSheet } from "../../src/lib/guardrails/videoBridgeContactSheet"; +import { + compareVideoFramesByGrayscale, + VIDEO_DEDUP_POLICY_VERSION, + VIDEO_DEDUP_THRESHOLD, +} from "../../src/lib/guardrails/videoBridgeHelpers"; import { calculateSamplingDecision, type VideoSamplingPolicy, } from "../../src/lib/guardrails/videoBridgeRuntime"; const SAMPLER_ITERATIONS = 2_000; +const DEDUP_FRAME_CAP = 16; +const DEDUP_ITERATIONS = 10; + +function mebibytes(bytes: number): string { + return (bytes / (1024 * 1024)).toFixed(2); +} + +async function benchDedupComparator(): Promise { + const frames = await Promise.all( + Array.from({ length: DEDUP_FRAME_CAP }, async (_unused, index) => ({ + dataUri: await syntheticJpegFrame(index, 1024, 576), + timestampSeconds: index, + })) + ); + await compareVideoFramesByGrayscale(frames[0], frames[1]); + const memoryBefore = process.memoryUsage(); + const maxRssBefore = process.resourceUsage().maxRSS * 1024; + const cpuBefore = process.cpuUsage(); + const wallBefore = performance.now(); + let comparisons = 0; + for (let iteration = 0; iteration < DEDUP_ITERATIONS; iteration++) { + for (let index = 1; index < frames.length; index++) { + await compareVideoFramesByGrayscale(frames[index - 1], frames[index]); + comparisons += 1; + } + } + const wallMs = performance.now() - wallBefore; + const cpu = process.cpuUsage(cpuBefore); + const memoryAfter = process.memoryUsage(); + const maxRssAfter = process.resourceUsage().maxRSS * 1024; + const cpuMs = (cpu.user + cpu.system) / 1000; + + console.log("== Visual dedup comparator (synthetic 1024x576 JPEG, bounded) =="); + console.log( + `policy=${VIDEO_DEDUP_POLICY_VERSION} threshold=${VIDEO_DEDUP_THRESHOLD} frames=${DEDUP_FRAME_CAP} iterations=${DEDUP_ITERATIONS} comparisons=${comparisons}` + ); + console.log( + `wall_ms=${wallMs.toFixed(1)} cpu_ms=${cpuMs.toFixed(1)} cpu_ms/comparison=${(cpuMs / comparisons).toFixed(3)}` + ); + console.log( + `rss_delta_MiB=${mebibytes(memoryAfter.rss - memoryBefore.rss)} heap_delta_MiB=${mebibytes(memoryAfter.heapUsed - memoryBefore.heapUsed)} max_rss_delta_MiB=${mebibytes(Math.max(0, maxRssAfter - maxRssBefore))}` + ); + console.log( + "Scope: comparator decode/resize/delta cost only; this does not measure caption-model quality." + ); +} function benchSampler(): void { console.log("== Sampler timestamp-selection cost (pure, per call) =="); @@ -47,12 +101,12 @@ function benchSampler(): void { } } -async function syntheticJpegFrame(index: number): Promise { +async function syntheticJpegFrame(index: number, width = 512, height = 288): Promise { const { default: sharp } = await import("sharp"); const buffer = await sharp({ create: { - width: 512, - height: 288, + width, + height, channels: 3, background: { r: (index * 37) % 255, g: (index * 91) % 255, b: (index * 53) % 255 }, }, @@ -86,5 +140,7 @@ async function benchContactSheet(): Promise { } } +await benchDedupComparator(); +console.log(""); benchSampler(); await benchContactSheet(); diff --git a/src/lib/guardrails/modalityBridge/bridgeCache.ts b/src/lib/guardrails/modalityBridge/bridgeCache.ts index a1bdd4ede2..8406037c7a 100644 --- a/src/lib/guardrails/modalityBridge/bridgeCache.ts +++ b/src/lib/guardrails/modalityBridge/bridgeCache.ts @@ -11,6 +11,9 @@ import type { VisionBridgeRuntimeSettings } from "@/shared/constants/modalityBri export interface BridgeCacheKeyOptions { kind?: string; + dedupCandidateFrameCount?: number; + dedupPolicyVersion?: string; + dedupThreshold?: number; extractorVersion?: string; policyVersion?: string; strategy?: string; @@ -38,6 +41,9 @@ export function bridgeCacheKey( kind: options.kind ?? "media-frame", model, prompt, + dedupCandidateFrameCount: options.dedupCandidateFrameCount, + dedupPolicyVersion: options.dedupPolicyVersion, + dedupThreshold: options.dedupThreshold, policyVersion: options.policyVersion, extractorVersion: options.extractorVersion, strategy: options.strategy, diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 153f04bcfe..a703fb1bb5 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -23,7 +23,11 @@ import { formatVideoTimestamp, loadVideoPartBytes, replaceVideoParts, + resolveVideoDedupCandidateFrameCount, VIDEO_BRIDGE_MAX_BYTES, + VIDEO_DEDUP_MAX_CANDIDATE_FRAMES, + VIDEO_DEDUP_POLICY_VERSION, + VIDEO_DEDUP_THRESHOLD, type DescribeVideoDependencies, type DescribedVideo, type VideoFusionTelemetry, @@ -86,9 +90,9 @@ function waitForVideoBridgePromise(promise: Promise, signal: AbortSignal): }); } -const VIDEO_BRIDGE_RESULT_CACHE_VERSION = "v3"; -const VIDEO_BRIDGE_RESULT_CACHE_POLICY = "default"; -const VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND = "video-result-v3"; +const VIDEO_BRIDGE_RESULT_CACHE_VERSION = "v4"; +const VIDEO_BRIDGE_RESULT_CACHE_POLICY = "sampling-then-dedup-v2"; +const VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND = "video-result-v4"; const VIDEO_BRIDGE_DOWNLOAD_FLIGHT_VERSION = "v1"; function buildVideoDownloadFlightKey( @@ -132,6 +136,9 @@ interface VideoResultCacheMetadata { prompt: string; frameCount: number; maxVideos: number; + dedupCandidateFrameCount: number; + dedupPolicyVersion: string; + dedupThreshold: number; durationSeconds: number; framesRequested: number; framesExtracted: number; @@ -152,6 +159,9 @@ interface VideoResultCacheMetadata { type VideoResultCacheIdentity = Pick< VideoResultCacheMetadata, | "cacheVersion" + | "dedupCandidateFrameCount" + | "dedupPolicyVersion" + | "dedupThreshold" | "extractorVersion" | "frameCount" | "maxVideos" @@ -163,6 +173,9 @@ type VideoResultCacheIdentity = Pick< const VIDEO_RESULT_CACHE_IDENTITY_KEYS: readonly (keyof VideoResultCacheIdentity)[] = [ "cacheVersion", + "dedupCandidateFrameCount", + "dedupPolicyVersion", + "dedupThreshold", "extractorVersion", "frameCount", "maxVideos", @@ -179,6 +192,9 @@ function createVideoResultCacheIdentity( ): VideoResultCacheIdentity { return { cacheVersion: VIDEO_BRIDGE_RESULT_CACHE_VERSION, + dedupCandidateFrameCount: resolveVideoDedupCandidateFrameCount(runtime.frameCount), + dedupPolicyVersion: VIDEO_DEDUP_POLICY_VERSION, + dedupThreshold: VIDEO_DEDUP_THRESHOLD, extractorVersion: VIDEO_BRIDGE_RESULT_CACHE_VERSION, frameCount: runtime.frameCount, maxVideos: runtime.maxVideos, @@ -196,6 +212,9 @@ function buildVideoResultCacheKey( ): string { return bridgeCacheKey(contentFingerprint, identity.prompt, identity.model, { kind: VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND, + dedupCandidateFrameCount: identity.dedupCandidateFrameCount, + dedupPolicyVersion: identity.dedupPolicyVersion, + dedupThreshold: identity.dedupThreshold, extractorVersion: identity.extractorVersion, policyVersion: identity.policyVersion, strategy: identity.strategy, @@ -269,7 +288,11 @@ function isVideoResultCacheMetadata( !isFiniteNonNegativeInteger(record.framesRequested) || !isFiniteNonNegativeInteger(record.framesExtracted) || !isFiniteNonNegativeInteger(record.framesUsed) || - record.framesExtracted > record.framesRequested || + !isFiniteNonNegativeInteger(record.dedupCandidateFrameCount) || + record.dedupCandidateFrameCount < 1 || + record.dedupCandidateFrameCount > VIDEO_DEDUP_MAX_CANDIDATE_FRAMES || + record.framesExtracted > record.dedupCandidateFrameCount || + record.framesUsed > record.framesRequested || record.framesUsed > record.framesExtracted ) { return false; @@ -293,6 +316,11 @@ function isVideoResultCacheMetadata( } return ( typeof record.cacheVersion === "string" && + typeof record.dedupPolicyVersion === "string" && + typeof record.dedupThreshold === "number" && + Number.isFinite(record.dedupThreshold) && + record.dedupThreshold >= 0 && + record.dedupThreshold <= 1 && typeof record.policyVersion === "string" && typeof record.extractorVersion === "string" && typeof record.strategy === "string" && diff --git a/src/lib/guardrails/videoBridgeHelpers.ts b/src/lib/guardrails/videoBridgeHelpers.ts index c868dd7296..f9e7a384ab 100644 --- a/src/lib/guardrails/videoBridgeHelpers.ts +++ b/src/lib/guardrails/videoBridgeHelpers.ts @@ -274,40 +274,86 @@ export interface VideoFrameDeduplicationResult { type VideoFrameComparator = ( previous: VideoCaptionFrame, - current: VideoCaptionFrame + current: VideoCaptionFrame, + signal?: AbortSignal ) => Promise; -const VIDEO_DEDUP_THRESHOLD = 0.04; +export const VIDEO_DEDUP_POLICY_VERSION = "grayscale-16x16-mean-cells-v2"; +export const VIDEO_DEDUP_THRESHOLD = 0.04; +const VIDEO_DEDUP_CELL_DELTA_THRESHOLD = 0.05; +export const VIDEO_DEDUP_MAX_CANDIDATE_FRAMES = 16; -async function compareVideoFramesByGrayscale( +/** + * Expand a final caption budget into the bounded pool evaluated by visual deduplication. + * + * @param frameCount - Requested number of frames that may reach captioning. + * @returns One candidate for a one-frame budget, otherwise twice the budget capped at 16. + */ +export function resolveVideoDedupCandidateFrameCount(frameCount: number): number { + const normalizedFrameCount = Number.isFinite(frameCount) ? Math.floor(frameCount) : 1; + const finalFrameCount = Math.max( + 1, + Math.min(VIDEO_DEDUP_MAX_CANDIDATE_FRAMES, normalizedFrameCount) + ); + if (finalFrameCount === 1) return 1; + return Math.min(VIDEO_DEDUP_MAX_CANDIDATE_FRAMES, finalFrameCount * 2); +} + +function throwIfVideoDedupAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw new Error("Video Bridge processing timed out or was aborted"); +} + +/** + * Compare JPEG frames using the versioned 16x16 grayscale visual policy. + * + * @param previous - Last frame retained by deduplication. + * @param current - Candidate frame being evaluated. + * @param signal - Optional request cancellation signal checked around asynchronous image work. + * @returns The larger of mean luma delta and the ratio of materially changed cells. + * @throws When cancelled or when either frame cannot be decoded as a JPEG data URI. + */ +export async function compareVideoFramesByGrayscale( previous: VideoCaptionFrame, - current: VideoCaptionFrame + current: VideoCaptionFrame, + signal?: AbortSignal ): Promise { + throwIfVideoDedupAborted(signal); const decode = (dataUri: string): Buffer => { const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri); if (!match) throw new Error("Video frame is not a JPEG data URI"); return Buffer.from(match[1], "base64"); }; const { default: sharp } = await import("sharp"); + throwIfVideoDedupAborted(signal); const [left, right] = await Promise.all( [previous, current].map((frame) => sharp(decode(frame.dataUri)).resize(16, 16, { fit: "fill" }).greyscale().raw().toBuffer() ) ); + throwIfVideoDedupAborted(signal); if (left.length !== right.length || left.length === 0) { throw new Error("Video frame comparison returned invalid dimensions"); } let difference = 0; + let changedCells = 0; for (let index = 0; index < left.length; index++) { - difference += Math.abs(left[index] - right[index]) / 255; + const cellDifference = Math.abs(left[index] - right[index]) / 255; + difference += cellDifference; + if (cellDifference >= VIDEO_DEDUP_CELL_DELTA_THRESHOLD) changedCells += 1; } - return difference / left.length; + return Math.max(difference / left.length, changedCells / left.length); } export async function deduplicateVideoFrames( frames: readonly VideoCaptionFrame[], - options: { compare?: VideoFrameComparator; threshold?: number } = {} + options: { + compare?: VideoFrameComparator; + maxFrames?: number; + signal?: AbortSignal; + threshold?: number; + } = {} ): Promise { + throwIfVideoDedupAborted(options.signal); if (frames.length < 2) return { dropped: 0, frames: [...frames] }; const compare = options.compare ?? compareVideoFramesByGrayscale; const threshold = @@ -317,23 +363,37 @@ export async function deduplicateVideoFrames( const kept: VideoCaptionFrame[] = [frames[0]]; let dropped = 0; for (let index = 1; index < frames.length; index++) { + throwIfVideoDedupAborted(options.signal); const current = frames[index]; if (index === frames.length - 1) { kept.push(current); continue; } try { - const distance = await compare(kept[kept.length - 1], current); + const distance = await compare(kept[kept.length - 1], current, options.signal); + throwIfVideoDedupAborted(options.signal); if (Number.isFinite(distance) && distance <= threshold) { dropped += 1; continue; } } catch { + throwIfVideoDedupAborted(options.signal); // A malformed or unsupported frame must never reduce visual coverage. } kept.push(current); } - return { dropped, frames: kept }; + throwIfVideoDedupAborted(options.signal); + const maxFrames = + typeof options.maxFrames === "number" && Number.isFinite(options.maxFrames) + ? Math.max(1, Math.floor(options.maxFrames)) + : kept.length; + if (kept.length <= maxFrames) return { dropped, frames: kept }; + if (maxFrames === 1) return { dropped, frames: [kept[0]] }; + const capped = Array.from({ length: maxFrames }, (_unused, index) => { + const sourceIndex = Math.round((index * (kept.length - 1)) / (maxFrames - 1)); + return kept[sourceIndex]; + }); + return { dropped, frames: capped }; } function normalizeBase64(base64: string): string { @@ -456,15 +516,19 @@ export async function describeVideoPart( if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted"); if (bytes.byteLength > maxBytes) throw new Error("Video exceeds the maximum size"); const extractFrames = deps.extractFrames ?? extractVideoFramesViaBroker; + const candidateFrameCount = resolveVideoDedupCandidateFrameCount(options.frameCount); const extracted = await extractFrames(bytes, { focusWindow: options.focusWindow, - frameCount: options.frameCount, + frameCount: candidateFrameCount, samplingPolicy: options.samplingPolicy, signal, timeoutMs: options.timeoutMs, }); - const deduplicated = await deduplicateVideoFrames(extracted.frames); + const deduplicated = await deduplicateVideoFrames(extracted.frames, { + maxFrames: options.frameCount, + signal, + }); const contactSheet = part.contactSheet ? await buildVideoContactSheet(deduplicated.frames, { signal, diff --git a/tests/fixtures/videoBridgeDedupFixtures.ts b/tests/fixtures/videoBridgeDedupFixtures.ts new file mode 100644 index 0000000000..862a3b918c --- /dev/null +++ b/tests/fixtures/videoBridgeDedupFixtures.ts @@ -0,0 +1,53 @@ +import sharp from "sharp"; + +type Rectangle = { + height: number; + value: number; + width: number; + x: number; + y: number; +}; + +const FIXTURE_WIDTH = 256; +const FIXTURE_HEIGHT = 144; + +async function renderJpeg(rectangles: readonly Rectangle[]): Promise { + const pixels = Buffer.alloc(FIXTURE_WIDTH * FIXTURE_HEIGHT * 3, 255); + for (const rectangle of rectangles) { + for (let y = rectangle.y; y < rectangle.y + rectangle.height; y++) { + for (let x = rectangle.x; x < rectangle.x + rectangle.width; x++) { + const offset = (y * FIXTURE_WIDTH + x) * 3; + pixels[offset] = rectangle.value; + pixels[offset + 1] = rectangle.value; + pixels[offset + 2] = rectangle.value; + } + } + } + const jpeg = await sharp(pixels, { + raw: { channels: 3, height: FIXTURE_HEIGHT, width: FIXTURE_WIDTH }, + }) + .jpeg({ chromaSubsampling: "4:4:4", quality: 100 }) + .toBuffer(); + return `data:image/jpeg;base64,${jpeg.toString("base64")}`; +} + +export async function createVideoDedupFixtures(): Promise<{ + smallMotion: readonly [string, string]; + staticFrame: string; + visibleText: readonly [string, string]; +}> { + const staticFrame = await renderJpeg([{ height: 48, value: 0, width: 48, x: 64, y: 48 }]); + const movedFrame = await renderJpeg([{ height: 48, value: 0, width: 48, x: 68, y: 48 }]); + // Rectangular strokes stand in for glyphs without depending on platform fonts. + const textBefore = [ + { height: 64, value: 0, width: 8, x: 32, y: 32 }, + { height: 64, value: 0, width: 8, x: 48, y: 32 }, + { height: 64, value: 0, width: 8, x: 64, y: 32 }, + ] as const; + const textAfter = [...textBefore, { height: 64, value: 0, width: 8, x: 80, y: 32 }] as const; + return { + smallMotion: [staticFrame, movedFrame], + staticFrame, + visibleText: [await renderJpeg(textBefore), await renderJpeg(textAfter)], + }; +} diff --git a/tests/unit/guardrails/videoBridgeDedup.test.ts b/tests/unit/guardrails/videoBridgeDedup.test.ts index bbadb0bc3e..500f5b10b7 100644 --- a/tests/unit/guardrails/videoBridgeDedup.test.ts +++ b/tests/unit/guardrails/videoBridgeDedup.test.ts @@ -3,8 +3,40 @@ import test from "node:test"; import { deduplicateVideoFrames, + resolveVideoDedupCandidateFrameCount, type VideoCaptionFrame, } from "../../../src/lib/guardrails/videoBridgeHelpers.ts"; +import { createVideoDedupFixtures } from "../../fixtures/videoBridgeDedupFixtures.ts"; + +const fixturesPromise = createVideoDedupFixtures(); + +test("dedup candidate count doubles the caption budget within the hard frame bound", () => { + assert.equal(resolveVideoDedupCandidateFrameCount(1), 1); + assert.equal(resolveVideoDedupCandidateFrameCount(3), 6); + assert.equal(resolveVideoDedupCandidateFrameCount(8), 16); + assert.equal(resolveVideoDedupCandidateFrameCount(9), 16); + assert.equal(resolveVideoDedupCandidateFrameCount(Number.NaN), 1); +}); + +test("deduplication stops scheduling comparator work after abort", async () => { + const controller = new AbortController(); + let comparisons = 0; + const pending = deduplicateVideoFrames( + [frame(1), frame(2), frame(3), frame(4), frame(5), frame(6)], + { + compare: async () => { + comparisons += 1; + await new Promise((resolve) => setTimeout(resolve, 30)); + return 0.2; + }, + signal: controller.signal, + } + ); + setTimeout(() => controller.abort(), 5); + + await assert.rejects(pending, /aborted/i); + assert.equal(comparisons, 1); +}); const frame = ( timestampSeconds: number, @@ -37,13 +69,80 @@ test("deduplication keeps visually distinct frames", async () => { assert.equal(result.dropped, 0); }); -test("deduplication fails open when the visual comparator errors", async () => { - const result = await deduplicateVideoFrames([frame(1), frame(2)], { - compare: async () => { - throw new Error("invalid JPEG"); - }, - }); +test("deduplication applies the final cap after comparison while preserving both endpoints", async () => { + const result = await deduplicateVideoFrames( + [frame(1), frame(2), frame(3), frame(4), frame(5), frame(6)], + { + compare: async (_previous, current) => + current.timestampSeconds === 2 || current.timestampSeconds === 4 ? 0.01 : 0.2, + maxFrames: 3, + threshold: 0.05, + } + ); - assert.equal(result.frames.length, 2); + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 5, 6] + ); + assert.equal(result.dropped, 2, "only visual duplicates count as dedup drops"); +}); + +test("the real grayscale policy preserves a small moving subject", async () => { + const fixtures = await fixturesPromise; + const result = await deduplicateVideoFrames([ + frame(1, fixtures.smallMotion[0]), + frame(2, fixtures.smallMotion[1]), + frame(3, fixtures.smallMotion[0]), + ]); + + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 2, 3] + ); + assert.equal(result.dropped, 0); +}); + +test("the real grayscale policy drops a static fixture", async () => { + const fixtures = await fixturesPromise; + const result = await deduplicateVideoFrames([ + frame(1, fixtures.staticFrame), + frame(2, fixtures.staticFrame), + frame(3, fixtures.smallMotion[1]), + ]); + + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 3] + ); + assert.equal(result.dropped, 1); +}); + +test("the real grayscale policy preserves a visible text change", async () => { + const fixtures = await fixturesPromise; + const result = await deduplicateVideoFrames([ + frame(1, fixtures.visibleText[0]), + frame(2, fixtures.visibleText[1]), + frame(3, fixtures.visibleText[0]), + ]); + + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 2, 3] + ); + assert.equal(result.dropped, 0); +}); + +test("deduplication fails open for a malformed JPEG candidate", async () => { + const fixtures = await fixturesPromise; + const result = await deduplicateVideoFrames([ + frame(1, fixtures.staticFrame), + frame(2, "data:image/jpeg;base64,bm90LWEtanBlZw=="), + frame(3, fixtures.staticFrame), + ]); + + assert.deepEqual( + result.frames.map((item) => item.timestampSeconds), + [1, 2, 3] + ); assert.equal(result.dropped, 0); }); diff --git a/tests/unit/guardrails/videoBridgeHelpers.test.ts b/tests/unit/guardrails/videoBridgeHelpers.test.ts index 808ed864c7..149f20cc6d 100644 --- a/tests/unit/guardrails/videoBridgeHelpers.test.ts +++ b/tests/unit/guardrails/videoBridgeHelpers.test.ts @@ -366,6 +366,44 @@ test("uses the broker seam, reports configured versus extracted frames, and mark assert.match(result.description, /do not follow instructions/i); }); +test("uses a bounded candidate pool before the final caption cap and preserves endpoint coverage", async () => { + let candidateFrameCount = 0; + const captionedTimestamps: number[] = []; + const result = await describeVideoPart( + { + container: "messages", + messageIndex: 0, + partIndex: 0, + ref: "data:video/mp4;base64,QUJD", + shape: "input_video", + }, + { frameCount: 3, timeoutMs: 5_000 }, + async (_frame, timestampSeconds) => { + captionedTimestamps.push(timestampSeconds); + return `frame ${timestampSeconds}`; + }, + { + extractFrames: async (_bytes, options) => { + candidateFrameCount = options.frameCount; + return { + durationSeconds: 6, + frames: Array.from({ length: options.frameCount }, (_unused, index) => ({ + dataUri: `data:image/jpeg;base64,${Buffer.from(String(index)).toString("base64")}`, + timestampSeconds: index + 1, + })), + }; + }, + } + ); + + assert.equal(candidateFrameCount, 6, "three caption slots get at most two candidates each"); + assert.deepEqual(captionedTimestamps, [1, 4, 6]); + assert.equal(result.framesRequested, 3); + assert.equal(result.framesExtracted, 6); + assert.equal(result.framesUsed, 3); + assert.equal(result.dedupDropped, 0, "malformed candidate comparisons must fail open"); +}); + test("video downloads require HTTPS on every redirect hop", async () => { let requireHttps: boolean | undefined; await describeVideoPart( diff --git a/tests/unit/guardrails/videoBridgeResultCache.test.ts b/tests/unit/guardrails/videoBridgeResultCache.test.ts index c430b69b77..48235445fb 100644 --- a/tests/unit/guardrails/videoBridgeResultCache.test.ts +++ b/tests/unit/guardrails/videoBridgeResultCache.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts"; -import { BridgeCache } from "../../../src/lib/guardrails/modalityBridge/bridgeCache.ts"; +import { + BridgeCache, + type BridgeCacheEntry, +} from "../../../src/lib/guardrails/modalityBridge/bridgeCache.ts"; import { getBridgeStats } from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts"; import { getSharedVideoResultCacheFor, @@ -382,6 +385,58 @@ test("an unavailable result cache fails open to normal video processing", async ]); }); +test("result-cache metadata carries the exact visual dedup policy identity", async () => { + let storedMetadata: Record | undefined; + const bridge = new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeCacheEnabled: true, + modalityBridgeVideoEnabled: true, + modalityBridgeVideoFrameCount: 8, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + modalityBridgeVisionPrompt: "FU-03 policy identity", + }), + getCapabilities: () => ({ supportsVideo: false }), + selectVisionModel: async () => "openai/gpt-4o-mini", + resultCache: { + delete: () => undefined, + getEntry: () => undefined, + setEntry: (_key: string, entry: BridgeCacheEntry) => { + storedMetadata = entry.metadata; + }, + }, + describePart: async () => ({ + dedupDropped: 2, + description: "[Video description: policy-bound result]", + durationSeconds: 3, + framesExtracted: 16, + framesRequested: 8, + framesUsed: 8, + }), + }, + }); + + await bridge.preCall( + { + model: "example/text-only", + messages: [ + { + role: "user", + content: [{ type: "input_video", video_url: "data:video/mp4;base64,RlUtMDM=" }], + }, + ], + }, + {} + ); + + assert.ok(storedMetadata); + assert.equal(storedMetadata.cacheVersion, "v4"); + assert.equal(storedMetadata.policyVersion, "sampling-then-dedup-v2"); + assert.equal(storedMetadata.dedupPolicyVersion, "grayscale-16x16-mean-cells-v2"); + assert.equal(storedMetadata.dedupThreshold, 0.04); + assert.equal(storedMetadata.dedupCandidateFrameCount, 16); +}); + test("a corrupt result-cache payload is discarded and recomputed", async () => { let describeCalls = 0; const corruptCache = { @@ -390,9 +445,12 @@ test("a corrupt result-cache payload is discarded and recomputed", async () => { value: 42 as unknown as string, producerModel: "openai/gpt-4o-mini", metadata: { - cacheVersion: "v3", - policyVersion: "default", - extractorVersion: "v3", + cacheVersion: "v4", + dedupCandidateFrameCount: 16, + dedupPolicyVersion: "grayscale-16x16-mean-cells-v2", + dedupThreshold: 0.04, + policyVersion: "sampling-then-dedup-v2", + extractorVersion: "v4", strategy: "uniform", model: "openai/gpt-4o-mini", prompt: "FU-01 corrupt cache", @@ -449,9 +507,12 @@ test("a corrupt result-cache payload is discarded and recomputed", async () => { test("invalid numeric result-cache metadata is deleted and recomputed", async (t) => { const cachedValue = "[Video description: cached numeric metadata]"; const validMetadata = (): Record => ({ - cacheVersion: "v3", - policyVersion: "default", - extractorVersion: "v3", + cacheVersion: "v4", + dedupCandidateFrameCount: 16, + dedupPolicyVersion: "grayscale-16x16-mean-cells-v2", + dedupThreshold: 0.04, + policyVersion: "sampling-then-dedup-v2", + extractorVersion: "v4", strategy: "uniform", model: "openai/gpt-4o-mini", prompt: "FU-01 numeric cache validation", @@ -482,9 +543,10 @@ test("invalid numeric result-cache metadata is deleted and recomputed", async (t }, { name: "negative frame count", mutate: (metadata) => (metadata.framesUsed = -1) }, { - name: "more extracted than requested", - mutate: (metadata) => (metadata.framesExtracted = 9), + name: "more extracted than the dedup candidate budget", + mutate: (metadata) => (metadata.framesExtracted = 17), }, + { name: "more used than requested", mutate: (metadata) => (metadata.framesUsed = 9) }, { name: "more used than extracted", mutate: (metadata) => (metadata.framesUsed = 7) }, { name: "dedup and used exceed extracted", diff --git a/tests/unit/modality-bridge-cache.test.ts b/tests/unit/modality-bridge-cache.test.ts index 0ebb81b9e0..b999a40b55 100644 --- a/tests/unit/modality-bridge-cache.test.ts +++ b/tests/unit/modality-bridge-cache.test.ts @@ -23,6 +23,37 @@ test("key framing prevents boundary-shift collisions between fields", () => { assert.notEqual(bridgeCacheKey("x", "yz", "m"), bridgeCacheKey("x", "y", "zm")); }); +test("video cache keys change with every visual dedup policy dimension", () => { + const base = { + dedupCandidateFrameCount: 16, + dedupPolicyVersion: "grayscale-16x16-mean-cells-v2", + dedupThreshold: 0.04, + }; + const key = bridgeCacheKey("video", "describe", "gpt-4o-mini", base); + + assert.notEqual( + key, + bridgeCacheKey("video", "describe", "gpt-4o-mini", { + ...base, + dedupPolicyVersion: "grayscale-16x16-mean-cells-v3", + }) + ); + assert.notEqual( + key, + bridgeCacheKey("video", "describe", "gpt-4o-mini", { + ...base, + dedupThreshold: 0.05, + }) + ); + assert.notEqual( + key, + bridgeCacheKey("video", "describe", "gpt-4o-mini", { + ...base, + dedupCandidateFrameCount: 8, + }) + ); +}); + test("get/set roundtrip and TTL expiry", () => { let now = 1000; const cache = new BridgeCache({ maxEntries: 10, ttlMs: 500, now: () => now });