fix(video): harden visual frame deduplication (#11382)

Merged via consolidated batch validation (worktree `.claude/worktrees/batch-0824d`), stacked on the just-merged #11362 as documented. Moves the Video Bridge frame cap to post-dedup, bounds the perceptual candidate pool to at most 2x budget (max 16), includes the dedup policy/version in result-cache identity, adds cooperative abort checks to the comparator loop. Static gates green; own dedup/cache-version regression suite passed in the combined-batch run (grayscale-16x16-mean-cells-v2 policy, real fixtures). Thanks!
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-08-24 09:26:33 -03:00
committed by Markus Hartung
parent 761d38f433
commit 7715825cb8
11 changed files with 495 additions and 45 deletions

View File

@@ -1,24 +1,78 @@
/**
* Video Bridge benchmarks (VB-FU-07 sampler overhead + VB-FU-09 contact sheet A/B).
* Video Bridge benchmarks (VB-FU-03 dedup comparator, VB-FU-07 sampler overhead,
* and VB-FU-09 contact sheet A/B).
*
* Run: node --import tsx/esm scripts/perf/video-bridge-bench.ts
*
* 1. Sampler: measures the pure timestamp-selection cost of uniform vs
* 1. Dedup: measures bounded CPU and process-memory observations for the
* production 16x16 grayscale comparator over the hard 16-frame candidate cap.
* 2. Sampler: measures the pure timestamp-selection cost of uniform vs
* scene_aware vs segment_aware for growing scene-candidate counts. The
* ffmpeg scene-detection pass is shared by both aware policies and is
* I/O-bound, so the incremental policy cost is exactly this selection step.
* 2. Contact sheet: composes synthetic JPEG frames into the timestamped grid
* 3. Contact sheet: composes synthetic JPEG frames into the timestamped grid
* and compares payload bytes + model calls against individual frames.
*/
import { performance } from "node:perf_hooks";
import { buildVideoContactSheet } from "../../src/lib/guardrails/videoBridgeContactSheet";
import {
compareVideoFramesByGrayscale,
VIDEO_DEDUP_POLICY_VERSION,
VIDEO_DEDUP_THRESHOLD,
} from "../../src/lib/guardrails/videoBridgeHelpers";
import {
calculateSamplingDecision,
type VideoSamplingPolicy,
} from "../../src/lib/guardrails/videoBridgeRuntime";
const SAMPLER_ITERATIONS = 2_000;
const DEDUP_FRAME_CAP = 16;
const DEDUP_ITERATIONS = 10;
function mebibytes(bytes: number): string {
return (bytes / (1024 * 1024)).toFixed(2);
}
async function benchDedupComparator(): Promise<void> {
const frames = await Promise.all(
Array.from({ length: DEDUP_FRAME_CAP }, async (_unused, index) => ({
dataUri: await syntheticJpegFrame(index, 1024, 576),
timestampSeconds: index,
}))
);
await compareVideoFramesByGrayscale(frames[0], frames[1]);
const memoryBefore = process.memoryUsage();
const maxRssBefore = process.resourceUsage().maxRSS * 1024;
const cpuBefore = process.cpuUsage();
const wallBefore = performance.now();
let comparisons = 0;
for (let iteration = 0; iteration < DEDUP_ITERATIONS; iteration++) {
for (let index = 1; index < frames.length; index++) {
await compareVideoFramesByGrayscale(frames[index - 1], frames[index]);
comparisons += 1;
}
}
const wallMs = performance.now() - wallBefore;
const cpu = process.cpuUsage(cpuBefore);
const memoryAfter = process.memoryUsage();
const maxRssAfter = process.resourceUsage().maxRSS * 1024;
const cpuMs = (cpu.user + cpu.system) / 1000;
console.log("== Visual dedup comparator (synthetic 1024x576 JPEG, bounded) ==");
console.log(
`policy=${VIDEO_DEDUP_POLICY_VERSION} threshold=${VIDEO_DEDUP_THRESHOLD} frames=${DEDUP_FRAME_CAP} iterations=${DEDUP_ITERATIONS} comparisons=${comparisons}`
);
console.log(
`wall_ms=${wallMs.toFixed(1)} cpu_ms=${cpuMs.toFixed(1)} cpu_ms/comparison=${(cpuMs / comparisons).toFixed(3)}`
);
console.log(
`rss_delta_MiB=${mebibytes(memoryAfter.rss - memoryBefore.rss)} heap_delta_MiB=${mebibytes(memoryAfter.heapUsed - memoryBefore.heapUsed)} max_rss_delta_MiB=${mebibytes(Math.max(0, maxRssAfter - maxRssBefore))}`
);
console.log(
"Scope: comparator decode/resize/delta cost only; this does not measure caption-model quality."
);
}
function benchSampler(): void {
console.log("== Sampler timestamp-selection cost (pure, per call) ==");
@@ -47,12 +101,12 @@ function benchSampler(): void {
}
}
async function syntheticJpegFrame(index: number): Promise<string> {
async function syntheticJpegFrame(index: number, width = 512, height = 288): Promise<string> {
const { default: sharp } = await import("sharp");
const buffer = await sharp({
create: {
width: 512,
height: 288,
width,
height,
channels: 3,
background: { r: (index * 37) % 255, g: (index * 91) % 255, b: (index * 53) % 255 },
},
@@ -86,5 +140,7 @@ async function benchContactSheet(): Promise<void> {
}
}
await benchDedupComparator();
console.log("");
benchSampler();
await benchContactSheet();