From 79fa6befa0df92204208cf3a0126a989d23e705e Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Mon, 24 Aug 2026 02:19:22 -0300 Subject: [PATCH] fix(video): make one-frame scene sampling deterministic --- ...ending-video-bridge-scene-aware-sampler.md | 1 + docs/security/GUARDRAILS.md | 5 +- src/lib/guardrails/videoBridgeRuntime.ts | 9 + .../video-bridge-sampler-ffmpeg.test.ts | 223 ++++++++++++++++++ .../guardrails/videoBridgeSampler.test.ts | 20 ++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixes/pending-video-bridge-scene-aware-sampler.md create mode 100644 tests/integration/video-bridge-sampler-ffmpeg.test.ts diff --git a/changelog.d/fixes/pending-video-bridge-scene-aware-sampler.md b/changelog.d/fixes/pending-video-bridge-scene-aware-sampler.md new file mode 100644 index 0000000000..1338b6a1cb --- /dev/null +++ b/changelog.d/fixes/pending-video-bridge-scene-aware-sampler.md @@ -0,0 +1 @@ +- **fix(video-bridge):** fall back to the deterministic active-window midpoint when a one-frame scene-aware budget cannot preserve both timeline ends; a real FFmpeg fixture matrix now covers rapid cuts, gradual changes, static and short clips, and detector failure. diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index f20cb80527..88ecc014f5 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -328,7 +328,10 @@ fixed FFmpeg pass over the already validated local stream, select bounded uniform midpoints on detector failure, timeout, malformed output, or an empty candidate set. Segment-aware mode allocates midpoint samples proportionally to the validated scene intervals. The hard 16-frame cap is -applied after selection in every policy. A caller may optionally provide a +applied after selection in every policy. When a scene-aware request has only a +one-frame budget, it uses the uniform midpoint of the active full-video or focus +window and reports `policyEffective: uniform`: a single selected scene frame +cannot preserve both temporal ends. A caller may optionally provide a finite focus window (`start`/`end` seconds); bounds are clamped to the media duration, reversed or non-finite windows are rejected, and all sampling policies are performed only inside the normalized interval. The resulting diff --git a/src/lib/guardrails/videoBridgeRuntime.ts b/src/lib/guardrails/videoBridgeRuntime.ts index fea9769381..2b57bffc8b 100644 --- a/src/lib/guardrails/videoBridgeRuntime.ts +++ b/src/lib/guardrails/videoBridgeRuntime.ts @@ -339,6 +339,15 @@ export function calculateSamplingDecision( } const frameCount = uniform.length; + if (frameCount === 1) { + return { + candidateCount: candidates.length, + ...(focusWindow ? { focusWindow } : {}), + policyEffective: "uniform", + policyRequested: "scene_aware", + timestamps: uniform, + }; + } const selected = candidates.length <= frameCount ? [...candidates] diff --git a/tests/integration/video-bridge-sampler-ffmpeg.test.ts b/tests/integration/video-bridge-sampler-ffmpeg.test.ts new file mode 100644 index 0000000000..c33bb770d9 --- /dev/null +++ b/tests/integration/video-bridge-sampler-ffmpeg.test.ts @@ -0,0 +1,223 @@ +/** + * Real FFmpeg fixture gate for the scene-aware Video Bridge sampler. + * + * Run explicitly because FFmpeg is an optional operational dependency: + * RUN_VIDEO_BRIDGE_FFMPEG=1 node --import tsx/esm --test \ + * tests/integration/video-bridge-sampler-ffmpeg.test.ts + */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { + extractVideoFramesFromBytes, + type VideoCommandRunner, +} from "../../src/lib/guardrails/videoBridgeRuntime.ts"; + +const execFileAsync = promisify(execFile); +const REAL_FFMPEG_ENABLED = process.env.RUN_VIDEO_BRIDGE_FFMPEG === "1"; +const REAL_FFMPEG_SKIP = REAL_FFMPEG_ENABLED + ? false + : "Set RUN_VIDEO_BRIDGE_FFMPEG=1 to run the real FFmpeg fixture matrix"; + +const realRunner: VideoCommandRunner = async (executable, args, options) => { + const result = await execFileAsync(executable, [...args], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + windowsHide: true, + }); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; +}; + +async function createFixture( + directory: string, + name: string, + inputArgs: readonly string[], + videoFilter: string +): Promise { + const outputPath = join(directory, `${name}.mkv`); + await realRunner( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + ...inputArgs, + "-vf", + videoFilter, + "-c:v", + "ffv1", + "-y", + outputPath, + ], + { timeoutMs: 30_000 } + ); + return readFile(outputPath); +} + +async function createRapidEdgeCutFixture(directory: string): Promise { + const outputPath = join(directory, "rapid-edge-cuts.mkv"); + await realRunner( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x64:r=10:d=0.2", + "-f", + "lavfi", + "-i", + "color=c=black:s=64x64:r=10:d=2.6", + "-f", + "lavfi", + "-i", + "color=c=white:s=64x64:r=10:d=0.2", + "-filter_complex", + "[0:v][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "ffv1", + "-y", + outputPath, + ], + { timeoutMs: 30_000 } + ); + return readFile(outputPath); +} + +async function sample(bytes: Buffer, frameCount: number, runner = realRunner) { + return extractVideoFramesFromBytes(bytes, { + frameCount, + maxDurationSeconds: 600, + runner, + samplingPolicy: "scene_aware", + timeoutMs: 30_000, + }); +} + +test( + "scene-aware sampling handles the canonical real FFmpeg fixture matrix", + { skip: REAL_FFMPEG_SKIP }, + async (context) => { + const directory = await mkdtemp(join(tmpdir(), "omniroute-video-sampler-fixtures-")); + context.after(async () => rm(directory, { force: true, recursive: true })); + + const rapidCuts = await createRapidEdgeCutFixture(directory); + await context.test("rapid cuts near both ends retain coverage within the cap", async () => { + const result = await sample(rapidCuts, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.2, 0.5, 2.8] + ); + assert.deepEqual(result.sampling, { + candidateCount: 2, + policyEffective: "scene_aware", + policyRequested: "scene_aware", + }); + assert.ok(result.frames.length <= 16); + }); + + await context.test("one frame falls back to the full-window midpoint", async () => { + const result = await sample(rapidCuts, 1); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [1.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 2, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + }); + + const staticVideo = await createFixture( + directory, + "static", + ["-f", "lavfi", "-i", "color=c=blue:s=64x64:r=10:d=4"], + "format=yuv420p" + ); + await context.test("a static scene falls back to uniform midpoints", async () => { + const result = await sample(staticVideo, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 0, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + }); + + const slowChange = await createFixture( + directory, + "slow-change", + ["-f", "lavfi", "-i", "nullsrc=s=64x64:r=10:d=4"], + "geq=lum='clip(16+200*T/4,16,235)':cb=128:cr=128,format=yuv420p" + ); + await context.test("a gradual luminance change does not become a false scene cut", async () => { + const result = await sample(slowChange, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.equal(result.sampling.candidateCount, 0); + assert.equal(result.sampling.policyEffective, "uniform"); + }); + + const shortVideo = await createFixture( + directory, + "short", + ["-f", "lavfi", "-i", "color=c=yellow:s=64x64:r=10:d=0.4"], + "format=yuv420p" + ); + await context.test("a sub-second clip remains deterministic and bounded", async () => { + const result = await sample(shortVideo, 8); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.2] + ); + assert.equal(result.sampling.policyEffective, "uniform"); + }); + + await context.test( + "a detector failure falls back while real frame extraction continues", + async () => { + const detectorFailureRunner: VideoCommandRunner = async (executable, args, options) => { + if (args.some((arg) => arg.includes("showinfo"))) { + throw new Error("fixture scene detector failure"); + } + return realRunner(executable, args, options); + }; + const result = await sample(staticVideo, 4, detectorFailureRunner); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 0, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + } + ); + } +); diff --git a/tests/unit/guardrails/videoBridgeSampler.test.ts b/tests/unit/guardrails/videoBridgeSampler.test.ts index b8d391a1dc..ec7304655a 100644 --- a/tests/unit/guardrails/videoBridgeSampler.test.ts +++ b/tests/unit/guardrails/videoBridgeSampler.test.ts @@ -29,6 +29,26 @@ test("scene-aware sampling falls back to deterministic uniform midpoints for a s assert.equal(decision.candidateCount, 0); }); +test("scene-aware sampling falls back to the midpoint when one frame cannot cover both ends", () => { + const decision = calculateSamplingDecision(8, 1, "scene_aware", [0.25, 7.75]); + + assert.deepEqual(decision.timestamps, [4]); + assert.equal(decision.policyRequested, "scene_aware"); + assert.equal(decision.policyEffective, "uniform"); + assert.equal(decision.candidateCount, 2); +}); + +test("one-frame scene-aware fallback uses the active focus-window midpoint", () => { + const decision = calculateSamplingDecision(10, 1, "scene_aware", [2.25, 7.75], { + endSeconds: 8, + startSeconds: 2, + }); + + assert.deepEqual(decision.timestamps, [5]); + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.focusWindow, { endSeconds: 8, startSeconds: 2 }); +}); + test("scene candidates are parsed from showinfo output and malformed values are ignored", () => { const output = [ "[Parsed_showinfo_0 @ 0x1] n:1 pts_time:1.250",