diff --git a/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md b/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md new file mode 100644 index 0000000000..17a48aa416 --- /dev/null +++ b/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md @@ -0,0 +1 @@ +- **fix(video-bridge):** make opt-in segment-aware sampling use one bounded structural FFmpeg pass (scene, freeze, blur, exposure, and SI/TI), preserve long trailing segments, fail open to uniform sampling, and add real-media structural-oracle, overhead, post-dedup caption-call, and false-positive evidence while holding unconfigured model quality and gain-versus-cost claims ([#11381](https://github.com/diegosouzapw/OmniRoute/pull/11381)). diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index 96837a7924..d849ad338a 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -327,7 +327,8 @@ fixed FFmpeg pass over the already validated local stream, select bounded `showinfo` scene timestamps, and fall back deterministically to the same uniform midpoints on detector failure, timeout, malformed output, or an empty candidate set. Segment-aware mode allocates midpoint samples proportionally to -the validated scene intervals. The hard 16-frame cap is +the validated scene intervals; segment-aware evidence and fallback behavior are +detailed below. The hard 16-frame cap is applied after selection in every policy. When a scene-aware request has only a one-frame budget, it uses the uniform midpoint of the active full-video or focus window and reports `policyEffective: uniform`: a single selected scene frame @@ -338,6 +339,44 @@ policies are performed only inside the normalized interval. The resulting window is included in sampling metadata and in the untrusted description prefix so downstream models can distinguish a focused excerpt from the full timeline. + +#### FU-07 structural segment evidence + +`segment_aware` uses one bounded pre-analysis pass over the already validated +local video stream. The fixed filter chain first scales to at most 320 pixels +wide, detects scene changes and frozen intervals, then samples at 1 frame per +second for blur, average luma, and spatial/temporal information. The pass is +limited to 600 structural samples, one FFmpeg/filter thread, the same +`file`-only protocol and container allowlists, a 1 MiB process-output bound, +and at most 30 seconds inside the broker's shared abort/deadline. It never +accepts a command, filter, path, or URL from the request. + +The structural values are deterministic sampling evidence, not semantic video +understanding. They do not infer subjects, actions, captions, speech, or user +intent. Scene and freeze boundaries form segments; freeze coverage, blur, +exposure, spatial detail, and temporal change only influence how the existing +1–16 frame budget is allocated. A fully frozen segment is capped at one frame, +while non-frozen segments compete for the remaining budget. When boundaries +outnumber frames, uniform timeline coverage is retained so rapid early cuts +cannot hide a long trailing segment. Scene boundaries within the 1-second +analysis resolution of a freeze boundary are coalesced. + +Missing filters, malformed/empty evidence, a detector error, or the bounded +pre-analysis timeout fail open to the exact uniform midpoint policy. A caller +abort or broker deadline does not fail open: it terminates the in-flight +subprocess, prevents later frame extraction, and the private temporary tree is +removed in `finally`. + +`scripts/perf/video-bridge-fu07-eval.ts` generates deterministic real FFmpeg +fixtures for post-dedup caption-call savings, dense-motion budget allocation, +blur/exposure/SI-TI evidence, rapid cuts with a long tail, and gradual-fade +false positives. It records pre-analysis wall time and, where `/usr/bin/time` +is available, child CPU and peak RSS. Its quality checks are structural oracles +only. Real caption-model quality remains `HOLD` because this harness has no +authorized endpoint or frozen judge. Monetary savings also remain `HOLD` +unless `--caption-cost-per-call-usd` supplies an explicit positive per-call +estimate; the script never fabricates either result. + Each frame is limited to 4 MiB, all raw frames together to 23 MiB, and the serialized broker response to 32 MiB. A private temporary directory is removed in `finally`. OmniRoute does not bundle FFmpeg and does not accept a custom diff --git a/scripts/perf/video-bridge-fu07-eval.ts b/scripts/perf/video-bridge-fu07-eval.ts new file mode 100644 index 0000000000..33cd67665a --- /dev/null +++ b/scripts/perf/video-bridge-fu07-eval.ts @@ -0,0 +1,493 @@ +/** + * Real-media FU-07 structural-sampling evaluation. + * + * Run: node --import tsx/esm scripts/perf/video-bridge-fu07-eval.ts + * Optional estimate: append --caption-cost-per-call-usd . + * + * This evaluates deterministic structural oracles, not semantic model quality. + * Model quality and monetary savings remain HOLD without an external receipt. + */ +import { execFile } from "node:child_process"; +import { access, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { promisify } from "node:util"; + +import { deduplicateVideoFrames } from "../../src/lib/guardrails/videoBridgeHelpers"; +import { + analyzeVideoStructure, + calculateSamplingDecision, + extractFramesFromLocalVideo, + readBoundedExtractedFrames, + type VideoCommandRunner, + type VideoStructuralAnalysis, + type VideoStructuralSample, +} from "../../src/lib/guardrails/videoBridgeRuntime"; + +const execFileAsync = promisify(execFile); +const REQUIRED_FILTERS = ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"]; +const TIME_MARKER = "__FU07_TIME__"; + +interface ChildCost { + maxRssKiB: number | null; + systemSeconds: number | null; + userSeconds: number | null; + wallMs: number; +} + +interface FixtureResult { + captionCallsAvoided: number; + childCost: ChildCost; + freezeIntervals: number; + name: string; + oracle: Record; + passed: boolean; + sceneCandidates: number; + structuralFrames: number; + uniformFrames: number; +} + +function average(values: Array): number | null { + const finite = values.filter( + (value): value is number => value !== null && value !== undefined && Number.isFinite(value) + ); + return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null; +} + +function samplesIn( + analysis: VideoStructuralAnalysis, + startSeconds: number, + endSeconds: number +): VideoStructuralSample[] { + return analysis.samples.filter( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); +} + +async function generateFixture(outputPath: string, args: readonly string[]): Promise { + await execFileAsync( + "ffmpeg", + ["-hide_banner", "-loglevel", "error", ...args, "-threads", "1", "-y", outputPath], + { maxBuffer: 1024 * 1024, timeout: 30_000 } + ); +} + +async function generateStaticFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=blue:s=320x180:d=8:r=12", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + ]); +} + +async function generateMixedFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=6:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function generateBlurExposureFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=3:r=12", + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=3:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v]gblur=sigma=12[blur];[blur][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function generateDenseTailFixture(outputPath: string): Promise { + const args: string[] = []; + for (const source of [ + "color=c=black:s=160x90:d=0.5:r=10", + "color=c=white:s=160x90:d=0.5:r=10", + "color=c=black:s=160x90:d=0.5:r=10", + "color=c=white:s=160x90:d=0.5:r=10", + "testsrc2=s=160x90:d=8:r=10", + ]) { + args.push("-f", "lavfi", "-i", source); + } + args.push( + "-filter_complex", + "[0:v][1:v][2:v][3:v][4:v]concat=n=5:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast" + ); + await generateFixture(outputPath, args); +} + +async function generateGradualFadeFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=white:s=320x180:d=8:r=12", + "-vf", + "fade=t=out:st=0:d=8,format=yuv420p", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function supportsTimeBinary(): Promise { + try { + await access("/usr/bin/time"); + return true; + } catch { + return false; + } +} + +function parseTimeCost(stderr: string, wallMs: number): ChildCost { + const match = new RegExp(`${TIME_MARKER} ([\\d.]+) ([\\d.]+) ([\\d.]+)`).exec(stderr); + return { + maxRssKiB: match ? Number(match[3]) : null, + systemSeconds: match ? Number(match[2]) : null, + userSeconds: match ? Number(match[1]) : null, + wallMs, + }; +} + +async function timedAnalysis( + inputPath: string, + durationSeconds: number, + useTimeBinary: boolean +): Promise<{ analysis: VideoStructuralAnalysis; cost: ChildCost }> { + let cost: ChildCost = { + maxRssKiB: null, + systemSeconds: null, + userSeconds: null, + wallMs: 0, + }; + const runner: VideoCommandRunner = async (executable, args, options) => { + const startedAt = performance.now(); + const command = useTimeBinary ? "/usr/bin/time" : executable; + const commandArgs = useTimeBinary + ? ["-f", `${TIME_MARKER} %U %S %M`, executable, ...args] + : [...args]; + const result = await execFileAsync(command, commandArgs, { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + }); + cost = parseTimeCost(String(result.stderr), performance.now() - startedAt); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; + }; + const analysis = await analyzeVideoStructure(inputPath, { + durationSeconds, + runner, + streamIndex: 0, + timeoutMs: 30_000, + }); + return { analysis, cost }; +} + +function sampling( + durationSeconds: number, + frameCount: number, + analysis: VideoStructuralAnalysis +): { structural: number[]; uniform: number[] } { + const uniform = calculateSamplingDecision(durationSeconds, frameCount, "uniform").timestamps; + const structural = calculateSamplingDecision( + durationSeconds, + frameCount, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ).timestamps; + return { structural, uniform }; +} + +async function captionCallsAfterDedup( + inputPath: string, + outputDirectory: string, + samplingPolicy: "segment_aware" | "uniform" +): Promise { + await mkdir(outputDirectory, { mode: 0o700 }); + const frames = await extractFramesFromLocalVideo(inputPath, outputDirectory, { + durationSeconds: 8, + frameCount: 8, + samplingPolicy, + streamIndex: 0, + timeoutMs: 30_000, + }); + const bytes = await readBoundedExtractedFrames(frames); + const deduplicated = await deduplicateVideoFrames( + frames.map((frame, index) => ({ + dataUri: `data:image/jpeg;base64,${bytes[index].toString("base64")}`, + timestampSeconds: frame.timestampSeconds, + })) + ); + return deduplicated.frames.length; +} + +function result( + name: string, + cost: ChildCost, + analysis: VideoStructuralAnalysis, + uniform: number[], + structural: number[], + oracle: Record, + captionCallsAvoided = 0 +): FixtureResult { + const booleans = Object.values(oracle).filter( + (value): value is boolean => typeof value === "boolean" + ); + return { + captionCallsAvoided, + childCost: cost, + freezeIntervals: analysis.freezeIntervals.length, + name, + oracle, + passed: booleans.every(Boolean), + sceneCandidates: analysis.sceneCandidates.length, + structuralFrames: structural.length, + uniformFrames: uniform.length, + }; +} + +async function main(): Promise { + const version = await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + const filters = await execFileAsync("ffmpeg", ["-hide_banner", "-filters"], { + maxBuffer: 2 * 1024 * 1024, + timeout: 5_000, + }); + const missingFilters = REQUIRED_FILTERS.filter( + (filter) => !new RegExp(`\\b${filter}\\b`).test(String(filters.stdout)) + ); + if (missingFilters.length > 0) + throw new Error(`Missing required FFmpeg filters: ${missingFilters.join(", ")}`); + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-eval-")); + const useTimeBinary = await supportsTimeBinary(); + const results: FixtureResult[] = []; + try { + const staticPath = join(directory, "static.mp4"); + await generateStaticFixture(staticPath); + const staticRun = await timedAnalysis(staticPath, 8, useTimeBinary); + const staticSampling = sampling(8, 8, staticRun.analysis); + const uniformCaptionCalls = await captionCallsAfterDedup( + staticPath, + join(directory, "static-uniform"), + "uniform" + ); + const structuralCaptionCalls = await captionCallsAfterDedup( + staticPath, + join(directory, "static-structural"), + "segment_aware" + ); + const staticCaptionCallsAvoided = Math.max(0, uniformCaptionCalls - structuralCaptionCalls); + results.push( + result( + "static-caption-savings", + staticRun.cost, + staticRun.analysis, + staticSampling.uniform, + staticSampling.structural, + { + fullFreezeDetected: staticRun.analysis.freezeIntervals.some( + (interval) => interval.startSeconds <= 1 && interval.endSeconds >= 7 + ), + oneIncrementalCaptionCallAvoided: staticCaptionCallsAvoided === 1, + structuralCaptionCalls, + uniformCaptionCalls, + }, + staticCaptionCallsAvoided + ) + ); + + const mixedPath = join(directory, "mixed.mp4"); + await generateMixedFixture(mixedPath); + const mixedRun = await timedAnalysis(mixedPath, 10, useTimeBinary); + const mixedSampling = sampling(10, 4, mixedRun.analysis); + const uniformDense = mixedSampling.uniform.filter((timestamp) => timestamp > 6).length; + const structuralDense = mixedSampling.structural.filter((timestamp) => timestamp > 6).length; + results.push( + result( + "dense-budget-quality-oracle", + mixedRun.cost, + mixedRun.analysis, + mixedSampling.uniform, + mixedSampling.structural, + { + denseFramesStructural: structuralDense, + denseFramesUniform: uniformDense, + denseRegionGetsMoreBudget: structuralDense > uniformDense, + frozenRegionRetainsCoverage: mixedSampling.structural.some((timestamp) => timestamp < 6), + } + ) + ); + + const qualityPath = join(directory, "blur-exposure.mp4"); + await generateBlurExposureFixture(qualityPath); + const qualityRun = await timedAnalysis(qualityPath, 10, useTimeBinary); + const qualitySampling = sampling(10, 6, qualityRun.analysis); + const blurred = samplesIn(qualityRun.analysis, 0, 3); + const dark = samplesIn(qualityRun.analysis, 3, 6); + const sharp = samplesIn(qualityRun.analysis, 6, 10); + const blurredBlur = average(blurred.map((sample) => sample.blur)); + const blurredSpatial = average(blurred.map((sample) => sample.spatialInformation)); + const darkLuma = average(dark.map((sample) => sample.brightness)); + const sharpBlur = average(sharp.map((sample) => sample.blur)); + const sharpSpatial = average(sharp.map((sample) => sample.spatialInformation)); + const sharpTemporal = average(sharp.map((sample) => sample.temporalInformation)); + const sharpLuma = average(sharp.map((sample) => sample.brightness)); + results.push( + result( + "blur-exposure-spatial-temporal-evidence", + qualityRun.cost, + qualityRun.analysis, + qualitySampling.uniform, + qualitySampling.structural, + { + blurMetricSeparated: + blurredBlur !== null && sharpBlur !== null && Math.abs(blurredBlur - sharpBlur) >= 0.05, + blurredBlur: blurredBlur ?? "missing", + darkLuma: darkLuma ?? "missing", + exposureSeparated: darkLuma !== null && sharpLuma !== null && sharpLuma - darkLuma >= 50, + sharpBlur: sharpBlur ?? "missing", + sharpSpatial: sharpSpatial ?? "missing", + sharpTemporal: sharpTemporal ?? "missing", + spatialDetailSeparated: + blurredSpatial !== null && sharpSpatial !== null && sharpSpatial - blurredSpatial >= 20, + structuralKeepsSharpRegion: + qualitySampling.structural.filter((timestamp) => timestamp >= 6).length >= 2, + temporalChangeDetected: sharpTemporal !== null && sharpTemporal >= 5, + } + ) + ); + + const tailPath = join(directory, "dense-tail.mp4"); + await generateDenseTailFixture(tailPath); + const tailRun = await timedAnalysis(tailPath, 10, useTimeBinary); + const tailSampling = sampling(10, 4, tailRun.analysis); + results.push( + result( + "dense-cuts-long-tail-regression", + tailRun.cost, + tailRun.analysis, + tailSampling.uniform, + tailSampling.structural, + { + multipleEarlyCuts: tailRun.analysis.sceneCandidates.length >= 3, + trailingEightSecondsRepresented: tailSampling.structural.some( + (timestamp) => timestamp > 2 + ), + } + ) + ); + + const fadePath = join(directory, "gradual-fade.mp4"); + await generateGradualFadeFixture(fadePath); + const fadeRun = await timedAnalysis(fadePath, 8, useTimeBinary); + const fadeSampling = sampling(8, 4, fadeRun.analysis); + results.push( + result( + "gradual-fade-false-positive", + fadeRun.cost, + fadeRun.analysis, + fadeSampling.uniform, + fadeSampling.structural, + { + hardCutFalsePositives: fadeRun.analysis.sceneCandidates.length, + noHardCutBurst: fadeRun.analysis.sceneCandidates.length <= 1, + noCaptionBudgetPruning: fadeSampling.structural.length === fadeSampling.uniform.length, + } + ) + ); + } finally { + await rm(directory, { force: true, recursive: true }); + } + + const callsAvoided = results.reduce((sum, fixture) => sum + fixture.captionCallsAvoided, 0); + const costFlag = process.argv.indexOf("--caption-cost-per-call-usd"); + const explicitCost = Number(costFlag >= 0 ? process.argv[costFlag + 1] : Number.NaN); + const report = { + captionCost: + Number.isFinite(explicitCost) && explicitCost > 0 + ? { + estimatedUsdAvoided: callsAvoided * explicitCost, + source: "explicit environment input", + status: "ESTIMATED_FROM_INPUT", + } + : { + reason: "--caption-cost-per-call-usd was not supplied with a positive number", + status: "HOLD", + }, + ffmpegVersion: String(version.stdout).split("\n")[0], + fixtures: results, + modelQuality: { + reason: + "No authorized real caption-model endpoint, credentials, or frozen judge rubric were configured; deterministic structural oracles are not semantic quality.", + status: "HOLD", + }, + gainCostComparison: { + reason: + "The real post-dedup caption-call delta is measured, but no authorized caption latency/cost receipt or child CPU/RSS receipt is configured.", + status: "HOLD", + }, + resourceCost: useTimeBinary + ? { source: "/usr/bin/time", status: "MEASURED" } + : { + reason: "/usr/bin/time is unavailable; wall time is measured but child CPU/RSS are not", + status: "HOLD", + }, + summary: { + captionCallsAvoided: callsAvoided, + failed: results.filter((fixture) => !fixture.passed).map((fixture) => fixture.name), + passed: results.filter((fixture) => fixture.passed).length, + total: results.length, + }, + timeBinary: useTimeBinary ? "/usr/bin/time" : null, + }; + console.log(JSON.stringify(report, null, 2)); + if (report.summary.failed.length > 0) process.exitCode = 1; +} + +await main(); diff --git a/src/lib/guardrails/videoBridgeRuntime.ts b/src/lib/guardrails/videoBridgeRuntime.ts index 2b57bffc8b..736d240c19 100644 --- a/src/lib/guardrails/videoBridgeRuntime.ts +++ b/src/lib/guardrails/videoBridgeRuntime.ts @@ -69,6 +69,24 @@ export interface VideoSamplingDecision extends VideoSamplingMetadata { timestamps: number[]; } +export interface VideoStructuralInterval { + endSeconds: number; + startSeconds: number; +} +export interface VideoStructuralSample { + blur?: number | null; + brightness?: number | null; + sceneScore?: number | null; + spatialInformation?: number | null; + temporalInformation?: number | null; + timestampSeconds: number; +} +export interface VideoStructuralAnalysis { + freezeIntervals: VideoStructuralInterval[]; + samples: VideoStructuralSample[]; + sceneCandidates: number[]; +} + export function resolveVideoFocusWindow( durationSeconds: number, bounds: VideoFocusBounds @@ -95,6 +113,10 @@ export const VIDEO_FRAME_MAX_BYTES = 4 * 1024 * 1024; export const VIDEO_FRAMES_TOTAL_MAX_BYTES = 23 * 1024 * 1024; export const VIDEO_MAX_DIMENSION = 8_192; export const VIDEO_MAX_PIXELS = 33_554_432; +const VIDEO_STRUCTURAL_ANALYSIS_FPS = 1; +const VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES = 600; +const VIDEO_STRUCTURAL_ANALYSIS_MAX_WIDTH = 320; +const VIDEO_STRUCTURAL_SCENE_THRESHOLD = 10; const SAFE_FORMATS = new Set([ "3g2", @@ -111,7 +133,6 @@ const SAFE_FORMATS = new Set([ "webm", ]); const SAFE_FORMAT_WHITELIST = [...SAFE_FORMATS].join(","); - const defaultRunner: VideoCommandRunner = async (executable, args, options) => { const result = await execFileAsync(executable, [...args], { encoding: "utf8", @@ -122,7 +143,6 @@ const defaultRunner: VideoCommandRunner = async (executable, args, options) => { }); return { stdout: String(result.stdout), stderr: String(result.stderr) }; }; - function assertLocalPath(filePath: string): void { if (!isAbsolute(filePath) || filePath.includes("\0") || filePath.includes("://")) { throw new Error("Video runtime requires a local path"); @@ -223,7 +243,6 @@ function normalizeSceneCandidates( } return [...unique].sort((left, right) => left - right); } - export function parseSceneChangeTimestamps(output: string, durationSeconds: number): number[] { const candidates: number[] = []; const timestampPattern = /\bpts_time:([+-]?(?:\d+(?:\.\d*)?|\.\d+))\b/g; @@ -233,67 +252,256 @@ export function parseSceneChangeTimestamps(output: string, durationSeconds: numb } return normalizeSceneCandidates(durationSeconds, candidates); } - -/** Allocate midpoint samples proportionally across validated scene segments. */ +const STRUCTURAL_METRIC_FIELDS = { + "lavfi.blur": "blur", + "lavfi.scd.score": "sceneScore", + "lavfi.signalstats.YAVG": "brightness", + "lavfi.siti.si": "spatialInformation", + "lavfi.siti.ti": "temporalInformation", +} as const; +function parseStructuralSamples(output: string, durationSeconds: number): VideoStructuralSample[] { + const samples = new Map(); + const pattern = /\bpts_time:([+-]?(?:\d+(?:\.\d*)?|\.\d+))[^\n]*\r?\n([A-Za-z0-9_.]+)=([^\s]+)/g; + for (const match of output.matchAll(pattern)) { + const timestamp = Number(Number(match[1]).toFixed(3)); + const field = STRUCTURAL_METRIC_FIELDS[match[2] as keyof typeof STRUCTURAL_METRIC_FIELDS]; + const metric = Number(match[3]); + const unusable = !field || timestamp < 0 || timestamp >= durationSeconds; + if (unusable || (!Number.isFinite(metric) && !samples.has(timestamp))) continue; + if (!samples.has(timestamp)) { + if (samples.size >= VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES) continue; + samples.set(timestamp, { + timestampSeconds: timestamp, + }); + } + const sample = samples.get(timestamp); + if (sample) sample[field] = Number.isFinite(metric) ? metric : null; + } + return [...samples.values()].sort( + (left, right) => left.timestampSeconds - right.timestampSeconds + ); +} +function parseStructuralMetricEvents(output: string, metric: string): number[] { + const pattern = new RegExp(`${metric}:\\s*([+-]?(?:\\d+(?:\\.\\d*)?|\\.\\d+))`, "g"); + return [...output.matchAll(pattern)].map((match) => Number(match[1])).filter(Number.isFinite); +} +function parseFreezeIntervals(output: string, durationSeconds: number): VideoStructuralInterval[] { + const starts = parseStructuralMetricEvents(output, "freeze_start"); + const ends = parseStructuralMetricEvents(output, "freeze_end"); + const durations = parseStructuralMetricEvents(output, "freeze_duration"); + return starts + .map((start, index) => { + const startSeconds = Math.max(0, Math.min(durationSeconds, start)); + const inferredEnd = start + (durations[index] ?? durationSeconds - start); + const endSeconds = Math.max( + startSeconds, + Math.min(durationSeconds, ends[index] ?? inferredEnd) + ); + return { endSeconds, startSeconds }; + }) + .filter((interval) => interval.endSeconds - interval.startSeconds >= 1); +} +export function parseVideoStructuralAnalysis( + metadataOutput: string, + diagnosticOutput: string, + durationSeconds: number +): VideoStructuralAnalysis { + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + throw new Error("Video structural analysis requires a positive duration"); + } + const samples = parseStructuralSamples(metadataOutput, durationSeconds); + const diagnosticScenes = [ + ...diagnosticOutput.matchAll(/lavfi\.scd\.score:\s*[\d.]+,\s*lavfi\.scd\.time:\s*([\d.]+)/g), + ].map((match) => Number(match[1])); + return { + freezeIntervals: parseFreezeIntervals(diagnosticOutput, durationSeconds), + samples, + sceneCandidates: normalizeSceneCandidates(durationSeconds, [ + ...samples + .filter((sample) => (sample.sceneScore ?? 0) >= VIDEO_STRUCTURAL_SCENE_THRESHOLD) + .map((sample) => sample.timestampSeconds), + ...diagnosticScenes, + ]), + }; +} +interface StructuralSamplingSegment { + endSeconds: number; + frozen: boolean; + priority: number; + startSeconds: number; +} +function averageStructuralMetric(values: Array): number | null { + const finite = values.filter( + (value): value is number => value !== null && value !== undefined && Number.isFinite(value) + ); + return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null; +} +function normalizedStructuralMetric( + samples: readonly VideoStructuralSample[], + field: Exclude, + fallback: number, + scale: number +): number { + return Math.min( + 1, + Math.max( + 0, + (averageStructuralMetric(samples.map((sample) => sample[field])) ?? fallback) / scale + ) + ); +} +function structuralSegmentPriority( + startSeconds: number, + endSeconds: number, + analysis: VideoStructuralAnalysis +): StructuralSamplingSegment { + const length = endSeconds - startSeconds; + const samples = analysis.samples.filter( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); + const freezeCoverage = Math.min( + 1, + analysis.freezeIntervals.reduce( + (sum, interval) => + sum + + Math.max( + 0, + Math.min(endSeconds, interval.endSeconds) - Math.max(startSeconds, interval.startSeconds) + ), + 0 + ) / length + ); + const spatial = normalizedStructuralMetric(samples, "spatialInformation", 40, 100); + const temporal = normalizedStructuralMetric(samples, "temporalInformation", 10, 30); + const sharpness = 1 - normalizedStructuralMetric(samples, "blur", 10, 20); + const brightness = averageStructuralMetric(samples.map((sample) => sample.brightness)); + const exposure = brightness === null || (brightness >= 24 && brightness <= 232) ? 1 : 0.25; + const interest = exposure * (0.2 + spatial * 0.3 + temporal * 0.4 + sharpness * 0.1); + const maxTemporal = Math.max(0, ...samples.map((sample) => sample.temporalInformation ?? 0)); + return { + endSeconds, + frozen: freezeCoverage >= 0.8 && maxTemporal <= 1, + priority: length * Math.max(0.05, interest) * (1 - freezeCoverage * 0.75), + startSeconds, + }; +} +function allocateStructuralFrames( + segments: readonly StructuralSamplingSegment[], + frameCount: number +): number[] { + if (segments.length > frameCount) return segments.map(() => 0); + const allocation = segments.map(() => 1); + let remaining = frameCount - segments.length; + const totalPriority = segments.reduce( + (sum, segment) => sum + (segment.frozen ? 0 : segment.priority), + 0 + ); + if (totalPriority <= 0) return allocation; + const idealExtras = segments.map((segment) => + segment.frozen ? 0 : (segment.priority / totalPriority) * remaining + ); + const extras = idealExtras.map((value) => Math.floor(value)); + remaining -= extras.reduce((sum, value) => sum + value, 0); + const remainderOrder = idealExtras + .map((value, index) => ({ index, remainder: value - Math.floor(value) })) + .sort((left, right) => right.remainder - left.remainder || left.index - right.index); + for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1; + return allocation.map((value, index) => value + extras[index]); +} +function timestampsFromSegmentAllocation( + segments: readonly Pick[], + allocation: readonly number[] +): number[] { + return segments.flatMap((segment, segmentIndex) => + Array.from( + { length: allocation[segmentIndex] }, + (_unused, index) => + segment.startSeconds + + ((index + 0.5) * (segment.endSeconds - segment.startSeconds)) / allocation[segmentIndex] + ) + ); +} +function calculateLengthWeightedSegmentTimestamps( + startSeconds: number, + endSeconds: number, + frameCount: number, + boundaries: readonly number[] +): number[] { + const uniform = calculateFrameTimestamps(endSeconds - startSeconds, frameCount).map( + (timestamp) => timestamp + startSeconds + ); + const starts = [startSeconds, ...boundaries]; + const ends = [...boundaries, endSeconds]; + const segments = starts.map((start, index) => ({ + endSeconds: ends[index], + frozen: false, + priority: ends[index] - start, + startSeconds: start, + })); + return segments.length > frameCount + ? uniform + : timestampsFromSegmentAllocation(segments, allocateStructuralFrames(segments, frameCount)); +} +/** Allocate a bounded caption budget across validated structural segments. */ export function calculateSegmentAwareTimestamps( durationSeconds: number, requestedFrameCount: number, sceneCandidates: readonly number[], - focusWindow: VideoFocusWindow | null = null + focusWindow: VideoFocusWindow | null = null, + structuralAnalysis: VideoStructuralAnalysis | null = null ): number[] { const startSeconds = focusWindow?.startSeconds ?? 0; const endSeconds = focusWindow?.endSeconds ?? durationSeconds; const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map( (timestamp) => timestamp + startSeconds ); - const boundaries = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter( - (timestamp) => timestamp > startSeconds && timestamp < endSeconds + const structuralBoundaries = structuralAnalysis?.freezeIntervals.flatMap((interval) => [ + interval.startSeconds, + interval.endSeconds, + ]); + const sceneBoundaries = sceneCandidates.filter( + (candidate) => + !structuralBoundaries?.some( + (boundary) => Math.abs(candidate - boundary) <= 1 / VIDEO_STRUCTURAL_ANALYSIS_FPS + ) ); - if (boundaries.length === 0) return uniform; - const segmentStarts = [startSeconds, ...boundaries]; - const segmentEnds = [...boundaries, endSeconds]; - const lengths = segmentStarts.map((segmentStart, index) => segmentEnds[index] - segmentStart); - const segmentCount = lengths.length; - const frameCount = uniform.length; - if (segmentCount > frameCount) { - return [...uniform].map((timestamp, index) => { - const segmentIndex = Math.min( - segmentCount - 1, - Math.floor((index * segmentCount) / frameCount) - ); - const segmentStart = segmentStarts[segmentIndex]; - const segmentEnd = segmentEnds[segmentIndex]; - return segmentStart + (segmentEnd - segmentStart) / 2; - }); + const boundaries = normalizeSceneCandidates(durationSeconds, [ + ...sceneBoundaries, + ...(structuralBoundaries ?? []), + ]).filter((timestamp) => timestamp > startSeconds && timestamp < endSeconds); + if (!structuralAnalysis) { + return boundaries.length === 0 + ? uniform + : calculateLengthWeightedSegmentTimestamps( + startSeconds, + endSeconds, + uniform.length, + boundaries + ); } - const allocation = lengths.map(() => 1); - let remaining = frameCount - segmentCount; - const idealExtra = lengths.map((length) => (length / (endSeconds - startSeconds)) * remaining); - const extras = idealExtra.map((value) => Math.floor(value)); - remaining -= extras.reduce((sum, value) => sum + value, 0); - const remainderOrder = idealExtra - .map((value, index) => ({ index, remainder: value - Math.floor(value) })) - .sort((left, right) => right.remainder - left.remainder || left.index - right.index); - for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1; - for (let index = 0; index < allocation.length; index++) allocation[index] += extras[index]; - const timestamps: number[] = []; - for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) { - const count = allocation[segmentIndex]; - const segmentStart = segmentStarts[segmentIndex]; - const segmentLength = lengths[segmentIndex]; - for (let index = 0; index < count; index++) { - timestamps.push(segmentStart + ((index + 0.5) * segmentLength) / count); - } + const starts = [startSeconds, ...boundaries]; + const ends = [...boundaries, endSeconds]; + const segments = starts.map((start, index) => + structuralSegmentPriority(start, ends[index], structuralAnalysis) + ); + const allocation = allocateStructuralFrames(segments, uniform.length); + if (segments.length > uniform.length) { + return calculateLengthWeightedSegmentTimestamps( + startSeconds, + endSeconds, + uniform.length, + boundaries + ); } - return timestamps; + return timestampsFromSegmentAllocation(segments, allocation); } - export function calculateSamplingDecision( durationSeconds: number, requestedFrameCount: number, policy: VideoSamplingPolicy, sceneCandidates: readonly number[] = [], - focusWindow: VideoFocusWindow | null = null + focusWindow: VideoFocusWindow | null = null, + structuralAnalysis: VideoStructuralAnalysis | null = null ): VideoSamplingDecision { const startSeconds = focusWindow?.startSeconds ?? 0; const endSeconds = focusWindow?.endSeconds ?? durationSeconds; @@ -309,21 +517,17 @@ export function calculateSamplingDecision( timestamps: uniform, }; } - const candidates = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter( - (timestamp) => timestamp >= startSeconds && timestamp < endSeconds + (timestamp) => timestamp > startSeconds && timestamp < endSeconds ); - if (candidates.length === 0) { - return { - candidateCount: 0, - ...(focusWindow ? { focusWindow } : {}), - policyEffective: "uniform", - policyRequested: policy, - timestamps: uniform, - }; - } - - if (policy === "segment_aware") { + const focusHasSample = structuralAnalysis?.samples.some( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); + const focusHasFreeze = structuralAnalysis?.freezeIntervals.some( + (interval) => interval.startSeconds < endSeconds && interval.endSeconds > startSeconds + ); + const hasStructuralEvidence = Boolean(focusHasSample || focusHasFreeze); + if (policy === "segment_aware" && (candidates.length > 0 || hasStructuralEvidence)) { return { candidateCount: candidates.length, ...(focusWindow ? { focusWindow } : {}), @@ -333,11 +537,20 @@ export function calculateSamplingDecision( durationSeconds, requestedFrameCount, candidates, - focusWindow + focusWindow, + structuralAnalysis ), }; } - + if (candidates.length === 0) { + return { + candidateCount: 0, + ...(focusWindow ? { focusWindow } : {}), + policyEffective: "uniform", + policyRequested: policy, + timestamps: uniform, + }; + } const frameCount = uniform.length; if (frameCount === 1) { return { @@ -378,7 +591,6 @@ export function calculateSamplingDecision( timestamps: selected, }; } - export async function detectSceneChangeTimestamps( inputPath: string, options: { @@ -421,7 +633,72 @@ export async function detectSceneChangeTimestamps( ); return parseSceneChangeTimestamps(`${result.stdout}\n${result.stderr}`, options.durationSeconds); } - +const STRUCTURAL_ANALYSIS_FILTER = [ + `scale=w='min(${VIDEO_STRUCTURAL_ANALYSIS_MAX_WIDTH},iw)':h=-2:flags=fast_bilinear`, + `scdet=threshold=${VIDEO_STRUCTURAL_SCENE_THRESHOLD}`, + "freezedetect=n=-60dB:d=1", + `fps=${VIDEO_STRUCTURAL_ANALYSIS_FPS}`, + "siti", + "blurdetect=radius=10:block_width=32:block_height=32", + "signalstats", + ...[ + "lavfi.scd.score", + "lavfi.siti.si", + "lavfi.siti.ti", + "lavfi.blur", + "lavfi.signalstats.YAVG", + ].map((key) => `metadata=mode=print:key=${key}:file=-`), +].join(","); +export async function analyzeVideoStructure( + inputPath: string, + options: { + durationSeconds: number; + runner?: VideoCommandRunner; + signal?: AbortSignal; + streamIndex: number; + timeoutMs?: number; + } +): Promise { + assertLocalPath(inputPath); + if (!Number.isFinite(options.durationSeconds) || options.durationSeconds <= 0) { + throw new Error("Video structural analysis requires a positive duration"); + } + if (!Number.isInteger(options.streamIndex) || options.streamIndex < 0) { + throw new Error("Video stream index is invalid"); + } + const result = await (options.runner ?? defaultRunner)( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "info", + "-nostats", + "-protocol_whitelist", + "file", + "-format_whitelist", + SAFE_FORMAT_WHITELIST, + "-threads", + "1", + "-filter_threads", + "1", + "-i", + inputPath, + "-map", + `0:${options.streamIndex}`, + "-vf", + STRUCTURAL_ANALYSIS_FILTER, + "-an", + "-frames:v", + String(VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES), + "-f", + "null", + "-", + ], + { signal: options.signal, timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000) } + ); + return parseVideoStructuralAnalysis(result.stdout, result.stderr, options.durationSeconds); +} export async function probeLocalVideo( inputPath: string, options: { @@ -556,18 +833,31 @@ export async function extractFramesFromLocalVideo( assertLocalPath(outputDirectory); const policy = options.samplingPolicy ?? "uniform"; let sceneCandidates: number[] = []; + let structuralAnalysis: VideoStructuralAnalysis | null = null; if (policy !== "uniform") { try { - sceneCandidates = await detectSceneChangeTimestamps(inputPath, { - durationSeconds: options.durationSeconds, - runner: options.runner, - signal: options.signal, - streamIndex: options.streamIndex, - timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), - }); + if (policy === "segment_aware") { + structuralAnalysis = await analyzeVideoStructure(inputPath, { + durationSeconds: options.durationSeconds, + runner: options.runner, + signal: options.signal, + streamIndex: options.streamIndex, + timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), + }); + sceneCandidates = structuralAnalysis.sceneCandidates; + } else { + sceneCandidates = await detectSceneChangeTimestamps(inputPath, { + durationSeconds: options.durationSeconds, + runner: options.runner, + signal: options.signal, + streamIndex: options.streamIndex, + timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), + }); + } } catch { if (options.signal?.aborted) throw new Error("Video extraction request aborted"); sceneCandidates = []; + structuralAnalysis = null; } } const focusWindow = options.focusWindow @@ -578,7 +868,8 @@ export async function extractFramesFromLocalVideo( options.frameCount, policy, sceneCandidates, - focusWindow + focusWindow, + structuralAnalysis ); if (!Number.isInteger(options.streamIndex) || options.streamIndex < 0) { throw new Error("Video stream index is invalid"); diff --git a/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts b/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts new file mode 100644 index 0000000000..5c5695f018 --- /dev/null +++ b/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts @@ -0,0 +1,486 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { + analyzeVideoStructure, + calculateSamplingDecision, + extractFramesFromLocalVideo, + extractVideoFramesFromBytes, + parseVideoStructuralAnalysis, + type VideoCommandRunner, + type VideoStructuralAnalysis, +} from "../../../src/lib/guardrails/videoBridgeRuntime.ts"; + +const execFileAsync = promisify(execFile); + +async function writeFrozenThenMotionFixture(fixturePath: string): Promise { + await execFileAsync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=6:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-threads", + "1", + "-y", + fixturePath, + ], + { timeout: 30_000 } + ); +} + +const realRunner: VideoCommandRunner = async (executable, args, options) => { + const result = await execFileAsync(executable, [...args], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + }); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; +}; + +function structuralAnalysis( + overrides: Partial = {} +): VideoStructuralAnalysis { + return { + freezeIntervals: [{ endSeconds: 6, startSeconds: 0 }], + samples: [ + { + blur: null, + brightness: 16, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 1, + }, + { + blur: 4.8, + brightness: 121, + sceneScore: 42, + spatialInformation: 120, + temporalInformation: 32, + timestampSeconds: 6, + }, + { + blur: 4.9, + brightness: 122, + sceneScore: 0, + spatialInformation: 118, + temporalInformation: 28, + timestampSeconds: 8, + }, + ], + sceneCandidates: [6], + ...overrides, + }; +} + +test("parses scene, freeze, blur, exposure, and spatial-temporal evidence", () => { + const metadata = [ + "frame:0 pts:0 pts_time:0", + "lavfi.scd.score=0.000", + "frame:0 pts:0 pts_time:0", + "lavfi.siti.si=0.00", + "frame:0 pts:0 pts_time:0", + "lavfi.siti.ti=0.00", + "frame:0 pts:0 pts_time:0", + "lavfi.blur=-nan", + "frame:0 pts:0 pts_time:0", + "lavfi.signalstats.YAVG=16", + "frame:6 pts:6 pts_time:6", + "lavfi.scd.score=41.013", + "frame:6 pts:6 pts_time:6", + "lavfi.siti.si=108.50", + "frame:6 pts:6 pts_time:6", + "lavfi.siti.ti=66.51", + "frame:6 pts:6 pts_time:6", + "lavfi.blur=4.75", + "frame:6 pts:6 pts_time:6", + "lavfi.signalstats.YAVG=121.5", + ].join("\n"); + const stderr = [ + "lavfi.freezedetect.freeze_start: 0", + "lavfi.freezedetect.freeze_duration: 6", + "lavfi.freezedetect.freeze_end: 6", + ].join("\n"); + + const analysis = parseVideoStructuralAnalysis(metadata, stderr, 10); + + assert.deepEqual(analysis.sceneCandidates, [6]); + assert.deepEqual(analysis.freezeIntervals, [{ endSeconds: 6, startSeconds: 0 }]); + assert.deepEqual(analysis.samples, [ + { + blur: null, + brightness: 16, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 0, + }, + { + blur: 4.75, + brightness: 121.5, + sceneScore: 41.013, + spatialInformation: 108.5, + temporalInformation: 66.51, + timestampSeconds: 6, + }, + ]); +}); + +test("runs all structural filters in one fixed, local-only, bounded FFmpeg pass", async () => { + const calls: Array<{ args: string[]; timeoutMs: number }> = []; + const runner: VideoCommandRunner = async (executable, args, options) => { + assert.equal(executable, "ffmpeg"); + calls.push({ args: [...args], timeoutMs: options.timeoutMs }); + return { + stderr: "lavfi.freezedetect.freeze_start: 0\nlavfi.freezedetect.freeze_end: 2", + stdout: "frame:0 pts:0 pts_time:0\nlavfi.scd.score=0", + }; + }; + + await analyzeVideoStructure("/tmp/input.mp4", { + durationSeconds: 8, + runner, + streamIndex: 2, + timeoutMs: 4_000, + }); + + assert.equal(calls.length, 1, "structural analysis must decode the video exactly once"); + assert.equal(calls[0].timeoutMs, 4_000); + assert.ok(calls[0].args.includes("-nostdin")); + assert.deepEqual(calls[0].args.slice(calls[0].args.indexOf("-map"), -1), [ + "-map", + "0:2", + "-vf", + calls[0].args[calls[0].args.indexOf("-vf") + 1], + "-an", + "-frames:v", + "600", + "-f", + "null", + ]); + const filter = calls[0].args[calls[0].args.indexOf("-vf") + 1]; + for (const expected of ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"]) { + assert.match(filter, new RegExp(expected)); + } + assert.equal( + calls[0].args.some((argument) => argument.includes("://")), + false + ); +}); + +test("spends one frame on a frozen segment and reallocates the budget to dense motion", () => { + const analysis = structuralAnalysis(); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.equal(decision.timestamps.length, 4); + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 6).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 6).length, 3); +}); + +test("avoids redundant caption work for an entirely frozen video", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [{ endSeconds: 8, startSeconds: 0 }], + samples: [ + { + blur: null, + brightness: 81, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 4, + }, + ], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision(8, 8, "segment_aware", [], null, analysis); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.equal(decision.timestamps.length, 1); + assert.deepEqual(decision.timestamps, [4]); +}); + +test("does not prune a moving clip when freeze evidence is absent", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [], + samples: [ + { + blur: 4.8, + brightness: 120, + sceneScore: 0, + spatialInformation: 100, + temporalInformation: 30, + timestampSeconds: 1, + }, + { + blur: 4.9, + brightness: 122, + sceneScore: 0, + spatialInformation: 105, + temporalInformation: 32, + timestampSeconds: 7, + }, + ], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision(8, 4, "segment_aware", [], null, analysis); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.deepEqual(decision.timestamps, [1, 3, 5, 7]); +}); + +test("uses lower FFmpeg blur scores as sharper evidence for the extra frame", () => { + const common = { + brightness: 120, + sceneScore: 0, + spatialInformation: 50, + temporalInformation: 10, + }; + const analysis = structuralAnalysis({ + freezeIntervals: [], + samples: [ + { ...common, blur: 17, timestampSeconds: 1 }, + { ...common, blur: 4, timestampSeconds: 5 }, + ], + sceneCandidates: [4], + }); + const decision = calculateSamplingDecision(8, 3, "segment_aware", [4], null, analysis); + + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 4).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 4).length, 2); +}); + +test("malformed-only structural metadata fails open to uniform sampling", () => { + const analysis = parseVideoStructuralAnalysis("frame:0 pts:0 pts_time:0\nlavfi.blur=-nan", "", 8); + const decision = calculateSamplingDecision(8, 4, "segment_aware", [], null, analysis); + + assert.deepEqual(analysis.samples, []); + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.timestamps, [1, 3, 5, 7]); +}); + +test("keeps the long trailing segment when scene boundaries outnumber the frame budget", () => { + const decision = calculateSamplingDecision(20, 4, "segment_aware", [1, 2, 3, 4]); + + assert.equal(decision.timestamps.length, 4); + assert.ok( + decision.timestamps.some((timestamp) => timestamp > 4), + "the 16-second tail must not be dropped by early short cuts" + ); +}); + +test("preserves the legacy length-weighted allocation without structural evidence", () => { + const decision = calculateSamplingDecision(10, 8, "segment_aware", [2]); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.deepEqual( + decision.timestamps.map((timestamp) => Number(timestamp.toFixed(3))), + [0.5, 1.5, 2.667, 4, 5.333, 6.667, 8, 9.333] + ); +}); + +test("does not report a focus-window boundary as usable segment evidence", () => { + const decision = calculateSamplingDecision(10, 4, "segment_aware", [2], { + endSeconds: 8, + startSeconds: 2, + }); + + assert.equal(decision.policyEffective, "uniform"); + assert.equal(decision.candidateCount, 0); + assert.deepEqual(decision.timestamps, [2.75, 4.25, 5.75, 7.25]); +}); + +test("does not claim segment-aware evidence that falls outside the focus window", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [{ endSeconds: 10, startSeconds: 8 }], + samples: [{ timestampSeconds: 9, temporalInformation: 0 }], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + [], + { endSeconds: 8, startSeconds: 2 }, + analysis + ); + + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.timestamps, [2.75, 4.25, 5.75, 7.25]); +}); + +test("structural timeout fails open to uniform while an abort stops extraction", async () => { + let analysisCalls = 0; + const timeoutRunner: VideoCommandRunner = async (_executable, args) => { + if (args.some((argument) => argument.includes("freezedetect"))) { + analysisCalls += 1; + throw new Error("structural deadline exceeded"); + } + return { stderr: "", stdout: "" }; + }; + + const frames = await extractFramesFromLocalVideo("/tmp/input.mp4", "/tmp/frames", { + durationSeconds: 8, + frameCount: 4, + runner: timeoutRunner, + samplingPolicy: "segment_aware", + streamIndex: 0, + timeoutMs: 250, + }); + assert.equal(analysisCalls, 1); + assert.equal(frames.sampling.policyEffective, "uniform"); + assert.deepEqual( + frames.map((frame) => frame.timestampSeconds), + [1, 3, 5, 7] + ); + + const controller = new AbortController(); + let frameExtractionCalls = 0; + const abortRunner: VideoCommandRunner = async (_executable, args, options) => { + if (args.some((argument) => argument.includes("freezedetect"))) { + assert.equal(options.signal, controller.signal); + controller.abort(); + throw new Error("aborted inside structural analysis"); + } + frameExtractionCalls += 1; + return { stderr: "", stdout: "" }; + }; + await assert.rejects( + () => + extractFramesFromLocalVideo("/tmp/input.mp4", "/tmp/frames", { + durationSeconds: 8, + frameCount: 4, + runner: abortRunner, + samplingPolicy: "segment_aware", + signal: controller.signal, + streamIndex: 0, + timeoutMs: 250, + }), + /aborted/ + ); + assert.equal(frameExtractionCalls, 0); +}); + +test("real FFmpeg evidence distinguishes a frozen dark segment from dense motion", async (t) => { + try { + await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + } catch { + t.skip("FFmpeg is an optional runtime dependency"); + return; + } + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-real-")); + const fixturePath = join(directory, "frozen-then-motion.mp4"); + try { + await writeFrozenThenMotionFixture(fixturePath); + + const analysis = await analyzeVideoStructure(fixturePath, { + durationSeconds: 10, + streamIndex: 0, + timeoutMs: 30_000, + }); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ); + + assert.ok(analysis.samples.length >= 8); + assert.ok(analysis.sceneCandidates.some((timestamp) => Math.abs(timestamp - 6) <= 1)); + assert.ok( + analysis.freezeIntervals.some( + (interval) => interval.startSeconds <= 1 && interval.endSeconds >= 5 + ) + ); + assert.ok(analysis.samples.some((sample) => (sample.spatialInformation ?? 0) > 20)); + assert.ok(analysis.samples.some((sample) => (sample.temporalInformation ?? 0) > 5)); + assert.ok(analysis.samples.some((sample) => (sample.blur ?? 0) > 0)); + assert.ok(analysis.samples.some((sample) => (sample.brightness ?? 255) < 24)); + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 6).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 6).length, 3); + } finally { + await rm(directory, { force: true, recursive: true }); + } +}); + +test("real FFmpeg abort stops preanalysis, skips frame extraction, and cleans the private tree", async (t) => { + try { + await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + } catch { + t.skip("FFmpeg is an optional runtime dependency"); + return; + } + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-abort-")); + const fixturePath = join(directory, "abort.mp4"); + const controller = new AbortController(); + let privateInputPath = ""; + let analysisStarted = false; + let frameExtractionCalls = 0; + try { + await writeFrozenThenMotionFixture(fixturePath); + const bytes = await readFile(fixturePath); + const runner: VideoCommandRunner = async (executable, args, options) => { + if (executable === "ffprobe") privateInputPath = args.at(-1) ?? ""; + if (args.some((argument) => argument.includes("freezedetect"))) { + analysisStarted = true; + setTimeout(() => controller.abort(), 25); + } else if (executable === "ffmpeg") { + frameExtractionCalls += 1; + } + return realRunner(executable, args, options); + }; + + await assert.rejects( + () => + extractVideoFramesFromBytes(bytes, { + frameCount: 4, + maxDurationSeconds: 600, + runner, + samplingPolicy: "segment_aware", + signal: controller.signal, + timeoutMs: 30_000, + }), + /aborted/ + ); + assert.equal(analysisStarted, true); + assert.equal(frameExtractionCalls, 0); + assert.notEqual(privateInputPath, ""); + await assert.rejects(() => access(privateInputPath)); + } finally { + await rm(directory, { force: true, recursive: true }); + } +});