mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-24 16:12:23 +03:00
fix(video-bridge): validate structural segment sampling
This commit is contained in:
Diego Rodrigues de Sa e Souza
493
scripts/perf/video-bridge-fu07-eval.ts
Normal file
493
scripts/perf/video-bridge-fu07-eval.ts
Normal file
@@ -0,0 +1,493 @@
|
||||
/**
|
||||
* Real-media FU-07 structural-sampling evaluation.
|
||||
*
|
||||
* Run: node --import tsx/esm scripts/perf/video-bridge-fu07-eval.ts
|
||||
* Optional estimate: append --caption-cost-per-call-usd <positive number>.
|
||||
*
|
||||
* This evaluates deterministic structural oracles, not semantic model quality.
|
||||
* Model quality and monetary savings remain HOLD without an external receipt.
|
||||
*/
|
||||
import { execFile } from "node:child_process";
|
||||
import { access, mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { deduplicateVideoFrames } from "../../src/lib/guardrails/videoBridgeHelpers";
|
||||
import {
|
||||
analyzeVideoStructure,
|
||||
calculateSamplingDecision,
|
||||
extractFramesFromLocalVideo,
|
||||
readBoundedExtractedFrames,
|
||||
type VideoCommandRunner,
|
||||
type VideoStructuralAnalysis,
|
||||
type VideoStructuralSample,
|
||||
} from "../../src/lib/guardrails/videoBridgeRuntime";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const REQUIRED_FILTERS = ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"];
|
||||
const TIME_MARKER = "__FU07_TIME__";
|
||||
|
||||
interface ChildCost {
|
||||
maxRssKiB: number | null;
|
||||
systemSeconds: number | null;
|
||||
userSeconds: number | null;
|
||||
wallMs: number;
|
||||
}
|
||||
|
||||
interface FixtureResult {
|
||||
captionCallsAvoided: number;
|
||||
childCost: ChildCost;
|
||||
freezeIntervals: number;
|
||||
name: string;
|
||||
oracle: Record<string, boolean | number | string>;
|
||||
passed: boolean;
|
||||
sceneCandidates: number;
|
||||
structuralFrames: number;
|
||||
uniformFrames: number;
|
||||
}
|
||||
|
||||
function average(values: Array<number | null | undefined>): number | null {
|
||||
const finite = values.filter(
|
||||
(value): value is number => value !== null && value !== undefined && Number.isFinite(value)
|
||||
);
|
||||
return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null;
|
||||
}
|
||||
|
||||
function samplesIn(
|
||||
analysis: VideoStructuralAnalysis,
|
||||
startSeconds: number,
|
||||
endSeconds: number
|
||||
): VideoStructuralSample[] {
|
||||
return analysis.samples.filter(
|
||||
(sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds
|
||||
);
|
||||
}
|
||||
|
||||
async function generateFixture(outputPath: string, args: readonly string[]): Promise<void> {
|
||||
await execFileAsync(
|
||||
"ffmpeg",
|
||||
["-hide_banner", "-loglevel", "error", ...args, "-threads", "1", "-y", outputPath],
|
||||
{ maxBuffer: 1024 * 1024, timeout: 30_000 }
|
||||
);
|
||||
}
|
||||
|
||||
async function generateStaticFixture(outputPath: string): Promise<void> {
|
||||
await generateFixture(outputPath, [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=blue:s=320x180:d=8:r=12",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
]);
|
||||
}
|
||||
|
||||
async function generateMixedFixture(outputPath: string): Promise<void> {
|
||||
await generateFixture(outputPath, [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=320x180:d=6:r=12",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=320x180:d=4:r=12",
|
||||
"-filter_complex",
|
||||
"[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]",
|
||||
"-map",
|
||||
"[v]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
]);
|
||||
}
|
||||
|
||||
async function generateBlurExposureFixture(outputPath: string): Promise<void> {
|
||||
await generateFixture(outputPath, [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=320x180:d=3:r=12",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=black:s=320x180:d=3:r=12",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=s=320x180:d=4:r=12",
|
||||
"-filter_complex",
|
||||
"[0:v]gblur=sigma=12[blur];[blur][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]",
|
||||
"-map",
|
||||
"[v]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
]);
|
||||
}
|
||||
|
||||
async function generateDenseTailFixture(outputPath: string): Promise<void> {
|
||||
const args: string[] = [];
|
||||
for (const source of [
|
||||
"color=c=black:s=160x90:d=0.5:r=10",
|
||||
"color=c=white:s=160x90:d=0.5:r=10",
|
||||
"color=c=black:s=160x90:d=0.5:r=10",
|
||||
"color=c=white:s=160x90:d=0.5:r=10",
|
||||
"testsrc2=s=160x90:d=8:r=10",
|
||||
]) {
|
||||
args.push("-f", "lavfi", "-i", source);
|
||||
}
|
||||
args.push(
|
||||
"-filter_complex",
|
||||
"[0:v][1:v][2:v][3:v][4:v]concat=n=5:v=1:a=0,format=yuv420p[v]",
|
||||
"-map",
|
||||
"[v]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast"
|
||||
);
|
||||
await generateFixture(outputPath, args);
|
||||
}
|
||||
|
||||
async function generateGradualFadeFixture(outputPath: string): Promise<void> {
|
||||
await generateFixture(outputPath, [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=white:s=320x180:d=8:r=12",
|
||||
"-vf",
|
||||
"fade=t=out:st=0:d=8,format=yuv420p",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"ultrafast",
|
||||
]);
|
||||
}
|
||||
|
||||
async function supportsTimeBinary(): Promise<boolean> {
|
||||
try {
|
||||
await access("/usr/bin/time");
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function parseTimeCost(stderr: string, wallMs: number): ChildCost {
|
||||
const match = new RegExp(`${TIME_MARKER} ([\\d.]+) ([\\d.]+) ([\\d.]+)`).exec(stderr);
|
||||
return {
|
||||
maxRssKiB: match ? Number(match[3]) : null,
|
||||
systemSeconds: match ? Number(match[2]) : null,
|
||||
userSeconds: match ? Number(match[1]) : null,
|
||||
wallMs,
|
||||
};
|
||||
}
|
||||
|
||||
async function timedAnalysis(
|
||||
inputPath: string,
|
||||
durationSeconds: number,
|
||||
useTimeBinary: boolean
|
||||
): Promise<{ analysis: VideoStructuralAnalysis; cost: ChildCost }> {
|
||||
let cost: ChildCost = {
|
||||
maxRssKiB: null,
|
||||
systemSeconds: null,
|
||||
userSeconds: null,
|
||||
wallMs: 0,
|
||||
};
|
||||
const runner: VideoCommandRunner = async (executable, args, options) => {
|
||||
const startedAt = performance.now();
|
||||
const command = useTimeBinary ? "/usr/bin/time" : executable;
|
||||
const commandArgs = useTimeBinary
|
||||
? ["-f", `${TIME_MARKER} %U %S %M`, executable, ...args]
|
||||
: [...args];
|
||||
const result = await execFileAsync(command, commandArgs, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1024 * 1024,
|
||||
signal: options.signal,
|
||||
timeout: options.timeoutMs,
|
||||
});
|
||||
cost = parseTimeCost(String(result.stderr), performance.now() - startedAt);
|
||||
return { stderr: String(result.stderr), stdout: String(result.stdout) };
|
||||
};
|
||||
const analysis = await analyzeVideoStructure(inputPath, {
|
||||
durationSeconds,
|
||||
runner,
|
||||
streamIndex: 0,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
return { analysis, cost };
|
||||
}
|
||||
|
||||
function sampling(
|
||||
durationSeconds: number,
|
||||
frameCount: number,
|
||||
analysis: VideoStructuralAnalysis
|
||||
): { structural: number[]; uniform: number[] } {
|
||||
const uniform = calculateSamplingDecision(durationSeconds, frameCount, "uniform").timestamps;
|
||||
const structural = calculateSamplingDecision(
|
||||
durationSeconds,
|
||||
frameCount,
|
||||
"segment_aware",
|
||||
analysis.sceneCandidates,
|
||||
null,
|
||||
analysis
|
||||
).timestamps;
|
||||
return { structural, uniform };
|
||||
}
|
||||
|
||||
async function captionCallsAfterDedup(
|
||||
inputPath: string,
|
||||
outputDirectory: string,
|
||||
samplingPolicy: "segment_aware" | "uniform"
|
||||
): Promise<number> {
|
||||
await mkdir(outputDirectory, { mode: 0o700 });
|
||||
const frames = await extractFramesFromLocalVideo(inputPath, outputDirectory, {
|
||||
durationSeconds: 8,
|
||||
frameCount: 8,
|
||||
samplingPolicy,
|
||||
streamIndex: 0,
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
const bytes = await readBoundedExtractedFrames(frames);
|
||||
const deduplicated = await deduplicateVideoFrames(
|
||||
frames.map((frame, index) => ({
|
||||
dataUri: `data:image/jpeg;base64,${bytes[index].toString("base64")}`,
|
||||
timestampSeconds: frame.timestampSeconds,
|
||||
}))
|
||||
);
|
||||
return deduplicated.frames.length;
|
||||
}
|
||||
|
||||
function result(
|
||||
name: string,
|
||||
cost: ChildCost,
|
||||
analysis: VideoStructuralAnalysis,
|
||||
uniform: number[],
|
||||
structural: number[],
|
||||
oracle: Record<string, boolean | number | string>,
|
||||
captionCallsAvoided = 0
|
||||
): FixtureResult {
|
||||
const booleans = Object.values(oracle).filter(
|
||||
(value): value is boolean => typeof value === "boolean"
|
||||
);
|
||||
return {
|
||||
captionCallsAvoided,
|
||||
childCost: cost,
|
||||
freezeIntervals: analysis.freezeIntervals.length,
|
||||
name,
|
||||
oracle,
|
||||
passed: booleans.every(Boolean),
|
||||
sceneCandidates: analysis.sceneCandidates.length,
|
||||
structuralFrames: structural.length,
|
||||
uniformFrames: uniform.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const version = await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 });
|
||||
const filters = await execFileAsync("ffmpeg", ["-hide_banner", "-filters"], {
|
||||
maxBuffer: 2 * 1024 * 1024,
|
||||
timeout: 5_000,
|
||||
});
|
||||
const missingFilters = REQUIRED_FILTERS.filter(
|
||||
(filter) => !new RegExp(`\\b${filter}\\b`).test(String(filters.stdout))
|
||||
);
|
||||
if (missingFilters.length > 0)
|
||||
throw new Error(`Missing required FFmpeg filters: ${missingFilters.join(", ")}`);
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), "video-fu07-eval-"));
|
||||
const useTimeBinary = await supportsTimeBinary();
|
||||
const results: FixtureResult[] = [];
|
||||
try {
|
||||
const staticPath = join(directory, "static.mp4");
|
||||
await generateStaticFixture(staticPath);
|
||||
const staticRun = await timedAnalysis(staticPath, 8, useTimeBinary);
|
||||
const staticSampling = sampling(8, 8, staticRun.analysis);
|
||||
const uniformCaptionCalls = await captionCallsAfterDedup(
|
||||
staticPath,
|
||||
join(directory, "static-uniform"),
|
||||
"uniform"
|
||||
);
|
||||
const structuralCaptionCalls = await captionCallsAfterDedup(
|
||||
staticPath,
|
||||
join(directory, "static-structural"),
|
||||
"segment_aware"
|
||||
);
|
||||
const staticCaptionCallsAvoided = Math.max(0, uniformCaptionCalls - structuralCaptionCalls);
|
||||
results.push(
|
||||
result(
|
||||
"static-caption-savings",
|
||||
staticRun.cost,
|
||||
staticRun.analysis,
|
||||
staticSampling.uniform,
|
||||
staticSampling.structural,
|
||||
{
|
||||
fullFreezeDetected: staticRun.analysis.freezeIntervals.some(
|
||||
(interval) => interval.startSeconds <= 1 && interval.endSeconds >= 7
|
||||
),
|
||||
oneIncrementalCaptionCallAvoided: staticCaptionCallsAvoided === 1,
|
||||
structuralCaptionCalls,
|
||||
uniformCaptionCalls,
|
||||
},
|
||||
staticCaptionCallsAvoided
|
||||
)
|
||||
);
|
||||
|
||||
const mixedPath = join(directory, "mixed.mp4");
|
||||
await generateMixedFixture(mixedPath);
|
||||
const mixedRun = await timedAnalysis(mixedPath, 10, useTimeBinary);
|
||||
const mixedSampling = sampling(10, 4, mixedRun.analysis);
|
||||
const uniformDense = mixedSampling.uniform.filter((timestamp) => timestamp > 6).length;
|
||||
const structuralDense = mixedSampling.structural.filter((timestamp) => timestamp > 6).length;
|
||||
results.push(
|
||||
result(
|
||||
"dense-budget-quality-oracle",
|
||||
mixedRun.cost,
|
||||
mixedRun.analysis,
|
||||
mixedSampling.uniform,
|
||||
mixedSampling.structural,
|
||||
{
|
||||
denseFramesStructural: structuralDense,
|
||||
denseFramesUniform: uniformDense,
|
||||
denseRegionGetsMoreBudget: structuralDense > uniformDense,
|
||||
frozenRegionRetainsCoverage: mixedSampling.structural.some((timestamp) => timestamp < 6),
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
const qualityPath = join(directory, "blur-exposure.mp4");
|
||||
await generateBlurExposureFixture(qualityPath);
|
||||
const qualityRun = await timedAnalysis(qualityPath, 10, useTimeBinary);
|
||||
const qualitySampling = sampling(10, 6, qualityRun.analysis);
|
||||
const blurred = samplesIn(qualityRun.analysis, 0, 3);
|
||||
const dark = samplesIn(qualityRun.analysis, 3, 6);
|
||||
const sharp = samplesIn(qualityRun.analysis, 6, 10);
|
||||
const blurredBlur = average(blurred.map((sample) => sample.blur));
|
||||
const blurredSpatial = average(blurred.map((sample) => sample.spatialInformation));
|
||||
const darkLuma = average(dark.map((sample) => sample.brightness));
|
||||
const sharpBlur = average(sharp.map((sample) => sample.blur));
|
||||
const sharpSpatial = average(sharp.map((sample) => sample.spatialInformation));
|
||||
const sharpTemporal = average(sharp.map((sample) => sample.temporalInformation));
|
||||
const sharpLuma = average(sharp.map((sample) => sample.brightness));
|
||||
results.push(
|
||||
result(
|
||||
"blur-exposure-spatial-temporal-evidence",
|
||||
qualityRun.cost,
|
||||
qualityRun.analysis,
|
||||
qualitySampling.uniform,
|
||||
qualitySampling.structural,
|
||||
{
|
||||
blurMetricSeparated:
|
||||
blurredBlur !== null && sharpBlur !== null && Math.abs(blurredBlur - sharpBlur) >= 0.05,
|
||||
blurredBlur: blurredBlur ?? "missing",
|
||||
darkLuma: darkLuma ?? "missing",
|
||||
exposureSeparated: darkLuma !== null && sharpLuma !== null && sharpLuma - darkLuma >= 50,
|
||||
sharpBlur: sharpBlur ?? "missing",
|
||||
sharpSpatial: sharpSpatial ?? "missing",
|
||||
sharpTemporal: sharpTemporal ?? "missing",
|
||||
spatialDetailSeparated:
|
||||
blurredSpatial !== null && sharpSpatial !== null && sharpSpatial - blurredSpatial >= 20,
|
||||
structuralKeepsSharpRegion:
|
||||
qualitySampling.structural.filter((timestamp) => timestamp >= 6).length >= 2,
|
||||
temporalChangeDetected: sharpTemporal !== null && sharpTemporal >= 5,
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
const tailPath = join(directory, "dense-tail.mp4");
|
||||
await generateDenseTailFixture(tailPath);
|
||||
const tailRun = await timedAnalysis(tailPath, 10, useTimeBinary);
|
||||
const tailSampling = sampling(10, 4, tailRun.analysis);
|
||||
results.push(
|
||||
result(
|
||||
"dense-cuts-long-tail-regression",
|
||||
tailRun.cost,
|
||||
tailRun.analysis,
|
||||
tailSampling.uniform,
|
||||
tailSampling.structural,
|
||||
{
|
||||
multipleEarlyCuts: tailRun.analysis.sceneCandidates.length >= 3,
|
||||
trailingEightSecondsRepresented: tailSampling.structural.some(
|
||||
(timestamp) => timestamp > 2
|
||||
),
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
const fadePath = join(directory, "gradual-fade.mp4");
|
||||
await generateGradualFadeFixture(fadePath);
|
||||
const fadeRun = await timedAnalysis(fadePath, 8, useTimeBinary);
|
||||
const fadeSampling = sampling(8, 4, fadeRun.analysis);
|
||||
results.push(
|
||||
result(
|
||||
"gradual-fade-false-positive",
|
||||
fadeRun.cost,
|
||||
fadeRun.analysis,
|
||||
fadeSampling.uniform,
|
||||
fadeSampling.structural,
|
||||
{
|
||||
hardCutFalsePositives: fadeRun.analysis.sceneCandidates.length,
|
||||
noHardCutBurst: fadeRun.analysis.sceneCandidates.length <= 1,
|
||||
noCaptionBudgetPruning: fadeSampling.structural.length === fadeSampling.uniform.length,
|
||||
}
|
||||
)
|
||||
);
|
||||
} finally {
|
||||
await rm(directory, { force: true, recursive: true });
|
||||
}
|
||||
|
||||
const callsAvoided = results.reduce((sum, fixture) => sum + fixture.captionCallsAvoided, 0);
|
||||
const costFlag = process.argv.indexOf("--caption-cost-per-call-usd");
|
||||
const explicitCost = Number(costFlag >= 0 ? process.argv[costFlag + 1] : Number.NaN);
|
||||
const report = {
|
||||
captionCost:
|
||||
Number.isFinite(explicitCost) && explicitCost > 0
|
||||
? {
|
||||
estimatedUsdAvoided: callsAvoided * explicitCost,
|
||||
source: "explicit environment input",
|
||||
status: "ESTIMATED_FROM_INPUT",
|
||||
}
|
||||
: {
|
||||
reason: "--caption-cost-per-call-usd was not supplied with a positive number",
|
||||
status: "HOLD",
|
||||
},
|
||||
ffmpegVersion: String(version.stdout).split("\n")[0],
|
||||
fixtures: results,
|
||||
modelQuality: {
|
||||
reason:
|
||||
"No authorized real caption-model endpoint, credentials, or frozen judge rubric were configured; deterministic structural oracles are not semantic quality.",
|
||||
status: "HOLD",
|
||||
},
|
||||
gainCostComparison: {
|
||||
reason:
|
||||
"The real post-dedup caption-call delta is measured, but no authorized caption latency/cost receipt or child CPU/RSS receipt is configured.",
|
||||
status: "HOLD",
|
||||
},
|
||||
resourceCost: useTimeBinary
|
||||
? { source: "/usr/bin/time", status: "MEASURED" }
|
||||
: {
|
||||
reason: "/usr/bin/time is unavailable; wall time is measured but child CPU/RSS are not",
|
||||
status: "HOLD",
|
||||
},
|
||||
summary: {
|
||||
captionCallsAvoided: callsAvoided,
|
||||
failed: results.filter((fixture) => !fixture.passed).map((fixture) => fixture.name),
|
||||
passed: results.filter((fixture) => fixture.passed).length,
|
||||
total: results.length,
|
||||
},
|
||||
timeBinary: useTimeBinary ? "/usr/bin/time" : null,
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (report.summary.failed.length > 0) process.exitCode = 1;
|
||||
}
|
||||
|
||||
await main();
|
||||
Reference in New Issue
Block a user