diff --git a/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md b/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md new file mode 100644 index 0000000000..5a4f4b2fba --- /dev/null +++ b/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md @@ -0,0 +1 @@ +- **fix(video-bridge):** burn high-contrast timestamps into every bounded contact-sheet cell and add a real-model A/B harness whose promotion verdict stays `HOLD` until token, latency, and quality evidence is actually executed ([#11350](https://github.com/diegosouzapw/OmniRoute/pull/11350)) diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index 28cd0a4e38..907679a4b3 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -356,10 +356,23 @@ coverage. Output metadata separates extracted candidates, successfully used frames, and visual duplicates dropped. An explicitly marked video part may request a timestamped contact sheet. The -bridge builds at most a 4-column, 16-frame JPEG grid and labels the resulting -observation with every source timestamp. If `sharp` cannot decode or compose -the grid, the bridge falls back to the individual JPEG frames; a client abort -still propagates through the sheet operation. +bridge builds at most a 4-column, 16-frame JPEG grid. Every 512-pixel cell burns +its source timestamp into a high-contrast bottom band, while the same timestamps +remain in textual metadata for downstream association and audit. The complete +JPEG remains capped at 32 MiB. If `sharp` cannot decode or compose the grid, the +bridge falls back to the individual JPEG frames; a client abort still propagates +through the sheet operation. + +Promotion evidence is deliberately separate from the synthetic composition +microbenchmark. `scripts/perf/video-bridge-contact-sheet-eval.ts` defines a +schema-versioned A/B harness for real OpenAI-compatible vision models. It measures +provider-reported tokens, end-to-end wall latency (including sheet composition), +model-call count, and manifest-defined fact retention. Raw model responses are not +written to the report; only SHA-256 digests and matched fact IDs are retained. The +harness makes no network or paid model call unless `--execute-real` is passed and +`--model`, `OMNIROUTE_BASE_URL`, and `OMNIROUTE_API_KEY` are configured. Without +that explicit real run, its machine-readable verdict remains `HOLD`; synthetic +payload/call-count measurements alone are not promotion evidence. Callers may attach an optional `transcript.cues` array to a supported video part when they already possess aligned text. Each cue must carry `text`, a diff --git a/scripts/perf/video-bridge-bench.ts b/scripts/perf/video-bridge-bench.ts index 66c5fd7391..2b457d3b23 100644 --- a/scripts/perf/video-bridge-bench.ts +++ b/scripts/perf/video-bridge-bench.ts @@ -10,8 +10,10 @@ * scene_aware vs segment_aware for growing scene-candidate counts. The * ffmpeg scene-detection pass is shared by both aware policies and is * I/O-bound, so the incremental policy cost is exactly this selection step. - * 3. Contact sheet: composes synthetic JPEG frames into the timestamped grid - * and compares payload bytes + model calls against individual frames. + * 3. Contact sheet: composes synthetic JPEG frames into the visually timestamped + * grid and compares payload bytes + structural call counts. This microbenchmark + * does not measure real-model tokens, latency, or quality; use + * video-bridge-contact-sheet-eval.ts before considering promotion. */ import { performance } from "node:perf_hooks"; @@ -118,6 +120,9 @@ async function syntheticJpegFrame(index: number, width = 512, height = 288): Pro async function benchContactSheet(): Promise { console.log("\n== Contact sheet vs individual frames (synthetic 512x288 JPEG) =="); + console.log( + "STRUCTURAL ONLY: real-model tokens/latency/quality are unmeasured; promotion remains HOLD." + ); console.log("frames | sheet_ms sheet_KiB individual_KiB model_calls(sheet/individual)"); for (const frameCount of [1, 4, 8, 16]) { const frames = await Promise.all( diff --git a/scripts/perf/video-bridge-contact-sheet-eval.ts b/scripts/perf/video-bridge-contact-sheet-eval.ts new file mode 100644 index 0000000000..7020a5368c --- /dev/null +++ b/scripts/perf/video-bridge-contact-sheet-eval.ts @@ -0,0 +1,578 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; + +import { z } from "zod"; + +import { + buildVideoContactSheet, + type ContactSheetFrame, +} from "../../src/lib/guardrails/videoBridgeContactSheet"; + +export type VideoContactSheetEvalConfigurationState = "configured-not-executed" | "not-configured"; + +export interface VideoContactSheetEvalHoldReportInput { + caseCount: number; + configurationState: VideoContactSheetEvalConfigurationState; + missingConfiguration?: string[]; +} + +export interface VideoContactSheetEvalHoldReport { + caseCount: number; + execution: { + realModel: false; + state: VideoContactSheetEvalConfigurationState; + }; + kind: "video-contact-sheet-ab-eval"; + missingConfiguration: string[]; + promotion: { + reasons: ["REAL_MODEL_CONFIGURATION_MISSING" | "REAL_MODEL_EVAL_NOT_EXECUTED"]; + status: "HOLD"; + }; + results: []; + schemaVersion: 1; + summary: null; +} + +export interface VideoContactSheetEvalThresholds { + minLatencyReductionRatio: number; + minQualityRetention: number; + minQualityScore: number; + minTokenReductionRatio: number; +} + +export interface VideoContactSheetEvalAggregate { + latencyMs: number; + qualityScore: number; + totalTokens: number | null; +} + +export type VideoContactSheetPromotionReason = + | "LATENCY_REDUCTION_BELOW_THRESHOLD" + | "QUALITY_RETENTION_BELOW_THRESHOLD" + | "QUALITY_SCORE_BELOW_THRESHOLD" + | "TOKEN_REDUCTION_BELOW_THRESHOLD" + | "TOKEN_USAGE_UNAVAILABLE"; + +export interface VideoContactSheetPromotionDecision { + metrics: { + latencyReductionRatio: number; + qualityRetention: number; + tokenReductionRatio: number | null; + }; + reasons: VideoContactSheetPromotionReason[]; + status: "ELIGIBLE" | "HOLD"; +} + +const MAX_EVAL_FRAME_BASE64_CHARS = 5_592_408; + +const evalThresholdsSchema = z + .object({ + minLatencyReductionRatio: z.number().positive().max(1), + minQualityRetention: z.number().min(0).max(1), + minQualityScore: z.number().min(0).max(1), + minTokenReductionRatio: z.number().positive().max(1), + }) + .strict(); + +const evalManifestSchema = z + .object({ + cases: z + .array( + z + .object({ + expectedFacts: z + .array( + z + .object({ + id: z.string().min(1), + requiredTerms: z.array(z.string().min(1)).min(1), + timestampSeconds: z.number().finite().nonnegative(), + }) + .strict() + ) + .min(1), + frames: z + .array( + z + .object({ + dataUri: z + .string() + .max("data:image/jpeg;base64,".length + MAX_EVAL_FRAME_BASE64_CHARS) + .regex( + /^data:image\/jpeg;base64,[A-Za-z0-9+/=]{4,5592408}$/i, + "expected a bounded JPEG data URI" + ), + timestampSeconds: z.number().finite().nonnegative(), + }) + .strict() + ) + .min(1) + .max(16), + id: z.string().min(1), + prompt: z.string().min(1), + }) + .strict() + ) + .min(1), + id: z.string().min(1), + schemaVersion: z.literal(1), + thresholds: evalThresholdsSchema, + }) + .strict(); + +const chatCompletionSchema = z + .object({ + choices: z + .array( + z + .object({ + message: z.object({ content: z.string() }).passthrough(), + }) + .passthrough() + ) + .min(1), + usage: z + .object({ + completion_tokens: z.number().nonnegative().optional(), + prompt_tokens: z.number().nonnegative().optional(), + total_tokens: z.number().nonnegative().optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +export type VideoContactSheetEvalManifest = z.infer; + +export interface VideoContactSheetEvalConfig { + apiKey: string; + endpoint: string; + model: string; +} + +interface EvalFactScore { + matchedFactIds: string[]; + qualityScore: number; +} + +interface EvalPathResult extends EvalFactScore { + latencyMs: number; + modelCalls: number; + responseDigest: string; + totalTokens: number | null; +} + +export interface VideoContactSheetEvalCaseResult { + caseId: string; + individual: EvalPathResult; + sheet: EvalPathResult; +} + +export interface VideoContactSheetEvalExecutedReport { + caseCount: number; + execution: { + realModel: true; + state: "executed"; + }; + generatedAt: string; + kind: "video-contact-sheet-ab-eval"; + manifestDigest: string; + manifestId: string; + model: string; + promotion: VideoContactSheetPromotionDecision; + results: VideoContactSheetEvalCaseResult[]; + schemaVersion: 1; + summary: { + individual: VideoContactSheetEvalAggregate & { modelCalls: number }; + sheet: VideoContactSheetEvalAggregate & { modelCalls: number }; + }; + thresholds: VideoContactSheetEvalThresholds; +} + +type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise; + +export function createVideoContactSheetEvalHoldReport( + input: VideoContactSheetEvalHoldReportInput +): VideoContactSheetEvalHoldReport { + const reason = + input.configurationState === "not-configured" + ? "REAL_MODEL_CONFIGURATION_MISSING" + : "REAL_MODEL_EVAL_NOT_EXECUTED"; + return { + caseCount: input.caseCount, + execution: { + realModel: false, + state: input.configurationState, + }, + kind: "video-contact-sheet-ab-eval", + missingConfiguration: [...(input.missingConfiguration ?? [])], + promotion: { + reasons: [reason], + status: "HOLD", + }, + results: [], + schemaVersion: 1, + summary: null, + }; +} + +function reductionRatio(baseline: number, candidate: number): number { + if (baseline <= 0) return 0; + return (baseline - candidate) / baseline; +} + +export function assessVideoContactSheetPromotion(input: { + individual: VideoContactSheetEvalAggregate; + sheet: VideoContactSheetEvalAggregate; + thresholds: VideoContactSheetEvalThresholds; +}): VideoContactSheetPromotionDecision { + const latencyReductionRatio = reductionRatio(input.individual.latencyMs, input.sheet.latencyMs); + const qualityRetention = + input.individual.qualityScore > 0 + ? input.sheet.qualityScore / input.individual.qualityScore + : 0; + const tokenReductionRatio = + input.individual.totalTokens === null || input.sheet.totalTokens === null + ? null + : reductionRatio(input.individual.totalTokens, input.sheet.totalTokens); + const reasons: VideoContactSheetPromotionReason[] = []; + const requiredLatencyReduction = Math.max( + Number.EPSILON, + input.thresholds.minLatencyReductionRatio + ); + const requiredTokenReduction = Math.max(Number.EPSILON, input.thresholds.minTokenReductionRatio); + if (latencyReductionRatio < requiredLatencyReduction) { + reasons.push("LATENCY_REDUCTION_BELOW_THRESHOLD"); + } + if (input.sheet.qualityScore < input.thresholds.minQualityScore) { + reasons.push("QUALITY_SCORE_BELOW_THRESHOLD"); + } + if (qualityRetention < input.thresholds.minQualityRetention) { + reasons.push("QUALITY_RETENTION_BELOW_THRESHOLD"); + } + if (tokenReductionRatio === null) { + reasons.push("TOKEN_USAGE_UNAVAILABLE"); + } else if (tokenReductionRatio < requiredTokenReduction) { + reasons.push("TOKEN_REDUCTION_BELOW_THRESHOLD"); + } + return { + metrics: { + latencyReductionRatio, + qualityRetention, + tokenReductionRatio, + }, + reasons, + status: reasons.length === 0 ? "ELIGIBLE" : "HOLD", + }; +} + +function normalizeEvalText(value: string): string { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase(); +} + +function formatEvalTimestamp(timestampSeconds: number): string { + const totalMilliseconds = Math.max(0, Math.round(timestampSeconds * 1000)); + const minutes = Math.floor(totalMilliseconds / 60_000); + const seconds = Math.floor((totalMilliseconds % 60_000) / 1000); + const milliseconds = totalMilliseconds % 1000; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`; +} + +function scoreFacts( + response: string, + expectedFacts: VideoContactSheetEvalManifest["cases"][number]["expectedFacts"] +): EvalFactScore { + const normalizedResponse = normalizeEvalText(response); + const matchedFactIds = expectedFacts + .filter((fact) => { + const timestamp = normalizeEvalText(formatEvalTimestamp(fact.timestampSeconds)); + const timestampIndex = normalizedResponse.indexOf(timestamp); + if (timestampIndex < 0) return false; + const factWindow = normalizedResponse.slice( + Math.max(0, timestampIndex - 160), + Math.min(normalizedResponse.length, timestampIndex + timestamp.length + 160) + ); + return fact.requiredTerms.every((term) => factWindow.includes(normalizeEvalText(term))); + }) + .map((fact) => fact.id); + return { + matchedFactIds, + qualityScore: matchedFactIds.length / expectedFacts.length, + }; +} + +function digestResponse(response: string): string { + return createHash("sha256").update(response).digest("hex"); +} + +function sumTokens(values: Array): number | null { + if (values.some((value) => value === null)) return null; + return values.reduce((sum, value) => sum + (value ?? 0), 0); +} + +async function callVisionModel(input: { + config: VideoContactSheetEvalConfig; + dataUri: string; + fetchImpl: FetchLike; + prompt: string; +}): Promise<{ content: string; totalTokens: number | null }> { + const response = await input.fetchImpl(input.config.endpoint, { + body: JSON.stringify({ + messages: [ + { + content: [ + { text: input.prompt, type: "text" }, + { image_url: { url: input.dataUri }, type: "image_url" }, + ], + role: "user", + }, + ], + model: input.config.model, + temperature: 0, + }), + headers: { + authorization: `Bearer ${input.config.apiKey}`, + "content-type": "application/json", + }, + method: "POST", + }); + if (!response.ok) { + throw new Error(`Video contact-sheet eval request failed with HTTP ${response.status}`); + } + const parsed = chatCompletionSchema.parse(await response.json()); + const usage = parsed.usage; + const totalTokens = + usage?.total_tokens ?? + (usage?.prompt_tokens !== undefined && usage.completion_tokens !== undefined + ? usage.prompt_tokens + usage.completion_tokens + : null); + return { + content: parsed.choices[0].message.content, + totalTokens, + }; +} + +async function evaluateIndividualFrames(input: { + evalCase: VideoContactSheetEvalManifest["cases"][number]; + config: VideoContactSheetEvalConfig; + fetchImpl: FetchLike; +}): Promise { + const startedAt = performance.now(); + const calls: Array<{ content: string; totalTokens: number | null }> = []; + for (const frame of input.evalCase.frames) { + calls.push( + await callVisionModel({ + config: input.config, + dataUri: frame.dataUri, + fetchImpl: input.fetchImpl, + prompt: `${input.evalCase.prompt}\nAnalyze only the frame at ${formatEvalTimestamp(frame.timestampSeconds)}. Associate every observation with that exact timestamp label.`, + }) + ); + } + const content = calls.map((call) => call.content).join("\n"); + return { + ...scoreFacts(content, input.evalCase.expectedFacts), + latencyMs: performance.now() - startedAt, + modelCalls: calls.length, + responseDigest: digestResponse(content), + totalTokens: sumTokens(calls.map((call) => call.totalTokens)), + }; +} + +async function evaluateContactSheet(input: { + evalCase: VideoContactSheetEvalManifest["cases"][number]; + config: VideoContactSheetEvalConfig; + fetchImpl: FetchLike; +}): Promise { + const startedAt = performance.now(); + const sheet = await buildVideoContactSheet(input.evalCase.frames as ContactSheetFrame[], { + columns: 4, + timeoutMs: 30_000, + }); + if (!sheet.used || !sheet.dataUri) { + throw new Error("Video contact-sheet eval could not compose the bounded JPEG grid"); + } + const call = await callVisionModel({ + config: input.config, + dataUri: sheet.dataUri, + fetchImpl: input.fetchImpl, + prompt: `${input.evalCase.prompt}\nAnalyze every cell in the contact sheet. Timestamp labels are burned into each cell. Associate every observation with its visible timestamp.`, + }); + return { + ...scoreFacts(call.content, input.evalCase.expectedFacts), + latencyMs: performance.now() - startedAt, + modelCalls: 1, + responseDigest: digestResponse(call.content), + totalTokens: call.totalTokens, + }; +} + +function aggregatePathResults( + results: VideoContactSheetEvalCaseResult[], + path: "individual" | "sheet" +): VideoContactSheetEvalAggregate & { modelCalls: number } { + const pathResults = results.map((result) => result[path]); + return { + latencyMs: pathResults.reduce((sum, result) => sum + result.latencyMs, 0), + modelCalls: pathResults.reduce((sum, result) => sum + result.modelCalls, 0), + qualityScore: + pathResults.reduce((sum, result) => sum + result.qualityScore, 0) / pathResults.length, + totalTokens: sumTokens(pathResults.map((result) => result.totalTokens)), + }; +} + +export async function runVideoContactSheetEval(input: { + config: VideoContactSheetEvalConfig; + fetchImpl?: FetchLike; + manifest: VideoContactSheetEvalManifest; +}): Promise { + const manifest = evalManifestSchema.parse(input.manifest); + const endpoint = z.string().url().parse(input.config.endpoint); + const config = { + apiKey: z.string().min(1).parse(input.config.apiKey), + endpoint, + model: z.string().min(1).parse(input.config.model), + }; + const fetchImpl = input.fetchImpl ?? fetch; + const results: VideoContactSheetEvalCaseResult[] = []; + for (const evalCase of manifest.cases) { + const individual = await evaluateIndividualFrames({ config, evalCase, fetchImpl }); + const sheet = await evaluateContactSheet({ config, evalCase, fetchImpl }); + results.push({ caseId: evalCase.id, individual, sheet }); + } + const individual = aggregatePathResults(results, "individual"); + const sheet = aggregatePathResults(results, "sheet"); + const promotion = assessVideoContactSheetPromotion({ + individual, + sheet, + thresholds: manifest.thresholds, + }); + return { + caseCount: manifest.cases.length, + execution: { realModel: true, state: "executed" }, + generatedAt: new Date().toISOString(), + kind: "video-contact-sheet-ab-eval", + manifestDigest: createHash("sha256").update(JSON.stringify(manifest)).digest("hex"), + manifestId: manifest.id, + model: config.model, + promotion, + results, + schemaVersion: 1, + summary: { individual, sheet }, + thresholds: manifest.thresholds, + }; +} + +function readArgument(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function printUsage(): void { + console.log( + [ + "Usage:", + " node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest --model ", + " node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest --model --execute-real", + "", + "The default command validates configuration and emits HOLD without calling a model.", + "A real paid/networked run requires --execute-real, --model, and the documented variables:", + " OMNIROUTE_BASE_URL", + " OMNIROUTE_API_KEY", + "", + "Manifest v1: id, thresholds, and 1+ cases. Each case has 1-16 bounded JPEG data URIs,", + "timestamps, a prompt, and expectedFacts with timestampSeconds + requiredTerms.", + ].join("\n") + ); +} + +async function loadManifest(manifestPath: string): Promise { + const raw = await readFile(path.resolve(manifestPath), "utf8"); + return evalManifestSchema.parse(JSON.parse(raw)); +} + +function resolveChatCompletionsEndpoint(baseUrl: string): string { + const normalized = baseUrl.replace(/\/{1,8}$/u, ""); + if (normalized.endsWith("/v1/chat/completions")) return normalized; + if (normalized.endsWith("/v1")) return `${normalized}/chat/completions`; + return `${normalized}/v1/chat/completions`; +} + +async function main(): Promise { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + printUsage(); + return; + } + const manifestPath = readArgument("manifest"); + const model = readArgument("model"); + const missingConfiguration: string[] = []; + if (!manifestPath) missingConfiguration.push("--manifest"); + if (!model) missingConfiguration.push("--model"); + const baseUrl = process.env.OMNIROUTE_BASE_URL; + const apiKey = process.env.OMNIROUTE_API_KEY; + if (!baseUrl) missingConfiguration.push("OMNIROUTE_BASE_URL"); + if (!apiKey) missingConfiguration.push("OMNIROUTE_API_KEY"); + + let manifest: VideoContactSheetEvalManifest | null = null; + if (manifestPath) manifest = await loadManifest(manifestPath); + if (missingConfiguration.length > 0) { + console.log( + JSON.stringify( + createVideoContactSheetEvalHoldReport({ + caseCount: manifest?.cases.length ?? 0, + configurationState: "not-configured", + missingConfiguration, + }), + null, + 2 + ) + ); + return; + } + if (!process.argv.includes("--execute-real")) { + console.log( + JSON.stringify( + createVideoContactSheetEvalHoldReport({ + caseCount: manifest?.cases.length ?? 0, + configurationState: "configured-not-executed", + }), + null, + 2 + ) + ); + return; + } + if (!manifest || !baseUrl || !apiKey || !model) { + throw new Error("Video contact-sheet eval configuration was not resolved"); + } + console.log( + JSON.stringify( + await runVideoContactSheetEval({ + config: { apiKey, endpoint: resolveChatCompletionsEndpoint(baseUrl), model }, + manifest, + }), + null, + 2 + ) + ); +} + +const isMainModule = + typeof process.argv[1] === "string" && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isMainModule) { + main().catch(() => { + console.error("Video contact-sheet eval failed validation or execution."); + process.exitCode = 1; + }); +} diff --git a/src/lib/guardrails/videoBridgeContactSheet.ts b/src/lib/guardrails/videoBridgeContactSheet.ts index fc17627553..a0f1d0f1e8 100644 --- a/src/lib/guardrails/videoBridgeContactSheet.ts +++ b/src/lib/guardrails/videoBridgeContactSheet.ts @@ -21,6 +21,9 @@ export interface VideoContactSheetResult { const MAX_FRAMES = 16; const MAX_SHEET_BYTES = 32 * 1024 * 1024; +const LABEL_FONT_SIZE = 32; +const LABEL_HEIGHT = 64; +const LABEL_PADDING = 16; const TILE_SIZE = 512; function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult { @@ -33,11 +36,31 @@ function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult } function decodeFrame(dataUri: string): Buffer { - const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri); + const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]{4,5592408})$/i.exec(dataUri); if (!match) throw new Error("Contact sheet requires JPEG data URIs"); return Buffer.from(match[1], "base64"); } +function formatContactSheetTimestamp(timestampSeconds: number): string { + const totalMilliseconds = Math.max(0, Math.round(timestampSeconds * 1000)); + const minutes = Math.floor(totalMilliseconds / 60_000); + const seconds = Math.floor((totalMilliseconds % 60_000) / 1000); + const milliseconds = totalMilliseconds % 1000; + if (minutes > 999) return `t=${timestampSeconds.toExponential(3)}s`; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`; +} + +function buildTimestampLabel(timestampSeconds: number): Buffer { + const label = formatContactSheetTimestamp(timestampSeconds); + const labelTop = TILE_SIZE - LABEL_HEIGHT; + return Buffer.from( + ` + + ${label} + ` + ); +} + /** Build an optional bounded JPEG grid; every failure except abort is fail-safe to individual frames. */ export async function buildVideoContactSheet( frames: readonly ContactSheetFrame[], @@ -69,6 +92,7 @@ export async function buildVideoContactSheet( frames.map(async (frame) => sharp(decodeFrame(frame.dataUri)) .resize(TILE_SIZE, TILE_SIZE, { fit: "contain", background: "#000000" }) + .composite([{ input: buildTimestampLabel(frame.timestampSeconds), left: 0, top: 0 }]) .jpeg({ quality: 80 }) .toBuffer() ) @@ -101,7 +125,7 @@ export async function buildVideoContactSheet( used: true, width: columns * TILE_SIZE, }; - } catch (error) { + } catch { if (signal.aborted) throw new Error("Video contact sheet was aborted"); return fallback(frames); } finally { diff --git a/tests/unit/guardrails/videoBridgeContactSheet.test.ts b/tests/unit/guardrails/videoBridgeContactSheet.test.ts index 578baccefb..b8c41c83a0 100644 --- a/tests/unit/guardrails/videoBridgeContactSheet.test.ts +++ b/tests/unit/guardrails/videoBridgeContactSheet.test.ts @@ -15,6 +15,12 @@ async function frame(color: string, timestampSeconds: number) { return { dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, timestampSeconds }; } +function decodeJpegDataUri(dataUri: string): Buffer { + const prefix = "data:image/jpeg;base64,"; + assert.ok(dataUri.toLowerCase().startsWith(prefix), "expected a JPEG data URI"); + return Buffer.from(dataUri.slice(prefix.length), "base64"); +} + test("builds a bounded contact sheet and preserves timestamp labels", async () => { const result = await buildVideoContactSheet([ await frame("red", 1), @@ -28,6 +34,61 @@ test("builds a bounded contact sheet and preserves timestamp labels", async () = assert.equal(result.frames.length, 3); }); +test("renders a high-contrast timestamp label inside every contact-sheet cell", async () => { + const result = await buildVideoContactSheet([ + await frame("white", 1), + await frame("white", 65.25), + await frame("white", 130.5), + await frame("white", 600), + ]); + + assert.equal(result.used, true); + assert.equal(result.width, 1024); + assert.equal(result.height, 1024); + const { data, info } = await sharp(decodeJpegDataUri(result.dataUri ?? "")) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + assert.equal(info.channels, 3); + + const tileSize = 512; + const labelTop = 448; + const labelBottom = 512; + const labelFingerprints: string[] = []; + for (let index = 0; index < 4; index++) { + const tileLeft = (index % 2) * tileSize; + const tileTop = Math.floor(index / 2) * tileSize; + let darkPixels = 0; + let lightPixels = 0; + let contentLightPixels = 0; + const labelBytes: number[] = []; + + for (let y = labelTop; y < labelBottom; y++) { + for (let x = 0; x < tileSize; x++) { + const offset = ((tileTop + y) * info.width + tileLeft + x) * info.channels; + const luminance = (data[offset] + data[offset + 1] + data[offset + 2]) / 3; + if (luminance < 48) darkPixels += 1; + if (luminance > 208) lightPixels += 1; + labelBytes.push(Math.round(luminance)); + } + } + for (let y = 128; y < 384; y++) { + for (let x = 64; x < 448; x++) { + const offset = ((tileTop + y) * info.width + tileLeft + x) * info.channels; + const luminance = (data[offset] + data[offset + 1] + data[offset + 2]) / 3; + if (luminance > 208) contentLightPixels += 1; + } + } + + assert.ok(darkPixels > tileSize * 48, `cell ${index} should have a dark label band`); + assert.ok(lightPixels > 40, `cell ${index} should have light timestamp glyphs`); + assert.ok(contentLightPixels > 90_000, `cell ${index} should preserve visible frame content`); + labelFingerprints.push(Buffer.from(labelBytes).toString("base64")); + } + + assert.equal(new Set(labelFingerprints).size, 4, "each timestamp should render a distinct label"); +}); + test("contact sheet falls back to individual frames when decoding fails", async () => { const frames = [{ dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 2 }]; const result = await buildVideoContactSheet(frames); diff --git a/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts b/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts new file mode 100644 index 0000000000..9872df41e3 --- /dev/null +++ b/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts @@ -0,0 +1,177 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import sharp from "sharp"; + +import { + assessVideoContactSheetPromotion, + createVideoContactSheetEvalHoldReport, + runVideoContactSheetEval, +} from "../../../scripts/perf/video-bridge-contact-sheet-eval.ts"; + +async function evalFrame(color: string, timestampSeconds: number) { + const bytes = await sharp({ + create: { background: color, channels: 3, height: 32, width: 32 }, + }) + .jpeg() + .toBuffer(); + return { + dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, + timestampSeconds, + }; +} + +test("contact-sheet A/B eval remains HOLD when real-model configuration is missing", () => { + const report = createVideoContactSheetEvalHoldReport({ + caseCount: 0, + configurationState: "not-configured", + missingConfiguration: ["OMNIROUTE_API_KEY", "--model"], + }); + + assert.equal(report.schemaVersion, 1); + assert.equal(report.kind, "video-contact-sheet-ab-eval"); + assert.deepEqual(report.execution, { + realModel: false, + state: "not-configured", + }); + assert.deepEqual(report.promotion, { + reasons: ["REAL_MODEL_CONFIGURATION_MISSING"], + status: "HOLD", + }); + assert.deepEqual(report.missingConfiguration, ["OMNIROUTE_API_KEY", "--model"]); + assert.deepEqual(report.results, []); + assert.equal(report.summary, null); +}); + +test("contact-sheet A/B eval becomes eligible only with measured cost gains and retained quality", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 0.9, totalTokens: 1_000 }, + sheet: { latencyMs: 600, qualityScore: 0.9, totalTokens: 600 }, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 1, + minQualityScore: 0.8, + minTokenReductionRatio: 0.01, + }, + }); + + assert.deepEqual(decision, { + metrics: { + latencyReductionRatio: 0.4, + qualityRetention: 1, + tokenReductionRatio: 0.4, + }, + reasons: [], + status: "ELIGIBLE", + }); +}); + +test("contact-sheet A/B promotion remains HOLD for quality loss or absent token evidence", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + sheet: { latencyMs: 500, qualityScore: 0.7, totalTokens: null }, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 0.95, + minQualityScore: 0.8, + minTokenReductionRatio: 0.01, + }, + }); + + assert.equal(decision.status, "HOLD"); + assert.deepEqual(decision.reasons, [ + "QUALITY_SCORE_BELOW_THRESHOLD", + "QUALITY_RETENTION_BELOW_THRESHOLD", + "TOKEN_USAGE_UNAVAILABLE", + ]); + assert.equal(decision.metrics.tokenReductionRatio, null); +}); + +test("contact-sheet A/B promotion rejects zero cost gain even with permissive thresholds", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + sheet: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + thresholds: { + minLatencyReductionRatio: 0, + minQualityRetention: 1, + minQualityScore: 1, + minTokenReductionRatio: 0, + }, + }); + + assert.equal(decision.status, "HOLD"); + assert.deepEqual(decision.reasons, [ + "LATENCY_REDUCTION_BELOW_THRESHOLD", + "TOKEN_REDUCTION_BELOW_THRESHOLD", + ]); +}); + +test("contact-sheet A/B harness measures real-model calls without storing raw responses", async () => { + const responses = [ + "At 00:01.000 there is a red square.", + "At 00:05.000 there is a blue circle.", + "At 00:01.000 there is a red square; at 00:05.000 there is a blue circle.", + ]; + let requestCount = 0; + const report = await runVideoContactSheetEval({ + config: { + apiKey: "test-only-key", + endpoint: "https://eval.invalid/v1/chat/completions", + model: "vision-eval-model", + }, + fetchImpl: async () => { + const content = responses[requestCount]; + requestCount += 1; + return new Response( + JSON.stringify({ + choices: [{ message: { content } }], + usage: { completion_tokens: 20, prompt_tokens: 80, total_tokens: 100 }, + }), + { headers: { "content-type": "application/json" }, status: 200 } + ); + }, + manifest: { + cases: [ + { + expectedFacts: [ + { + id: "red-square", + requiredTerms: ["red", "square"], + timestampSeconds: 1, + }, + { + id: "blue-circle", + requiredTerms: ["blue", "circle"], + timestampSeconds: 5, + }, + ], + frames: [await evalFrame("red", 1), await evalFrame("blue", 5)], + id: "two-scenes", + prompt: "Describe the visible shape and color at each timestamp.", + }, + ], + id: "contact-sheet-fixture-v1", + schemaVersion: 1, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 1, + minQualityScore: 1, + minTokenReductionRatio: 0.01, + }, + }, + }); + + assert.equal(requestCount, 3); + assert.deepEqual(report.execution, { realModel: true, state: "executed" }); + assert.equal(report.results[0].individual.modelCalls, 2); + assert.equal(report.results[0].individual.totalTokens, 200); + assert.equal(report.results[0].individual.qualityScore, 1); + assert.equal(report.results[0].sheet.modelCalls, 1); + assert.equal(report.results[0].sheet.totalTokens, 100); + assert.equal(report.results[0].sheet.qualityScore, 1); + assert.equal("response" in report.results[0].individual, false); + assert.equal("response" in report.results[0].sheet, false); + assert.match(report.manifestDigest, /^[a-f0-9]{64}$/); + assert.match(report.results[0].individual.responseDigest, /^[a-f0-9]{64}$/); + assert.match(report.results[0].sheet.responseDigest, /^[a-f0-9]{64}$/); +});