diff --git a/changelog.d/features/11369-video-bridge-drilldown-isolation.md b/changelog.d/features/11369-video-bridge-drilldown-isolation.md new file mode 100644 index 0000000000..b0b499304d --- /dev/null +++ b/changelog.d/features/11369-video-bridge-drilldown-isolation.md @@ -0,0 +1 @@ +- **feat(video bridge):** harden the optional drill-down cache substrate with exact-path broker policy, canonical principal/session/media isolation, independent retained-byte quotas, cancellation-safe commits, rejection of excess or non-canonical Base64 padding and non-JPEG/truncated media, warning-sensitive full JPEG canonicalization that strips trailing polyglot bytes, server-derived dimensions, and auditable derivation metadata; production tenant binding and multi-resolution selection remain follow-up work ([#11369](https://github.com/diegosouzapw/OmniRoute/pull/11369)) diff --git a/changelog.d/fixes/11344-video-bridge-scene-aware-sampler.md b/changelog.d/fixes/11344-video-bridge-scene-aware-sampler.md new file mode 100644 index 0000000000..e33e40fab7 --- /dev/null +++ b/changelog.d/fixes/11344-video-bridge-scene-aware-sampler.md @@ -0,0 +1 @@ +- **fix(video-bridge):** fall back to the deterministic active-window midpoint when a one-frame scene-aware budget cannot preserve both timeline ends; a real FFmpeg fixture matrix now covers rapid cuts, gradual changes, static and short clips, and detector failure ([#11344](https://github.com/diegosouzapw/OmniRoute/pull/11344)). diff --git a/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md b/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md new file mode 100644 index 0000000000..5a4f4b2fba --- /dev/null +++ b/changelog.d/fixes/11350-video-bridge-contact-sheet-labels.md @@ -0,0 +1 @@ +- **fix(video-bridge):** burn high-contrast timestamps into every bounded contact-sheet cell and add a real-model A/B harness whose promotion verdict stays `HOLD` until token, latency, and quality evidence is actually executed ([#11350](https://github.com/diegosouzapw/OmniRoute/pull/11350)) diff --git a/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md b/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md new file mode 100644 index 0000000000..17a48aa416 --- /dev/null +++ b/changelog.d/maintenance/11381-video-bridge-fu07-structural-sampling.md @@ -0,0 +1 @@ +- **fix(video-bridge):** make opt-in segment-aware sampling use one bounded structural FFmpeg pass (scene, freeze, blur, exposure, and SI/TI), preserve long trailing segments, fail open to uniform sampling, and add real-media structural-oracle, overhead, post-dedup caption-call, and false-positive evidence while holding unconfigured model quality and gain-versus-cost claims ([#11381](https://github.com/diegosouzapw/OmniRoute/pull/11381)). diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 9dae1d0150..9fae13e631 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -5719,17 +5719,28 @@ paths: x-loopback-only: true tags: [System] summary: Read a bounded Video Bridge drill-down slice - description: Internal loopback/token-authenticated lookup into a short-lived per-session frame cache. It never downloads media or starts a subprocess; start/end and frame count only select already materialized frames. + description: Internal loopback/token-authenticated lookup into a short-lived cache isolated by an opaque principal, session, and media reference. It never downloads media or starts a subprocess; start/end and frame count only select already materialized, canonicalized JPEG frames whose dimensions were derived from their bytes. This cache substrate is not yet wired to the transparent Video Bridge request path and does not yet expose multi-resolution selection. security: [] parameters: + - in: header + name: x-omniroute-video-bridge-principal + required: true + description: Canonical visible-ASCII, opaque non-secret principal ID; production tenant derivation is required before enabling a caller + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: "^[!-~]{1,256}$" - in: query name: sessionId required: true - schema: { type: string, maxLength: 128 } + description: Canonical opaque ID without surrounding whitespace + schema: { type: string, minLength: 1, maxLength: 128 } - in: query name: videoRef required: true - schema: { type: string, maxLength: 4096 } + description: Canonical opaque reference without surrounding whitespace + schema: { type: string, minLength: 1, maxLength: 4096 } - in: query name: start required: false @@ -5743,25 +5754,58 @@ paths: required: false schema: { type: integer, minimum: 1, maximum: 16 } responses: - "200": { description: Bounded cached frame slice } - "403": { description: Trusted loopback/token identity required } + "200": { description: Bounded cached frame slice with derivation audit metadata } + "403": { description: Trusted loopback/token identity and principal required } "404": { description: Drill-down session or media key was not found } post: x-loopback-only: true tags: [System] summary: Store a bounded Video Bridge drill-down result - description: Internal lifecycle operation for explicitly authorized callers. The short-lived session cache is isolated by session and media reference and does not alter the primary request cost. + description: Internal lifecycle operation for explicitly authorized callers. The short-lived cache is isolated by principal, session, and media reference; enforces independent per-principal and global retained-byte quotas; accepts canonical Base64 only after a warning-sensitive bounded full JPEG decode/re-encode; strips trailing polyglot bytes; retains and charges only the canonical JPEG output; derives resolution from decoded bytes; and does not alter the primary request cost. The JSON wire budget includes Base64 overhead for the 32 MiB decoded-input ceiling. security: [] + parameters: + - in: header + name: x-omniroute-video-bridge-principal + required: true + description: Canonical visible-ASCII, opaque non-secret principal ID; production tenant derivation is required before enabling a caller + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: "^[!-~]{1,256}$" requestBody: required: true content: application/json: schema: type: object - required: [sessionId, videoRef, durationSeconds, frames] + additionalProperties: false + required: [sessionId, videoRef, derivation, durationSeconds, frames] properties: - sessionId: { type: string, maxLength: 128 } - videoRef: { type: string, maxLength: 4096 } + sessionId: + type: string + minLength: 1 + maxLength: 128 + description: Canonical opaque ID without surrounding whitespace + videoRef: + type: string + minLength: 1 + maxLength: 4096 + description: Canonical opaque reference without surrounding whitespace + derivation: + type: object + additionalProperties: false + required: [parentContentHash, policy, version] + properties: + parentContentHash: + type: string + pattern: "^sha256:[a-f0-9]{64}$" + policy: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._/-]{0,63}$" + version: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._/-]{0,63}$" durationSeconds: { type: number, exclusiveMinimum: 0, maximum: 600 } frames: type: array @@ -5769,27 +5813,43 @@ paths: maxItems: 16 items: type: object + additionalProperties: false required: [timestampSeconds, dataUri] properties: timestampSeconds: { type: number, minimum: 0 } - dataUri: { type: string, pattern: "^data:image/jpeg;base64," } + dataUri: + type: string + minLength: 27 + maxLength: 5592431 + description: Canonical Base64 data URI whose decoded bytes pass a warning-sensitive bounded full JPEG decode/re-encode; trailing bytes are discarded and width and height are derived server-side responses: "201": { description: Drill-down result stored } - "403": { description: Trusted loopback/token identity required } + "403": { description: Trusted loopback/token identity and principal required } "413": { description: Payload exceeds the bounded session budget } + "499": { description: Caller cancelled before the derivation was committed } delete: x-loopback-only: true tags: [System] summary: Delete a Video Bridge drill-down session security: [] parameters: + - in: header + name: x-omniroute-video-bridge-principal + required: true + description: Canonical visible-ASCII, opaque non-secret principal ID; production tenant derivation is required before enabling a caller + schema: + type: string + minLength: 1 + maxLength: 256 + pattern: "^[!-~]{1,256}$" - in: query name: sessionId required: true - schema: { type: string, maxLength: 128 } + description: Canonical opaque ID without surrounding whitespace + schema: { type: string, minLength: 1, maxLength: 128 } responses: "200": { description: Session entries removed } - "403": { description: Trusted loopback/token identity required } + "403": { description: Trusted loopback/token identity and principal required } /api/cache/stats: get: diff --git a/docs/security/GUARDRAILS.md b/docs/security/GUARDRAILS.md index a97d42514c..de3ea46673 100644 --- a/docs/security/GUARDRAILS.md +++ b/docs/security/GUARDRAILS.md @@ -327,8 +327,12 @@ fixed FFmpeg pass over the already validated local stream, select bounded `showinfo` scene timestamps, and fall back deterministically to the same uniform midpoints on detector failure, timeout, malformed output, or an empty candidate set. Segment-aware mode allocates midpoint samples proportionally to -the validated scene intervals. The hard 16-frame cap is -applied after selection in every policy. A caller may optionally provide a +the validated scene intervals; segment-aware evidence and fallback behavior are +detailed below. The hard 16-frame cap is +applied after selection in every policy. When a scene-aware request has only a +one-frame budget, it uses the uniform midpoint of the active full-video or focus +window and reports `policyEffective: uniform`: a single selected scene frame +cannot preserve both temporal ends. A caller may optionally provide a finite focus window (`start`/`end` seconds); bounds are clamped to the media duration, reversed or non-finite windows are rejected, and all sampling policies are performed only inside the normalized interval. The resulting @@ -348,6 +352,43 @@ cannot override the separate warning against following instructions visible or audible in the media. Textual focus never infers `start`/`end` or changes the temporal sampler. +#### FU-07 structural segment evidence + +`segment_aware` uses one bounded pre-analysis pass over the already validated +local video stream. The fixed filter chain first scales to at most 320 pixels +wide, detects scene changes and frozen intervals, then samples at 1 frame per +second for blur, average luma, and spatial/temporal information. The pass is +limited to 600 structural samples, one FFmpeg/filter thread, the same +`file`-only protocol and container allowlists, a 1 MiB process-output bound, +and at most 30 seconds inside the broker's shared abort/deadline. It never +accepts a command, filter, path, or URL from the request. + +The structural values are deterministic sampling evidence, not semantic video +understanding. They do not infer subjects, actions, captions, speech, or user +intent. Scene and freeze boundaries form segments; freeze coverage, blur, +exposure, spatial detail, and temporal change only influence how the existing +1–16 frame budget is allocated. A fully frozen segment is capped at one frame, +while non-frozen segments compete for the remaining budget. When boundaries +outnumber frames, uniform timeline coverage is retained so rapid early cuts +cannot hide a long trailing segment. Scene boundaries within the 1-second +analysis resolution of a freeze boundary are coalesced. + +Missing filters, malformed/empty evidence, a detector error, or the bounded +pre-analysis timeout fail open to the exact uniform midpoint policy. A caller +abort or broker deadline does not fail open: it terminates the in-flight +subprocess, prevents later frame extraction, and the private temporary tree is +removed in `finally`. + +`scripts/perf/video-bridge-fu07-eval.ts` generates deterministic real FFmpeg +fixtures for post-dedup caption-call savings, dense-motion budget allocation, +blur/exposure/SI-TI evidence, rapid cuts with a long tail, and gradual-fade +false positives. It records pre-analysis wall time and, where `/usr/bin/time` +is available, child CPU and peak RSS. Its quality checks are structural oracles +only. Real caption-model quality remains `HOLD` because this harness has no +authorized endpoint or frozen judge. Monetary savings also remain `HOLD` +unless `--caption-cost-per-call-usd` supplies an explicit positive per-call +estimate; the script never fabricates either result. + Each frame is limited to 4 MiB, all raw frames together to 23 MiB, and the serialized broker response to 32 MiB. A private temporary directory is removed in `finally`. OmniRoute does not bundle FFmpeg and does not accept a custom @@ -369,10 +410,23 @@ coverage. Output metadata separates extracted candidates, successfully used frames, and visual duplicates dropped. An explicitly marked video part may request a timestamped contact sheet. The -bridge builds at most a 4-column, 16-frame JPEG grid and labels the resulting -observation with every source timestamp. If `sharp` cannot decode or compose -the grid, the bridge falls back to the individual JPEG frames; a client abort -still propagates through the sheet operation. +bridge builds at most a 4-column, 16-frame JPEG grid. Every 512-pixel cell burns +its source timestamp into a high-contrast bottom band, while the same timestamps +remain in textual metadata for downstream association and audit. The complete +JPEG remains capped at 32 MiB. If `sharp` cannot decode or compose the grid, the +bridge falls back to the individual JPEG frames; a client abort still propagates +through the sheet operation. + +Promotion evidence is deliberately separate from the synthetic composition +microbenchmark. `scripts/perf/video-bridge-contact-sheet-eval.ts` defines a +schema-versioned A/B harness for real OpenAI-compatible vision models. It measures +provider-reported tokens, end-to-end wall latency (including sheet composition), +model-call count, and manifest-defined fact retention. Raw model responses are not +written to the report; only SHA-256 digests and matched fact IDs are retained. The +harness makes no network or paid model call unless `--execute-real` is passed and +`--model`, `OMNIROUTE_BASE_URL`, and `OMNIROUTE_API_KEY` are configured. Without +that explicit real run, its machine-readable verdict remains `HOLD`; synthetic +payload/call-count measurements alone are not promotion evidence. Callers may attach an optional `transcript.cues` array to a supported video part when they already possess aligned text. Each cue must carry `text`, a @@ -400,14 +454,39 @@ or download a second media copy; without that explicit track, it remains video-only. The internal `/api/modality-bridge/video/drilldown` lifecycle is a separate, -loopback/token-authenticated cache. It stores at most 16 JPEG frames per entry, -keeps entries isolated by session and video reference, expires them after ten -minutes, and supports bounded `start`/`end` reads or explicit session deletion. -Besides the per-entry limits, the cache enforces a global 256 MiB decoded-byte -budget: least-recently-used entries are evicted until new content fits, and an -entry larger than the whole budget is rejected outright. -It only slices materialized frames and cannot increase the cost of the primary -video request. +loopback/token-authenticated cache substrate. Every operation also requires a +canonical opaque principal ID. Before a production caller is enabled, it must +derive that ID from the authenticated tenant and must never forward a +client-selected value. Cache keys bind that principal to canonical session and +video-reference IDs, store only their SHA-256-derived keys, and scope both reads +and deletion to the same principal. The cache stores at most 16 derived JPEG +frames per entry, expires them after ten minutes, and supports bounded +`start`/`end` reads or explicit session deletion. + +Each principal is limited to 16 entries and 64 MiB of canonical JPEG data. Those +limits are independent from the global 64-entry/256 MiB ceiling: principal quota +pressure evicts only that principal's least-recently-used entries before global +LRU eviction is considered. Expired entries are swept from both principal and +global accounting on cache activity, while cancellation and validation failure do +not commit a partial replacement. + +The cache rejects non-canonical Base64, excess padding, non-JPEG media, malformed or +truncated JPEGs, and JPEGs that produce a warning during a bounded full-image `sharp` +decode. It re-encodes each accepted image as a canonical JPEG, derives width and height +from the decoded bytes instead of trusting caller fields, and discards any trailing +polyglot bytes rather than retaining them. Only the bounded canonical compressed buffer +is charged to both quotas. The JSON wire limit includes Base64 overhead for the 32 MiB +decoded-input ceiling. Every +stored derivation records its validated JPEG format/resolution, sampling policy, +derivation version, creation time, server-computed content hash, and hashed parent +reference plus the trusted caller's parent-content hash. Cancellation is checked +between asynchronous decode/hash phases before the atomic cache commit. + +This tranche does not yet connect a production producer to the route and does not +provide multi-resolution variant selection. The transparent Video Bridge request +path therefore incurs no added work, while tenant-bound principal derivation and +the full FU-08 multi-resolution lifecycle remain explicit follow-up work rather +than documented as complete behavior. Frames are captioned sequentially with the configured Video model. An empty Video override inherits the Vision setting; if both are empty, the Vision diff --git a/scripts/perf/video-bridge-bench.ts b/scripts/perf/video-bridge-bench.ts index 66c5fd7391..2b457d3b23 100644 --- a/scripts/perf/video-bridge-bench.ts +++ b/scripts/perf/video-bridge-bench.ts @@ -10,8 +10,10 @@ * scene_aware vs segment_aware for growing scene-candidate counts. The * ffmpeg scene-detection pass is shared by both aware policies and is * I/O-bound, so the incremental policy cost is exactly this selection step. - * 3. Contact sheet: composes synthetic JPEG frames into the timestamped grid - * and compares payload bytes + model calls against individual frames. + * 3. Contact sheet: composes synthetic JPEG frames into the visually timestamped + * grid and compares payload bytes + structural call counts. This microbenchmark + * does not measure real-model tokens, latency, or quality; use + * video-bridge-contact-sheet-eval.ts before considering promotion. */ import { performance } from "node:perf_hooks"; @@ -118,6 +120,9 @@ async function syntheticJpegFrame(index: number, width = 512, height = 288): Pro async function benchContactSheet(): Promise { console.log("\n== Contact sheet vs individual frames (synthetic 512x288 JPEG) =="); + console.log( + "STRUCTURAL ONLY: real-model tokens/latency/quality are unmeasured; promotion remains HOLD." + ); console.log("frames | sheet_ms sheet_KiB individual_KiB model_calls(sheet/individual)"); for (const frameCount of [1, 4, 8, 16]) { const frames = await Promise.all( diff --git a/scripts/perf/video-bridge-contact-sheet-eval.ts b/scripts/perf/video-bridge-contact-sheet-eval.ts new file mode 100644 index 0000000000..7020a5368c --- /dev/null +++ b/scripts/perf/video-bridge-contact-sheet-eval.ts @@ -0,0 +1,578 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { fileURLToPath } from "node:url"; + +import { z } from "zod"; + +import { + buildVideoContactSheet, + type ContactSheetFrame, +} from "../../src/lib/guardrails/videoBridgeContactSheet"; + +export type VideoContactSheetEvalConfigurationState = "configured-not-executed" | "not-configured"; + +export interface VideoContactSheetEvalHoldReportInput { + caseCount: number; + configurationState: VideoContactSheetEvalConfigurationState; + missingConfiguration?: string[]; +} + +export interface VideoContactSheetEvalHoldReport { + caseCount: number; + execution: { + realModel: false; + state: VideoContactSheetEvalConfigurationState; + }; + kind: "video-contact-sheet-ab-eval"; + missingConfiguration: string[]; + promotion: { + reasons: ["REAL_MODEL_CONFIGURATION_MISSING" | "REAL_MODEL_EVAL_NOT_EXECUTED"]; + status: "HOLD"; + }; + results: []; + schemaVersion: 1; + summary: null; +} + +export interface VideoContactSheetEvalThresholds { + minLatencyReductionRatio: number; + minQualityRetention: number; + minQualityScore: number; + minTokenReductionRatio: number; +} + +export interface VideoContactSheetEvalAggregate { + latencyMs: number; + qualityScore: number; + totalTokens: number | null; +} + +export type VideoContactSheetPromotionReason = + | "LATENCY_REDUCTION_BELOW_THRESHOLD" + | "QUALITY_RETENTION_BELOW_THRESHOLD" + | "QUALITY_SCORE_BELOW_THRESHOLD" + | "TOKEN_REDUCTION_BELOW_THRESHOLD" + | "TOKEN_USAGE_UNAVAILABLE"; + +export interface VideoContactSheetPromotionDecision { + metrics: { + latencyReductionRatio: number; + qualityRetention: number; + tokenReductionRatio: number | null; + }; + reasons: VideoContactSheetPromotionReason[]; + status: "ELIGIBLE" | "HOLD"; +} + +const MAX_EVAL_FRAME_BASE64_CHARS = 5_592_408; + +const evalThresholdsSchema = z + .object({ + minLatencyReductionRatio: z.number().positive().max(1), + minQualityRetention: z.number().min(0).max(1), + minQualityScore: z.number().min(0).max(1), + minTokenReductionRatio: z.number().positive().max(1), + }) + .strict(); + +const evalManifestSchema = z + .object({ + cases: z + .array( + z + .object({ + expectedFacts: z + .array( + z + .object({ + id: z.string().min(1), + requiredTerms: z.array(z.string().min(1)).min(1), + timestampSeconds: z.number().finite().nonnegative(), + }) + .strict() + ) + .min(1), + frames: z + .array( + z + .object({ + dataUri: z + .string() + .max("data:image/jpeg;base64,".length + MAX_EVAL_FRAME_BASE64_CHARS) + .regex( + /^data:image\/jpeg;base64,[A-Za-z0-9+/=]{4,5592408}$/i, + "expected a bounded JPEG data URI" + ), + timestampSeconds: z.number().finite().nonnegative(), + }) + .strict() + ) + .min(1) + .max(16), + id: z.string().min(1), + prompt: z.string().min(1), + }) + .strict() + ) + .min(1), + id: z.string().min(1), + schemaVersion: z.literal(1), + thresholds: evalThresholdsSchema, + }) + .strict(); + +const chatCompletionSchema = z + .object({ + choices: z + .array( + z + .object({ + message: z.object({ content: z.string() }).passthrough(), + }) + .passthrough() + ) + .min(1), + usage: z + .object({ + completion_tokens: z.number().nonnegative().optional(), + prompt_tokens: z.number().nonnegative().optional(), + total_tokens: z.number().nonnegative().optional(), + }) + .passthrough() + .optional(), + }) + .passthrough(); + +export type VideoContactSheetEvalManifest = z.infer; + +export interface VideoContactSheetEvalConfig { + apiKey: string; + endpoint: string; + model: string; +} + +interface EvalFactScore { + matchedFactIds: string[]; + qualityScore: number; +} + +interface EvalPathResult extends EvalFactScore { + latencyMs: number; + modelCalls: number; + responseDigest: string; + totalTokens: number | null; +} + +export interface VideoContactSheetEvalCaseResult { + caseId: string; + individual: EvalPathResult; + sheet: EvalPathResult; +} + +export interface VideoContactSheetEvalExecutedReport { + caseCount: number; + execution: { + realModel: true; + state: "executed"; + }; + generatedAt: string; + kind: "video-contact-sheet-ab-eval"; + manifestDigest: string; + manifestId: string; + model: string; + promotion: VideoContactSheetPromotionDecision; + results: VideoContactSheetEvalCaseResult[]; + schemaVersion: 1; + summary: { + individual: VideoContactSheetEvalAggregate & { modelCalls: number }; + sheet: VideoContactSheetEvalAggregate & { modelCalls: number }; + }; + thresholds: VideoContactSheetEvalThresholds; +} + +type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise; + +export function createVideoContactSheetEvalHoldReport( + input: VideoContactSheetEvalHoldReportInput +): VideoContactSheetEvalHoldReport { + const reason = + input.configurationState === "not-configured" + ? "REAL_MODEL_CONFIGURATION_MISSING" + : "REAL_MODEL_EVAL_NOT_EXECUTED"; + return { + caseCount: input.caseCount, + execution: { + realModel: false, + state: input.configurationState, + }, + kind: "video-contact-sheet-ab-eval", + missingConfiguration: [...(input.missingConfiguration ?? [])], + promotion: { + reasons: [reason], + status: "HOLD", + }, + results: [], + schemaVersion: 1, + summary: null, + }; +} + +function reductionRatio(baseline: number, candidate: number): number { + if (baseline <= 0) return 0; + return (baseline - candidate) / baseline; +} + +export function assessVideoContactSheetPromotion(input: { + individual: VideoContactSheetEvalAggregate; + sheet: VideoContactSheetEvalAggregate; + thresholds: VideoContactSheetEvalThresholds; +}): VideoContactSheetPromotionDecision { + const latencyReductionRatio = reductionRatio(input.individual.latencyMs, input.sheet.latencyMs); + const qualityRetention = + input.individual.qualityScore > 0 + ? input.sheet.qualityScore / input.individual.qualityScore + : 0; + const tokenReductionRatio = + input.individual.totalTokens === null || input.sheet.totalTokens === null + ? null + : reductionRatio(input.individual.totalTokens, input.sheet.totalTokens); + const reasons: VideoContactSheetPromotionReason[] = []; + const requiredLatencyReduction = Math.max( + Number.EPSILON, + input.thresholds.minLatencyReductionRatio + ); + const requiredTokenReduction = Math.max(Number.EPSILON, input.thresholds.minTokenReductionRatio); + if (latencyReductionRatio < requiredLatencyReduction) { + reasons.push("LATENCY_REDUCTION_BELOW_THRESHOLD"); + } + if (input.sheet.qualityScore < input.thresholds.minQualityScore) { + reasons.push("QUALITY_SCORE_BELOW_THRESHOLD"); + } + if (qualityRetention < input.thresholds.minQualityRetention) { + reasons.push("QUALITY_RETENTION_BELOW_THRESHOLD"); + } + if (tokenReductionRatio === null) { + reasons.push("TOKEN_USAGE_UNAVAILABLE"); + } else if (tokenReductionRatio < requiredTokenReduction) { + reasons.push("TOKEN_REDUCTION_BELOW_THRESHOLD"); + } + return { + metrics: { + latencyReductionRatio, + qualityRetention, + tokenReductionRatio, + }, + reasons, + status: reasons.length === 0 ? "ELIGIBLE" : "HOLD", + }; +} + +function normalizeEvalText(value: string): string { + return value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase(); +} + +function formatEvalTimestamp(timestampSeconds: number): string { + const totalMilliseconds = Math.max(0, Math.round(timestampSeconds * 1000)); + const minutes = Math.floor(totalMilliseconds / 60_000); + const seconds = Math.floor((totalMilliseconds % 60_000) / 1000); + const milliseconds = totalMilliseconds % 1000; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`; +} + +function scoreFacts( + response: string, + expectedFacts: VideoContactSheetEvalManifest["cases"][number]["expectedFacts"] +): EvalFactScore { + const normalizedResponse = normalizeEvalText(response); + const matchedFactIds = expectedFacts + .filter((fact) => { + const timestamp = normalizeEvalText(formatEvalTimestamp(fact.timestampSeconds)); + const timestampIndex = normalizedResponse.indexOf(timestamp); + if (timestampIndex < 0) return false; + const factWindow = normalizedResponse.slice( + Math.max(0, timestampIndex - 160), + Math.min(normalizedResponse.length, timestampIndex + timestamp.length + 160) + ); + return fact.requiredTerms.every((term) => factWindow.includes(normalizeEvalText(term))); + }) + .map((fact) => fact.id); + return { + matchedFactIds, + qualityScore: matchedFactIds.length / expectedFacts.length, + }; +} + +function digestResponse(response: string): string { + return createHash("sha256").update(response).digest("hex"); +} + +function sumTokens(values: Array): number | null { + if (values.some((value) => value === null)) return null; + return values.reduce((sum, value) => sum + (value ?? 0), 0); +} + +async function callVisionModel(input: { + config: VideoContactSheetEvalConfig; + dataUri: string; + fetchImpl: FetchLike; + prompt: string; +}): Promise<{ content: string; totalTokens: number | null }> { + const response = await input.fetchImpl(input.config.endpoint, { + body: JSON.stringify({ + messages: [ + { + content: [ + { text: input.prompt, type: "text" }, + { image_url: { url: input.dataUri }, type: "image_url" }, + ], + role: "user", + }, + ], + model: input.config.model, + temperature: 0, + }), + headers: { + authorization: `Bearer ${input.config.apiKey}`, + "content-type": "application/json", + }, + method: "POST", + }); + if (!response.ok) { + throw new Error(`Video contact-sheet eval request failed with HTTP ${response.status}`); + } + const parsed = chatCompletionSchema.parse(await response.json()); + const usage = parsed.usage; + const totalTokens = + usage?.total_tokens ?? + (usage?.prompt_tokens !== undefined && usage.completion_tokens !== undefined + ? usage.prompt_tokens + usage.completion_tokens + : null); + return { + content: parsed.choices[0].message.content, + totalTokens, + }; +} + +async function evaluateIndividualFrames(input: { + evalCase: VideoContactSheetEvalManifest["cases"][number]; + config: VideoContactSheetEvalConfig; + fetchImpl: FetchLike; +}): Promise { + const startedAt = performance.now(); + const calls: Array<{ content: string; totalTokens: number | null }> = []; + for (const frame of input.evalCase.frames) { + calls.push( + await callVisionModel({ + config: input.config, + dataUri: frame.dataUri, + fetchImpl: input.fetchImpl, + prompt: `${input.evalCase.prompt}\nAnalyze only the frame at ${formatEvalTimestamp(frame.timestampSeconds)}. Associate every observation with that exact timestamp label.`, + }) + ); + } + const content = calls.map((call) => call.content).join("\n"); + return { + ...scoreFacts(content, input.evalCase.expectedFacts), + latencyMs: performance.now() - startedAt, + modelCalls: calls.length, + responseDigest: digestResponse(content), + totalTokens: sumTokens(calls.map((call) => call.totalTokens)), + }; +} + +async function evaluateContactSheet(input: { + evalCase: VideoContactSheetEvalManifest["cases"][number]; + config: VideoContactSheetEvalConfig; + fetchImpl: FetchLike; +}): Promise { + const startedAt = performance.now(); + const sheet = await buildVideoContactSheet(input.evalCase.frames as ContactSheetFrame[], { + columns: 4, + timeoutMs: 30_000, + }); + if (!sheet.used || !sheet.dataUri) { + throw new Error("Video contact-sheet eval could not compose the bounded JPEG grid"); + } + const call = await callVisionModel({ + config: input.config, + dataUri: sheet.dataUri, + fetchImpl: input.fetchImpl, + prompt: `${input.evalCase.prompt}\nAnalyze every cell in the contact sheet. Timestamp labels are burned into each cell. Associate every observation with its visible timestamp.`, + }); + return { + ...scoreFacts(call.content, input.evalCase.expectedFacts), + latencyMs: performance.now() - startedAt, + modelCalls: 1, + responseDigest: digestResponse(call.content), + totalTokens: call.totalTokens, + }; +} + +function aggregatePathResults( + results: VideoContactSheetEvalCaseResult[], + path: "individual" | "sheet" +): VideoContactSheetEvalAggregate & { modelCalls: number } { + const pathResults = results.map((result) => result[path]); + return { + latencyMs: pathResults.reduce((sum, result) => sum + result.latencyMs, 0), + modelCalls: pathResults.reduce((sum, result) => sum + result.modelCalls, 0), + qualityScore: + pathResults.reduce((sum, result) => sum + result.qualityScore, 0) / pathResults.length, + totalTokens: sumTokens(pathResults.map((result) => result.totalTokens)), + }; +} + +export async function runVideoContactSheetEval(input: { + config: VideoContactSheetEvalConfig; + fetchImpl?: FetchLike; + manifest: VideoContactSheetEvalManifest; +}): Promise { + const manifest = evalManifestSchema.parse(input.manifest); + const endpoint = z.string().url().parse(input.config.endpoint); + const config = { + apiKey: z.string().min(1).parse(input.config.apiKey), + endpoint, + model: z.string().min(1).parse(input.config.model), + }; + const fetchImpl = input.fetchImpl ?? fetch; + const results: VideoContactSheetEvalCaseResult[] = []; + for (const evalCase of manifest.cases) { + const individual = await evaluateIndividualFrames({ config, evalCase, fetchImpl }); + const sheet = await evaluateContactSheet({ config, evalCase, fetchImpl }); + results.push({ caseId: evalCase.id, individual, sheet }); + } + const individual = aggregatePathResults(results, "individual"); + const sheet = aggregatePathResults(results, "sheet"); + const promotion = assessVideoContactSheetPromotion({ + individual, + sheet, + thresholds: manifest.thresholds, + }); + return { + caseCount: manifest.cases.length, + execution: { realModel: true, state: "executed" }, + generatedAt: new Date().toISOString(), + kind: "video-contact-sheet-ab-eval", + manifestDigest: createHash("sha256").update(JSON.stringify(manifest)).digest("hex"), + manifestId: manifest.id, + model: config.model, + promotion, + results, + schemaVersion: 1, + summary: { individual, sheet }, + thresholds: manifest.thresholds, + }; +} + +function readArgument(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function printUsage(): void { + console.log( + [ + "Usage:", + " node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest --model ", + " node --import tsx/esm scripts/perf/video-bridge-contact-sheet-eval.ts --manifest --model --execute-real", + "", + "The default command validates configuration and emits HOLD without calling a model.", + "A real paid/networked run requires --execute-real, --model, and the documented variables:", + " OMNIROUTE_BASE_URL", + " OMNIROUTE_API_KEY", + "", + "Manifest v1: id, thresholds, and 1+ cases. Each case has 1-16 bounded JPEG data URIs,", + "timestamps, a prompt, and expectedFacts with timestampSeconds + requiredTerms.", + ].join("\n") + ); +} + +async function loadManifest(manifestPath: string): Promise { + const raw = await readFile(path.resolve(manifestPath), "utf8"); + return evalManifestSchema.parse(JSON.parse(raw)); +} + +function resolveChatCompletionsEndpoint(baseUrl: string): string { + const normalized = baseUrl.replace(/\/{1,8}$/u, ""); + if (normalized.endsWith("/v1/chat/completions")) return normalized; + if (normalized.endsWith("/v1")) return `${normalized}/chat/completions`; + return `${normalized}/v1/chat/completions`; +} + +async function main(): Promise { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + printUsage(); + return; + } + const manifestPath = readArgument("manifest"); + const model = readArgument("model"); + const missingConfiguration: string[] = []; + if (!manifestPath) missingConfiguration.push("--manifest"); + if (!model) missingConfiguration.push("--model"); + const baseUrl = process.env.OMNIROUTE_BASE_URL; + const apiKey = process.env.OMNIROUTE_API_KEY; + if (!baseUrl) missingConfiguration.push("OMNIROUTE_BASE_URL"); + if (!apiKey) missingConfiguration.push("OMNIROUTE_API_KEY"); + + let manifest: VideoContactSheetEvalManifest | null = null; + if (manifestPath) manifest = await loadManifest(manifestPath); + if (missingConfiguration.length > 0) { + console.log( + JSON.stringify( + createVideoContactSheetEvalHoldReport({ + caseCount: manifest?.cases.length ?? 0, + configurationState: "not-configured", + missingConfiguration, + }), + null, + 2 + ) + ); + return; + } + if (!process.argv.includes("--execute-real")) { + console.log( + JSON.stringify( + createVideoContactSheetEvalHoldReport({ + caseCount: manifest?.cases.length ?? 0, + configurationState: "configured-not-executed", + }), + null, + 2 + ) + ); + return; + } + if (!manifest || !baseUrl || !apiKey || !model) { + throw new Error("Video contact-sheet eval configuration was not resolved"); + } + console.log( + JSON.stringify( + await runVideoContactSheetEval({ + config: { apiKey, endpoint: resolveChatCompletionsEndpoint(baseUrl), model }, + manifest, + }), + null, + 2 + ) + ); +} + +const isMainModule = + typeof process.argv[1] === "string" && + path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isMainModule) { + main().catch(() => { + console.error("Video contact-sheet eval failed validation or execution."); + process.exitCode = 1; + }); +} diff --git a/scripts/perf/video-bridge-fu07-eval.ts b/scripts/perf/video-bridge-fu07-eval.ts new file mode 100644 index 0000000000..33cd67665a --- /dev/null +++ b/scripts/perf/video-bridge-fu07-eval.ts @@ -0,0 +1,493 @@ +/** + * Real-media FU-07 structural-sampling evaluation. + * + * Run: node --import tsx/esm scripts/perf/video-bridge-fu07-eval.ts + * Optional estimate: append --caption-cost-per-call-usd . + * + * This evaluates deterministic structural oracles, not semantic model quality. + * Model quality and monetary savings remain HOLD without an external receipt. + */ +import { execFile } from "node:child_process"; +import { access, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { promisify } from "node:util"; + +import { deduplicateVideoFrames } from "../../src/lib/guardrails/videoBridgeHelpers"; +import { + analyzeVideoStructure, + calculateSamplingDecision, + extractFramesFromLocalVideo, + readBoundedExtractedFrames, + type VideoCommandRunner, + type VideoStructuralAnalysis, + type VideoStructuralSample, +} from "../../src/lib/guardrails/videoBridgeRuntime"; + +const execFileAsync = promisify(execFile); +const REQUIRED_FILTERS = ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"]; +const TIME_MARKER = "__FU07_TIME__"; + +interface ChildCost { + maxRssKiB: number | null; + systemSeconds: number | null; + userSeconds: number | null; + wallMs: number; +} + +interface FixtureResult { + captionCallsAvoided: number; + childCost: ChildCost; + freezeIntervals: number; + name: string; + oracle: Record; + passed: boolean; + sceneCandidates: number; + structuralFrames: number; + uniformFrames: number; +} + +function average(values: Array): number | null { + const finite = values.filter( + (value): value is number => value !== null && value !== undefined && Number.isFinite(value) + ); + return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null; +} + +function samplesIn( + analysis: VideoStructuralAnalysis, + startSeconds: number, + endSeconds: number +): VideoStructuralSample[] { + return analysis.samples.filter( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); +} + +async function generateFixture(outputPath: string, args: readonly string[]): Promise { + await execFileAsync( + "ffmpeg", + ["-hide_banner", "-loglevel", "error", ...args, "-threads", "1", "-y", outputPath], + { maxBuffer: 1024 * 1024, timeout: 30_000 } + ); +} + +async function generateStaticFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=blue:s=320x180:d=8:r=12", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + ]); +} + +async function generateMixedFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=6:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function generateBlurExposureFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=3:r=12", + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=3:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v]gblur=sigma=12[blur];[blur][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function generateDenseTailFixture(outputPath: string): Promise { + const args: string[] = []; + for (const source of [ + "color=c=black:s=160x90:d=0.5:r=10", + "color=c=white:s=160x90:d=0.5:r=10", + "color=c=black:s=160x90:d=0.5:r=10", + "color=c=white:s=160x90:d=0.5:r=10", + "testsrc2=s=160x90:d=8:r=10", + ]) { + args.push("-f", "lavfi", "-i", source); + } + args.push( + "-filter_complex", + "[0:v][1:v][2:v][3:v][4:v]concat=n=5:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast" + ); + await generateFixture(outputPath, args); +} + +async function generateGradualFadeFixture(outputPath: string): Promise { + await generateFixture(outputPath, [ + "-f", + "lavfi", + "-i", + "color=c=white:s=320x180:d=8:r=12", + "-vf", + "fade=t=out:st=0:d=8,format=yuv420p", + "-c:v", + "libx264", + "-preset", + "ultrafast", + ]); +} + +async function supportsTimeBinary(): Promise { + try { + await access("/usr/bin/time"); + return true; + } catch { + return false; + } +} + +function parseTimeCost(stderr: string, wallMs: number): ChildCost { + const match = new RegExp(`${TIME_MARKER} ([\\d.]+) ([\\d.]+) ([\\d.]+)`).exec(stderr); + return { + maxRssKiB: match ? Number(match[3]) : null, + systemSeconds: match ? Number(match[2]) : null, + userSeconds: match ? Number(match[1]) : null, + wallMs, + }; +} + +async function timedAnalysis( + inputPath: string, + durationSeconds: number, + useTimeBinary: boolean +): Promise<{ analysis: VideoStructuralAnalysis; cost: ChildCost }> { + let cost: ChildCost = { + maxRssKiB: null, + systemSeconds: null, + userSeconds: null, + wallMs: 0, + }; + const runner: VideoCommandRunner = async (executable, args, options) => { + const startedAt = performance.now(); + const command = useTimeBinary ? "/usr/bin/time" : executable; + const commandArgs = useTimeBinary + ? ["-f", `${TIME_MARKER} %U %S %M`, executable, ...args] + : [...args]; + const result = await execFileAsync(command, commandArgs, { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + }); + cost = parseTimeCost(String(result.stderr), performance.now() - startedAt); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; + }; + const analysis = await analyzeVideoStructure(inputPath, { + durationSeconds, + runner, + streamIndex: 0, + timeoutMs: 30_000, + }); + return { analysis, cost }; +} + +function sampling( + durationSeconds: number, + frameCount: number, + analysis: VideoStructuralAnalysis +): { structural: number[]; uniform: number[] } { + const uniform = calculateSamplingDecision(durationSeconds, frameCount, "uniform").timestamps; + const structural = calculateSamplingDecision( + durationSeconds, + frameCount, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ).timestamps; + return { structural, uniform }; +} + +async function captionCallsAfterDedup( + inputPath: string, + outputDirectory: string, + samplingPolicy: "segment_aware" | "uniform" +): Promise { + await mkdir(outputDirectory, { mode: 0o700 }); + const frames = await extractFramesFromLocalVideo(inputPath, outputDirectory, { + durationSeconds: 8, + frameCount: 8, + samplingPolicy, + streamIndex: 0, + timeoutMs: 30_000, + }); + const bytes = await readBoundedExtractedFrames(frames); + const deduplicated = await deduplicateVideoFrames( + frames.map((frame, index) => ({ + dataUri: `data:image/jpeg;base64,${bytes[index].toString("base64")}`, + timestampSeconds: frame.timestampSeconds, + })) + ); + return deduplicated.frames.length; +} + +function result( + name: string, + cost: ChildCost, + analysis: VideoStructuralAnalysis, + uniform: number[], + structural: number[], + oracle: Record, + captionCallsAvoided = 0 +): FixtureResult { + const booleans = Object.values(oracle).filter( + (value): value is boolean => typeof value === "boolean" + ); + return { + captionCallsAvoided, + childCost: cost, + freezeIntervals: analysis.freezeIntervals.length, + name, + oracle, + passed: booleans.every(Boolean), + sceneCandidates: analysis.sceneCandidates.length, + structuralFrames: structural.length, + uniformFrames: uniform.length, + }; +} + +async function main(): Promise { + const version = await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + const filters = await execFileAsync("ffmpeg", ["-hide_banner", "-filters"], { + maxBuffer: 2 * 1024 * 1024, + timeout: 5_000, + }); + const missingFilters = REQUIRED_FILTERS.filter( + (filter) => !new RegExp(`\\b${filter}\\b`).test(String(filters.stdout)) + ); + if (missingFilters.length > 0) + throw new Error(`Missing required FFmpeg filters: ${missingFilters.join(", ")}`); + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-eval-")); + const useTimeBinary = await supportsTimeBinary(); + const results: FixtureResult[] = []; + try { + const staticPath = join(directory, "static.mp4"); + await generateStaticFixture(staticPath); + const staticRun = await timedAnalysis(staticPath, 8, useTimeBinary); + const staticSampling = sampling(8, 8, staticRun.analysis); + const uniformCaptionCalls = await captionCallsAfterDedup( + staticPath, + join(directory, "static-uniform"), + "uniform" + ); + const structuralCaptionCalls = await captionCallsAfterDedup( + staticPath, + join(directory, "static-structural"), + "segment_aware" + ); + const staticCaptionCallsAvoided = Math.max(0, uniformCaptionCalls - structuralCaptionCalls); + results.push( + result( + "static-caption-savings", + staticRun.cost, + staticRun.analysis, + staticSampling.uniform, + staticSampling.structural, + { + fullFreezeDetected: staticRun.analysis.freezeIntervals.some( + (interval) => interval.startSeconds <= 1 && interval.endSeconds >= 7 + ), + oneIncrementalCaptionCallAvoided: staticCaptionCallsAvoided === 1, + structuralCaptionCalls, + uniformCaptionCalls, + }, + staticCaptionCallsAvoided + ) + ); + + const mixedPath = join(directory, "mixed.mp4"); + await generateMixedFixture(mixedPath); + const mixedRun = await timedAnalysis(mixedPath, 10, useTimeBinary); + const mixedSampling = sampling(10, 4, mixedRun.analysis); + const uniformDense = mixedSampling.uniform.filter((timestamp) => timestamp > 6).length; + const structuralDense = mixedSampling.structural.filter((timestamp) => timestamp > 6).length; + results.push( + result( + "dense-budget-quality-oracle", + mixedRun.cost, + mixedRun.analysis, + mixedSampling.uniform, + mixedSampling.structural, + { + denseFramesStructural: structuralDense, + denseFramesUniform: uniformDense, + denseRegionGetsMoreBudget: structuralDense > uniformDense, + frozenRegionRetainsCoverage: mixedSampling.structural.some((timestamp) => timestamp < 6), + } + ) + ); + + const qualityPath = join(directory, "blur-exposure.mp4"); + await generateBlurExposureFixture(qualityPath); + const qualityRun = await timedAnalysis(qualityPath, 10, useTimeBinary); + const qualitySampling = sampling(10, 6, qualityRun.analysis); + const blurred = samplesIn(qualityRun.analysis, 0, 3); + const dark = samplesIn(qualityRun.analysis, 3, 6); + const sharp = samplesIn(qualityRun.analysis, 6, 10); + const blurredBlur = average(blurred.map((sample) => sample.blur)); + const blurredSpatial = average(blurred.map((sample) => sample.spatialInformation)); + const darkLuma = average(dark.map((sample) => sample.brightness)); + const sharpBlur = average(sharp.map((sample) => sample.blur)); + const sharpSpatial = average(sharp.map((sample) => sample.spatialInformation)); + const sharpTemporal = average(sharp.map((sample) => sample.temporalInformation)); + const sharpLuma = average(sharp.map((sample) => sample.brightness)); + results.push( + result( + "blur-exposure-spatial-temporal-evidence", + qualityRun.cost, + qualityRun.analysis, + qualitySampling.uniform, + qualitySampling.structural, + { + blurMetricSeparated: + blurredBlur !== null && sharpBlur !== null && Math.abs(blurredBlur - sharpBlur) >= 0.05, + blurredBlur: blurredBlur ?? "missing", + darkLuma: darkLuma ?? "missing", + exposureSeparated: darkLuma !== null && sharpLuma !== null && sharpLuma - darkLuma >= 50, + sharpBlur: sharpBlur ?? "missing", + sharpSpatial: sharpSpatial ?? "missing", + sharpTemporal: sharpTemporal ?? "missing", + spatialDetailSeparated: + blurredSpatial !== null && sharpSpatial !== null && sharpSpatial - blurredSpatial >= 20, + structuralKeepsSharpRegion: + qualitySampling.structural.filter((timestamp) => timestamp >= 6).length >= 2, + temporalChangeDetected: sharpTemporal !== null && sharpTemporal >= 5, + } + ) + ); + + const tailPath = join(directory, "dense-tail.mp4"); + await generateDenseTailFixture(tailPath); + const tailRun = await timedAnalysis(tailPath, 10, useTimeBinary); + const tailSampling = sampling(10, 4, tailRun.analysis); + results.push( + result( + "dense-cuts-long-tail-regression", + tailRun.cost, + tailRun.analysis, + tailSampling.uniform, + tailSampling.structural, + { + multipleEarlyCuts: tailRun.analysis.sceneCandidates.length >= 3, + trailingEightSecondsRepresented: tailSampling.structural.some( + (timestamp) => timestamp > 2 + ), + } + ) + ); + + const fadePath = join(directory, "gradual-fade.mp4"); + await generateGradualFadeFixture(fadePath); + const fadeRun = await timedAnalysis(fadePath, 8, useTimeBinary); + const fadeSampling = sampling(8, 4, fadeRun.analysis); + results.push( + result( + "gradual-fade-false-positive", + fadeRun.cost, + fadeRun.analysis, + fadeSampling.uniform, + fadeSampling.structural, + { + hardCutFalsePositives: fadeRun.analysis.sceneCandidates.length, + noHardCutBurst: fadeRun.analysis.sceneCandidates.length <= 1, + noCaptionBudgetPruning: fadeSampling.structural.length === fadeSampling.uniform.length, + } + ) + ); + } finally { + await rm(directory, { force: true, recursive: true }); + } + + const callsAvoided = results.reduce((sum, fixture) => sum + fixture.captionCallsAvoided, 0); + const costFlag = process.argv.indexOf("--caption-cost-per-call-usd"); + const explicitCost = Number(costFlag >= 0 ? process.argv[costFlag + 1] : Number.NaN); + const report = { + captionCost: + Number.isFinite(explicitCost) && explicitCost > 0 + ? { + estimatedUsdAvoided: callsAvoided * explicitCost, + source: "explicit environment input", + status: "ESTIMATED_FROM_INPUT", + } + : { + reason: "--caption-cost-per-call-usd was not supplied with a positive number", + status: "HOLD", + }, + ffmpegVersion: String(version.stdout).split("\n")[0], + fixtures: results, + modelQuality: { + reason: + "No authorized real caption-model endpoint, credentials, or frozen judge rubric were configured; deterministic structural oracles are not semantic quality.", + status: "HOLD", + }, + gainCostComparison: { + reason: + "The real post-dedup caption-call delta is measured, but no authorized caption latency/cost receipt or child CPU/RSS receipt is configured.", + status: "HOLD", + }, + resourceCost: useTimeBinary + ? { source: "/usr/bin/time", status: "MEASURED" } + : { + reason: "/usr/bin/time is unavailable; wall time is measured but child CPU/RSS are not", + status: "HOLD", + }, + summary: { + captionCallsAvoided: callsAvoided, + failed: results.filter((fixture) => !fixture.passed).map((fixture) => fixture.name), + passed: results.filter((fixture) => fixture.passed).length, + total: results.length, + }, + timeBinary: useTimeBinary ? "/usr/bin/time" : null, + }; + console.log(JSON.stringify(report, null, 2)); + if (report.summary.failed.length > 0) process.exitCode = 1; +} + +await main(); diff --git a/src/app/api/modality-bridge/video/drilldown/route.ts b/src/app/api/modality-bridge/video/drilldown/route.ts index ba89c3053d..81c1131841 100644 --- a/src/app/api/modality-bridge/video/drilldown/route.ts +++ b/src/app/api/modality-bridge/video/drilldown/route.ts @@ -1,22 +1,149 @@ +import { z } from "zod"; + +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; import { createErrorResponse } from "@/lib/api/errorResponse"; import { - VIDEO_BRIDGE_BROKER_PATH, - isVideoBridgeBrokerInternalRequest, + resolveVideoBridgeDrilldownPrincipal, + VIDEO_BRIDGE_DRILLDOWN_PATH, } from "@/lib/guardrails/videoBridgeBrokerAuth"; import { + VideoDrilldownAbortedError, VideoDrilldownCache, - type VideoDrilldownFrame, + VideoDrilldownValidationError, + VIDEO_DRILLDOWN_MAX_ENTRY_BYTES, + VIDEO_DRILLDOWN_MAX_FRAME_DATA_URI_CHARS, } from "@/lib/guardrails/videoBridgeDrilldown"; import { resolveModelSyncInternalBaseUrl } from "@/shared/services/modelSyncScheduler"; +import { createLogger } from "@/shared/utils/logger"; + +const log = createLogger("video-bridge-drilldown"); export const dynamic = "force-dynamic"; export const revalidate = 0; -export const VIDEO_BRIDGE_DRILLDOWN_PATH = "/api/modality-bridge/video/drilldown"; -const MAX_BODY_BYTES = 34 * 1024 * 1024; +export { VIDEO_BRIDGE_DRILLDOWN_PATH }; +export const VIDEO_DRILLDOWN_MAX_BODY_BYTES = + Math.ceil(VIDEO_DRILLDOWN_MAX_ENTRY_BYTES / 3) * 4 + 64 * 1024; + +function isCanonicalOpaqueId(value: string): boolean { + return value === value.trim(); +} + +function isAsciiAlphaNumeric(code: number): boolean { + return ( + (code >= 0x30 && code <= 0x39) || + (code >= 0x41 && code <= 0x5a) || + (code >= 0x61 && code <= 0x7a) + ); +} + +function isDerivationToken(value: string): boolean { + if (value.length < 1 || value.length > 64 || !isAsciiAlphaNumeric(value.charCodeAt(0))) { + return false; + } + for (let index = 1; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if ( + !isAsciiAlphaNumeric(code) && + code !== 0x2e && + code !== 0x5f && + code !== 0x2f && + code !== 0x2d + ) { + return false; + } + } + return true; +} + +function isSha256Id(value: string): boolean { + if (value.length !== 71 || !value.startsWith("sha256:")) return false; + for (let index = 7; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (!((code >= 0x30 && code <= 0x39) || (code >= 0x61 && code <= 0x66))) return false; + } + return true; +} + +function isCanonicalNonNegativeNumber(value: string): boolean { + if (value.length < 1 || value.length > 64 || value !== value.trim()) return false; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0; +} + +function isCanonicalFrameCount(value: string): boolean { + if (value.length < 1 || value.length > 2) return false; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code < 0x30 || code > 0x39) return false; + } + const parsed = Number(value); + return parsed >= 1 && parsed <= 16; +} + +const SessionIdSchema = z + .string() + .min(1) + .max(128) + .refine(isCanonicalOpaqueId, "sessionId must not contain surrounding whitespace"); +const VideoRefSchema = z + .string() + .min(1) + .max(4096) + .refine(isCanonicalOpaqueId, "videoRef must not contain surrounding whitespace"); +const NonNegativeQueryNumberSchema = z + .string() + .refine(isCanonicalNonNegativeNumber) + .transform(Number); +const FrameCountQuerySchema = z.string().refine(isCanonicalFrameCount).transform(Number); +const DrilldownReadQuerySchema = z + .object({ + end: NonNegativeQueryNumberSchema.optional(), + frames: FrameCountQuerySchema.optional(), + sessionId: SessionIdSchema, + start: NonNegativeQueryNumberSchema.optional(), + videoRef: VideoRefSchema, + }) + .strict(); +const DrilldownDeleteQuerySchema = z.object({ sessionId: SessionIdSchema }).strict(); +const DrilldownDerivationSchema = z + .object({ + parentContentHash: z.string().refine(isSha256Id), + policy: z.string().refine(isDerivationToken), + version: z.string().refine(isDerivationToken), + }) + .strict(); +const DrilldownFrameSchema = z + .object({ + dataUri: z.string().min(1).max(VIDEO_DRILLDOWN_MAX_FRAME_DATA_URI_CHARS), + timestampSeconds: z.number().finite().nonnegative(), + }) + .strict(); +const DrilldownPostBodySchema = z + .object({ + derivation: DrilldownDerivationSchema, + durationSeconds: z.number().finite().positive().max(600), + frames: z.array(DrilldownFrameSchema).min(1).max(16), + sessionId: SessionIdSchema, + videoRef: VideoRefSchema, + }) + .strict() + .superRefine((value, context) => { + for (let index = 0; index < value.frames.length; index += 1) { + if (value.frames[index].timestampSeconds > value.durationSeconds) { + context.addIssue({ + code: "custom", + message: "frame timestamp exceeds duration", + path: ["frames", index, "timestampSeconds"], + }); + } + } + }); const drilldownCache = new VideoDrilldownCache({ maxEntries: 64, - // Global decoded-byte ceiling: without it, 64 entries × 32 MiB could pin ~2 GiB. + maxEntriesPerPrincipal: 16, + maxBytesPerPrincipal: 64 * 1024 * 1024, + // Global retained-JPEG ceiling: without it, 64 entries × 32 MiB could pin ~2 GiB. maxTotalBytes: 256 * 1024 * 1024, ttlMs: 10 * 60 * 1000, }); @@ -30,6 +157,47 @@ function invalid(message: string, status = 400): Response { return createErrorResponse({ status, message, type: "invalid_request" }); } +class VideoDrilldownRequestAbortedError extends Error {} + +function queryRecord(searchParams: URLSearchParams): Record { + const values: Record = {}; + for (const [key, value] of searchParams) { + const existing = values[key]; + values[key] = + existing === undefined + ? value + : Array.isArray(existing) + ? [...existing, value] + : [existing, value]; + } + return values; +} + +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +async function readBodyWithAbort(request: Request): Promise { + if (request.signal.aborted) throw new VideoDrilldownRequestAbortedError(); + return new Promise((resolve, reject) => { + const onAbort = () => { + request.signal.removeEventListener("abort", onAbort); + reject(new VideoDrilldownRequestAbortedError()); + }; + request.signal.addEventListener("abort", onAbort, { once: true }); + request.arrayBuffer().then( + (bytes) => { + request.signal.removeEventListener("abort", onAbort); + resolve(bytes); + }, + (error: unknown) => { + request.signal.removeEventListener("abort", onAbort); + reject(error); + } + ); + }); +} + function parseQuery(url: URL): { endSeconds?: number; frameCount?: number; @@ -37,28 +205,15 @@ function parseQuery(url: URL): { startSeconds?: number; videoRef: string; } | null { - const allowed = new Set(["end", "frames", "sessionId", "start", "videoRef"]); - if ([...url.searchParams.keys()].some((key) => !allowed.has(key))) return null; - const sessionId = url.searchParams.get("sessionId")?.trim() ?? ""; - const videoRef = url.searchParams.get("videoRef")?.trim() ?? ""; - if (!sessionId || !videoRef) return null; - const parseNumber = (name: string): number | undefined | null => { - const value = url.searchParams.get(name); - if (value === null) return undefined; - const parsed = Number(value); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; + const parsed = DrilldownReadQuerySchema.safeParse(queryRecord(url.searchParams)); + if (!parsed.success) return null; + return { + endSeconds: parsed.data.end, + frameCount: parsed.data.frames, + sessionId: parsed.data.sessionId, + startSeconds: parsed.data.start, + videoRef: parsed.data.videoRef, }; - const startSeconds = parseNumber("start"); - const endSeconds = parseNumber("end"); - const rawFrameCount = url.searchParams.get("frames"); - const frameCount = - rawFrameCount === null - ? undefined - : /^\d{1,2}$/.test(rawFrameCount) && Number(rawFrameCount) >= 1 && Number(rawFrameCount) <= 16 - ? Number(rawFrameCount) - : null; - if (startSeconds === null || endSeconds === null || frameCount === null) return null; - return { endSeconds, frameCount, sessionId, startSeconds, videoRef }; } interface VideoDrilldownRouteDependencies { @@ -71,60 +226,72 @@ export async function handleVideoDrilldownRequest( ): Promise { const url = new URL(request.url); if (url.pathname !== expectedPath()) return invalid("Invalid Video Bridge drill-down path", 404); - if (!isVideoBridgeBrokerInternalRequest(request, VIDEO_BRIDGE_BROKER_PATH)) { + const principalId = resolveVideoBridgeDrilldownPrincipal(request); + if (!principalId) { return invalid("This endpoint requires an authenticated internal loopback request", 403); } const cache = dependencies.cache ?? drilldownCache; if (request.method === "GET") { const query = parseQuery(url); if (!query) return invalid("Invalid Video Bridge drill-down query"); - const result = cache.get(query.sessionId, query.videoRef, query); + const result = cache.get(principalId, query.sessionId, query.videoRef, query); return result ? Response.json(result, { headers: { "Cache-Control": "no-store" } }) : invalid("Video Bridge drill-down result was not found", 404); } if (request.method === "DELETE") { - const sessionId = url.searchParams.get("sessionId")?.trim() ?? ""; - if (!sessionId || [...url.searchParams.keys()].some((key) => key !== "sessionId")) { - return invalid("A sessionId is required"); - } - return Response.json({ removed: cache.clearSession(sessionId) }); + const query = DrilldownDeleteQuerySchema.safeParse(queryRecord(url.searchParams)); + if (!query.success) return invalid("A canonical sessionId is required"); + return Response.json({ removed: cache.clearSession(principalId, query.data.sessionId) }); } if (request.method !== "POST") return invalid("Invalid Video Bridge drill-down method", 405); if (request.headers.get("content-type")?.toLowerCase() !== "application/json") { return invalid("Video Bridge drill-down requires application/json"); } const declaredLength = Number(request.headers.get("content-length")); - if (Number.isFinite(declaredLength) && declaredLength > MAX_BODY_BYTES) { + if (Number.isFinite(declaredLength) && declaredLength > VIDEO_DRILLDOWN_MAX_BODY_BYTES) { return invalid("Video Bridge drill-down payload is too large", 413); } let body: unknown; try { - const bytes = await request.arrayBuffer(); - if (bytes.byteLength > MAX_BODY_BYTES) + const bytes = await readBodyWithAbort(request); + if (bytes.byteLength > VIDEO_DRILLDOWN_MAX_BODY_BYTES) return invalid("Video Bridge drill-down payload is too large", 413); body = JSON.parse(Buffer.from(bytes).toString("utf8")); - } catch { + } catch (error: unknown) { + if (error instanceof VideoDrilldownRequestAbortedError) { + return invalid("Video Bridge drill-down request was cancelled", 499); + } return invalid("Video Bridge drill-down payload is invalid"); } - if (!body || typeof body !== "object") - return invalid("Video Bridge drill-down payload is invalid"); - const record = body as Record; - if ( - typeof record.sessionId !== "string" || - typeof record.videoRef !== "string" || - typeof record.durationSeconds !== "number" || - !Array.isArray(record.frames) - ) { - return invalid("Video Bridge drill-down payload is invalid"); + const parsed = DrilldownPostBodySchema.safeParse(body); + if (!parsed.success) return invalid("Video Bridge drill-down payload is invalid"); + await yieldToEventLoop(); + if (request.signal.aborted) { + return invalid("Video Bridge drill-down request was cancelled", 499); } try { - cache.put(record.sessionId, record.videoRef, { - durationSeconds: record.durationSeconds, - frames: record.frames as VideoDrilldownFrame[], + await cache.put(principalId, parsed.data.sessionId, parsed.data.videoRef, parsed.data, { + signal: request.signal, + }); + } catch (error: unknown) { + if (error instanceof VideoDrilldownValidationError) { + return invalid("Video Bridge drill-down payload is invalid"); + } + if (error instanceof VideoDrilldownAbortedError || request.signal.aborted) { + return invalid("Video Bridge drill-down request was cancelled", 499); + } + log.error( + { + errorName: error instanceof Error ? sanitizeErrorMessage(error.name) : "UnknownError", + }, + "Unexpected Video Bridge drill-down cache failure" + ); + return createErrorResponse({ + status: 500, + message: "Video Bridge drill-down could not be stored", + type: "server_error", }); - } catch { - return invalid("Video Bridge drill-down payload is invalid"); } return Response.json({ stored: true }, { status: 201, headers: { "Cache-Control": "no-store" } }); } diff --git a/src/lib/guardrails/videoBridgeBrokerAuth.ts b/src/lib/guardrails/videoBridgeBrokerAuth.ts index d4bdb8489e..c1096d2dde 100644 --- a/src/lib/guardrails/videoBridgeBrokerAuth.ts +++ b/src/lib/guardrails/videoBridgeBrokerAuth.ts @@ -3,7 +3,9 @@ import { randomUUID, timingSafeEqual } from "node:crypto"; import { AUTHZ_HEADER_PEER_LOCALITY } from "@/server/authz/headers"; export const VIDEO_BRIDGE_BROKER_PATH = "/api/modality-bridge/video/extract"; +export const VIDEO_BRIDGE_DRILLDOWN_PATH = "/api/modality-bridge/video/drilldown"; export const VIDEO_BRIDGE_BROKER_AUTH_HEADER = "x-omniroute-video-bridge-broker"; +export const VIDEO_BRIDGE_DRILLDOWN_PRINCIPAL_HEADER = "x-omniroute-video-bridge-principal"; const globalState = globalThis as typeof globalThis & { __omnirouteVideoBridgeBrokerToken?: string; @@ -20,8 +22,26 @@ export function buildVideoBridgeBrokerHeaders(): Record { return { [VIDEO_BRIDGE_BROKER_AUTH_HEADER]: brokerToken() }; } +function normalizeVideoBridgePrincipalId(value: string | null): string | null { + if (!value || value.length > 256) return null; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code < 0x21 || code > 0x7e) return null; + } + return value; +} + +export function buildVideoBridgeDrilldownHeaders(principalId: string): Record { + const normalized = normalizeVideoBridgePrincipalId(principalId); + if (!normalized) throw new Error("Video Bridge drill-down principal is invalid"); + return { + ...buildVideoBridgeBrokerHeaders(), + [VIDEO_BRIDGE_DRILLDOWN_PRINCIPAL_HEADER]: normalized, + }; +} + export function isVideoBridgeBrokerTokenRequest(request: Request, path: string): boolean { - if (path !== VIDEO_BRIDGE_BROKER_PATH) return false; + if (path !== VIDEO_BRIDGE_BROKER_PATH && path !== VIDEO_BRIDGE_DRILLDOWN_PATH) return false; const expected = brokerToken(); const provided = request.headers.get(VIDEO_BRIDGE_BROKER_AUTH_HEADER)?.trim() ?? ""; if (!provided || provided.length !== expected.length) return false; @@ -34,3 +54,10 @@ export function isVideoBridgeBrokerInternalRequest(request: Request, path: strin isVideoBridgeBrokerTokenRequest(request, path) ); } + +export function resolveVideoBridgeDrilldownPrincipal(request: Request): string | null { + if (!isVideoBridgeBrokerInternalRequest(request, VIDEO_BRIDGE_DRILLDOWN_PATH)) return null; + return normalizeVideoBridgePrincipalId( + request.headers.get(VIDEO_BRIDGE_DRILLDOWN_PRINCIPAL_HEADER) + ); +} diff --git a/src/lib/guardrails/videoBridgeContactSheet.ts b/src/lib/guardrails/videoBridgeContactSheet.ts index fc17627553..a0f1d0f1e8 100644 --- a/src/lib/guardrails/videoBridgeContactSheet.ts +++ b/src/lib/guardrails/videoBridgeContactSheet.ts @@ -21,6 +21,9 @@ export interface VideoContactSheetResult { const MAX_FRAMES = 16; const MAX_SHEET_BYTES = 32 * 1024 * 1024; +const LABEL_FONT_SIZE = 32; +const LABEL_HEIGHT = 64; +const LABEL_PADDING = 16; const TILE_SIZE = 512; function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult { @@ -33,11 +36,31 @@ function fallback(frames: readonly ContactSheetFrame[]): VideoContactSheetResult } function decodeFrame(dataUri: string): Buffer { - const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]+)$/i.exec(dataUri); + const match = /^data:image\/jpeg;base64,([A-Za-z0-9+/=]{4,5592408})$/i.exec(dataUri); if (!match) throw new Error("Contact sheet requires JPEG data URIs"); return Buffer.from(match[1], "base64"); } +function formatContactSheetTimestamp(timestampSeconds: number): string { + const totalMilliseconds = Math.max(0, Math.round(timestampSeconds * 1000)); + const minutes = Math.floor(totalMilliseconds / 60_000); + const seconds = Math.floor((totalMilliseconds % 60_000) / 1000); + const milliseconds = totalMilliseconds % 1000; + if (minutes > 999) return `t=${timestampSeconds.toExponential(3)}s`; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(milliseconds).padStart(3, "0")}`; +} + +function buildTimestampLabel(timestampSeconds: number): Buffer { + const label = formatContactSheetTimestamp(timestampSeconds); + const labelTop = TILE_SIZE - LABEL_HEIGHT; + return Buffer.from( + ` + + ${label} + ` + ); +} + /** Build an optional bounded JPEG grid; every failure except abort is fail-safe to individual frames. */ export async function buildVideoContactSheet( frames: readonly ContactSheetFrame[], @@ -69,6 +92,7 @@ export async function buildVideoContactSheet( frames.map(async (frame) => sharp(decodeFrame(frame.dataUri)) .resize(TILE_SIZE, TILE_SIZE, { fit: "contain", background: "#000000" }) + .composite([{ input: buildTimestampLabel(frame.timestampSeconds), left: 0, top: 0 }]) .jpeg({ quality: 80 }) .toBuffer() ) @@ -101,7 +125,7 @@ export async function buildVideoContactSheet( used: true, width: columns * TILE_SIZE, }; - } catch (error) { + } catch { if (signal.aborted) throw new Error("Video contact sheet was aborted"); return fallback(frames); } finally { diff --git a/src/lib/guardrails/videoBridgeDrilldown.ts b/src/lib/guardrails/videoBridgeDrilldown.ts index 330da73a46..e39eaf4a57 100644 --- a/src/lib/guardrails/videoBridgeDrilldown.ts +++ b/src/lib/guardrails/videoBridgeDrilldown.ts @@ -1,18 +1,49 @@ import { createHash } from "node:crypto"; +import sharp from "sharp"; + import { resolveVideoFocusWindow, type VideoFocusWindow } from "./videoBridgeRuntime"; -export interface VideoDrilldownFrame { +export interface VideoDrilldownFrameInput { dataUri: string; timestampSeconds: number; } +export interface VideoDrilldownFrame extends VideoDrilldownFrameInput { + height: number; + width: number; +} + +export interface VideoDrilldownDerivationInput { + parentContentHash: string; + policy: string; + version: string; +} + +export interface VideoDrilldownDerivationMetadata { + contentHash: string; + createdAt: number; + format: "image/jpeg"; + parent: { + contentHash: string; + referenceHash: string; + }; + policy: string; + resolution: { + height: number; + width: number; + }; + version: string; +} + export interface VideoDrilldownPutValue { + derivation: VideoDrilldownDerivationInput; durationSeconds: number; - frames: readonly VideoDrilldownFrame[]; + frames: readonly VideoDrilldownFrameInput[]; } export interface VideoDrilldownResult { + derivation: VideoDrilldownDerivationMetadata; durationSeconds: number; focusWindow?: VideoFocusWindow; frames: VideoDrilldownFrame[]; @@ -20,30 +51,273 @@ export interface VideoDrilldownResult { export interface VideoDrilldownCacheOptions { maxEntries: number; - /** Aggregate decoded-byte budget across every entry; oldest entries are evicted (LRU) to fit. */ + /** Per-principal entry quota, enforced before the global LRU ceiling. */ + maxEntriesPerPrincipal?: number; + /** Per-principal retained-JPEG-byte quota, independent from the global budget. */ + maxBytesPerPrincipal?: number; + /** Aggregate retained-JPEG-byte budget; oldest entries are evicted (LRU) to fit. */ maxTotalBytes?: number; now?: () => number; ttlMs: number; + normalizeJpeg?: VideoDrilldownJpegNormalizer; } -interface StoredDrilldown extends VideoDrilldownPutValue { +export type VideoDrilldownJpegNormalizer = ( + data: Buffer +) => Promise<{ data: Buffer; height: number; width: number }>; + +export class VideoDrilldownValidationError extends Error { + constructor(message: string) { + super(message); + this.name = "VideoDrilldownValidationError"; + } +} + +export class VideoDrilldownAbortedError extends Error { + constructor() { + super("Video Bridge drill-down was aborted"); + this.name = "VideoDrilldownAbortedError"; + } +} + +interface StoredDrilldown { bytes: number; + derivation: VideoDrilldownDerivationMetadata; + durationSeconds: number; expiresAt: number; - sessionId: string; + frames: StoredDrilldownFrame[]; + principalKey: string; + sessionKey: string; } -const MAX_FRAME_BYTES = 4 * 1024 * 1024; -const MAX_TOTAL_BYTES = 32 * 1024 * 1024; +interface StoredDrilldownFrame { + data: Buffer; + height: number; + timestampSeconds: number; + width: number; +} + +export const VIDEO_DRILLDOWN_MAX_FRAME_BYTES = 4 * 1024 * 1024; +export const VIDEO_DRILLDOWN_MAX_ENTRY_BYTES = 32 * 1024 * 1024; const MAX_DURATION_SECONDS = 600; +const MAX_FRAME_DIMENSION = 8192; +const JPEG_DATA_URI_PREFIX = "data:image/jpeg;base64,"; +export const VIDEO_DRILLDOWN_MAX_FRAME_DATA_URI_CHARS = + JPEG_DATA_URI_PREFIX.length + Math.ceil(VIDEO_DRILLDOWN_MAX_FRAME_BYTES / 3) * 4; -function cacheKey(sessionId: string, videoRef: string): string { - return createHash("sha256").update(`${sessionId}\0${videoRef}`).digest("hex"); +function validationFailure(message: string): never { + throw new VideoDrilldownValidationError(message); } -function validateFrames(value: VideoDrilldownPutValue): { - frames: VideoDrilldownFrame[]; +function throwIfAborted(signal?: AbortSignal): void { + if (signal?.aborted) throw new VideoDrilldownAbortedError(); +} + +function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +function isAsciiAlphaNumeric(code: number): boolean { + return ( + (code >= 0x30 && code <= 0x39) || + (code >= 0x41 && code <= 0x5a) || + (code >= 0x61 && code <= 0x7a) + ); +} + +function isDerivationToken(value: string): boolean { + if (value.length < 1 || value.length > 64 || !isAsciiAlphaNumeric(value.charCodeAt(0))) { + return false; + } + for (let index = 1; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if ( + !isAsciiAlphaNumeric(code) && + code !== 0x2e && + code !== 0x5f && + code !== 0x2f && + code !== 0x2d + ) { + return false; + } + } + return true; +} + +function isSha256Id(value: string): boolean { + if (value.length !== 71 || !value.startsWith("sha256:")) return false; + for (let index = 7; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (!((code >= 0x30 && code <= 0x39) || (code >= 0x61 && code <= 0x66))) return false; + } + return true; +} + +function isCanonicalBase64Alphabet(value: string): boolean { + if (value.length < 4 || value.length % 4 !== 0) return false; + const padding = value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0; + const contentLength = value.length - padding; + for (let index = 0; index < contentLength; index += 1) { + const code = value.charCodeAt(index); + if (!isAsciiAlphaNumeric(code) && code !== 0x2b && code !== 0x2f) return false; + } + for (let index = contentLength; index < value.length; index += 1) { + if (value.charCodeAt(index) !== 0x3d) return false; + } + return true; +} + +function digestKey(...parts: readonly string[]): string { + const hash = createHash("sha256"); + for (const part of parts) { + hash + .update(String(Buffer.byteLength(part, "utf8"))) + .update(":") + .update(part); + } + return hash.digest("hex"); +} + +function contentDigest(value: string | Buffer): string { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +function updateHashPart(hash: ReturnType, value: string | Buffer): void { + const bytes = typeof value === "string" ? Buffer.from(value, "utf8") : value; + hash.update(String(bytes.byteLength)).update(":").update(bytes); +} + +function validIdentity(principalId: string, sessionId: string, videoRef?: string): boolean { + return ( + validPrincipal(principalId) && + validOpaqueId(sessionId, 128) && + (videoRef === undefined || validOpaqueId(videoRef, 4096)) + ); +} + +function validPrincipal(principalId: string): boolean { + if (principalId.length < 1 || principalId.length > 256) return false; + for (let index = 0; index < principalId.length; index += 1) { + const code = principalId.charCodeAt(index); + if (code < 0x21 || code > 0x7e) return false; + } + return true; +} + +function validOpaqueId(value: string, maxLength: number): boolean { + return value.length >= 1 && value.length <= maxLength && value === value.trim(); +} + +async function normalizeJpegWithSharp( + data: Buffer +): Promise<{ data: Buffer; height: number; width: number }> { + if ( + data.byteLength < 4 || + data[0] !== 0xff || + data[1] !== 0xd8 || + data[data.byteLength - 2] !== 0xff || + data[data.byteLength - 1] !== 0xd9 + ) { + validationFailure("Invalid drill-down JPEG frame signature"); + } + try { + const image = sharp(data, { + failOn: "warning", + limitInputPixels: MAX_FRAME_DIMENSION * MAX_FRAME_DIMENSION, + sequentialRead: true, + }); + const metadata = await image.metadata(); + const height = metadata.height; + const width = metadata.width; + if ( + metadata.format !== "jpeg" || + !Number.isInteger(width) || + !Number.isInteger(height) || + !width || + !height || + width > MAX_FRAME_DIMENSION || + height > MAX_FRAME_DIMENSION + ) { + validationFailure("Invalid drill-down JPEG frame dimensions"); + } + // A thumbnail decode can stop before the complete entropy scan. Re-encoding the + // full image makes libvips surface scan warnings and strips any bytes trailing the + // source JPEG. Only this canonical compressed output is retained and charged. + const normalized = await image.clone().jpeg({ progressive: false }).toBuffer(); + if ( + normalized.byteLength < 4 || + normalized.byteLength > VIDEO_DRILLDOWN_MAX_FRAME_BYTES || + normalized[0] !== 0xff || + normalized[1] !== 0xd8 || + normalized[normalized.byteLength - 2] !== 0xff || + normalized[normalized.byteLength - 1] !== 0xd9 + ) { + validationFailure("Invalid canonical drill-down JPEG frame"); + } + return { data: normalized, height, width }; + } catch (error: unknown) { + if (error instanceof VideoDrilldownValidationError) throw error; + validationFailure("Invalid drill-down JPEG frame structure"); + } +} + +async function decodeCanonicalJpeg( + dataUri: string, + normalizeJpeg: VideoDrilldownJpegNormalizer, + signal?: AbortSignal +): Promise<{ + data: Buffer; + resolution: { height: number; width: number }; +}> { + throwIfAborted(signal); + if (!dataUri.startsWith(JPEG_DATA_URI_PREFIX)) { + validationFailure("Invalid drill-down JPEG frame"); + } + const encoded = dataUri.slice(JPEG_DATA_URI_PREFIX.length); + if (dataUri.length > VIDEO_DRILLDOWN_MAX_FRAME_DATA_URI_CHARS) { + validationFailure("Drill-down frame byte limit exceeded"); + } + if (!isCanonicalBase64Alphabet(encoded)) { + validationFailure("Drill-down JPEG must use canonical Base64"); + } + const data = Buffer.from(encoded, "base64"); + if (data.toString("base64") !== encoded) { + validationFailure("Drill-down JPEG must use canonical Base64"); + } + if (data.byteLength < 1 || data.byteLength > VIDEO_DRILLDOWN_MAX_FRAME_BYTES) { + validationFailure("Drill-down frame byte limit exceeded"); + } + throwIfAborted(signal); + const normalized = await normalizeJpeg(data); + throwIfAborted(signal); + if ( + !Buffer.isBuffer(normalized.data) || + normalized.data.byteLength < 1 || + normalized.data.byteLength > VIDEO_DRILLDOWN_MAX_FRAME_BYTES || + !Number.isInteger(normalized.width) || + !Number.isInteger(normalized.height) || + normalized.width < 1 || + normalized.height < 1 || + normalized.width > MAX_FRAME_DIMENSION || + normalized.height > MAX_FRAME_DIMENSION + ) { + validationFailure("Invalid canonical drill-down JPEG frame"); + } + return { + data: normalized.data, + resolution: { height: normalized.height, width: normalized.width }, + }; +} + +async function validateFrames( + value: VideoDrilldownPutValue, + normalizeJpeg: VideoDrilldownJpegNormalizer, + signal?: AbortSignal +): Promise<{ + frames: StoredDrilldownFrame[]; + resolution: { height: number; width: number }; totalBytes: number; -} { +}> { if ( !Number.isFinite(value.durationSeconds) || value.durationSeconds <= 0 || @@ -52,36 +326,112 @@ function validateFrames(value: VideoDrilldownPutValue): { value.frames.length < 1 || value.frames.length > 16 ) { - throw new Error("Invalid drill-down duration or frame count"); + validationFailure("Invalid drill-down duration or frame count"); } let totalBytes = 0; - const frames = value.frames.map((frame) => { + let resolution: { height: number; width: number } | undefined; + const frames: StoredDrilldownFrame[] = []; + for (const frame of value.frames) { + throwIfAborted(signal); if ( !frame || !Number.isFinite(frame.timestampSeconds) || frame.timestampSeconds < 0 || frame.timestampSeconds > value.durationSeconds || - !/^data:image\/jpeg;base64,[A-Za-z0-9+/=]+$/i.test(frame.dataUri) + typeof frame.dataUri !== "string" ) { - throw new Error("Invalid drill-down JPEG frame"); + validationFailure("Invalid drill-down JPEG frame"); } - const encoded = frame.dataUri.slice(frame.dataUri.indexOf(",") + 1); - const bytes = Math.floor((encoded.length * 3) / 4); - if (bytes < 1 || bytes > MAX_FRAME_BYTES) - throw new Error("Drill-down frame byte limit exceeded"); + const decoded = await decodeCanonicalJpeg(frame.dataUri, normalizeJpeg, signal); + if ( + resolution && + (resolution.height !== decoded.resolution.height || + resolution.width !== decoded.resolution.width) + ) { + validationFailure("Drill-down frames must use one auditable resolution"); + } + resolution ??= decoded.resolution; + const bytes = decoded.data.byteLength; totalBytes += bytes; - if (totalBytes > MAX_TOTAL_BYTES) throw new Error("Drill-down response byte limit exceeded"); - return { dataUri: frame.dataUri, timestampSeconds: frame.timestampSeconds }; - }); + if (totalBytes > VIDEO_DRILLDOWN_MAX_ENTRY_BYTES) { + validationFailure("Drill-down response byte limit exceeded"); + } + frames.push({ + data: decoded.data, + height: decoded.resolution.height, + timestampSeconds: frame.timestampSeconds, + width: decoded.resolution.width, + }); + } + const sortedFrames = frames.sort((left, right) => left.timestampSeconds - right.timestampSeconds); + if (!resolution) validationFailure("Invalid drill-down frame resolution"); return { - frames: frames.sort((left, right) => left.timestampSeconds - right.timestampSeconds), + frames: sortedFrames, + resolution, totalBytes, }; } +async function buildDerivationMetadata( + videoRef: string, + value: VideoDrilldownPutValue, + frames: readonly StoredDrilldownFrame[], + resolution: { height: number; width: number }, + createdAt: number, + signal?: AbortSignal +): Promise { + const derivation = value.derivation; + const parentContentHash = derivation?.parentContentHash; + const policy = derivation?.policy; + const version = derivation?.version; + if ( + typeof parentContentHash !== "string" || + !isSha256Id(parentContentHash) || + typeof policy !== "string" || + !isDerivationToken(policy) || + typeof version !== "string" || + !isDerivationToken(version) + ) { + validationFailure("Invalid drill-down derivation metadata"); + } + throwIfAborted(signal); + const hash = createHash("sha256"); + for (const part of [ + "video-drilldown/v1", + parentContentHash, + policy, + version, + String(value.durationSeconds), + ]) { + updateHashPart(hash, part); + } + for (const frame of frames) { + throwIfAborted(signal); + updateHashPart(hash, String(frame.timestampSeconds)); + updateHashPart(hash, `${frame.width}x${frame.height}`); + updateHashPart(hash, frame.data); + await yieldToEventLoop(); + } + throwIfAborted(signal); + return { + contentHash: `sha256:${hash.digest("hex")}`, + createdAt, + format: "image/jpeg", + parent: { + contentHash: parentContentHash, + referenceHash: contentDigest(videoRef), + }, + policy, + resolution: { ...resolution }, + version, + }; +} + export class VideoDrilldownCache { private readonly entries = new Map(); private readonly now: () => number; + private readonly principalUsage = new Map(); + private readonly normalizeJpeg: VideoDrilldownJpegNormalizer; private totalBytes = 0; constructor(private readonly options: VideoDrilldownCacheOptions) { @@ -91,6 +441,18 @@ export class VideoDrilldownCache { if (!Number.isInteger(options.maxEntries) || options.maxEntries < 1) { throw new Error("Drill-down cache entry limit is invalid"); } + if ( + options.maxEntriesPerPrincipal !== undefined && + (!Number.isInteger(options.maxEntriesPerPrincipal) || options.maxEntriesPerPrincipal < 1) + ) { + throw new Error("Drill-down cache principal entry quota is invalid"); + } + if ( + options.maxBytesPerPrincipal !== undefined && + (!Number.isInteger(options.maxBytesPerPrincipal) || options.maxBytesPerPrincipal < 1) + ) { + throw new Error("Drill-down cache principal byte quota is invalid"); + } if ( options.maxTotalBytes !== undefined && (!Number.isInteger(options.maxTotalBytes) || options.maxTotalBytes < 1) @@ -98,6 +460,7 @@ export class VideoDrilldownCache { throw new Error("Drill-down cache byte budget is invalid"); } this.now = options.now ?? Date.now; + this.normalizeJpeg = options.normalizeJpeg ?? normalizeJpegWithSharp; } private drop(key: string): void { @@ -105,26 +468,103 @@ export class VideoDrilldownCache { if (!stored) return; this.entries.delete(key); this.totalBytes -= stored.bytes; + const usage = this.principalUsage.get(stored.principalKey); + if (!usage) return; + usage.bytes -= stored.bytes; + usage.entries -= 1; + if (usage.entries === 0) this.principalUsage.delete(stored.principalKey); } - put(sessionId: string, videoRef: string, value: VideoDrilldownPutValue): void { - if (!sessionId || sessionId.length > 128 || !videoRef || videoRef.length > 4096) { - throw new Error("Drill-down cache key is invalid"); + private addUsage(principalKey: string, bytes: number): void { + const usage = this.principalUsage.get(principalKey) ?? { bytes: 0, entries: 0 }; + usage.bytes += bytes; + usage.entries += 1; + this.principalUsage.set(principalKey, usage); + } + + private sweepExpired(): void { + const now = this.now(); + for (const [key, stored] of this.entries) { + if (stored.expiresAt <= now) this.drop(key); } - const { frames, totalBytes } = validateFrames(value); + } + + private principalExceedsQuota(principalKey: string): boolean { + const usage = this.principalUsage.get(principalKey); + return Boolean( + usage && + ((this.options.maxEntriesPerPrincipal !== undefined && + usage.entries > this.options.maxEntriesPerPrincipal) || + (this.options.maxBytesPerPrincipal !== undefined && + usage.bytes > this.options.maxBytesPerPrincipal)) + ); + } + + private evictOldestForPrincipal(principalKey: string, protectedKey: string): void { + for (const [key, stored] of this.entries) { + if (stored.principalKey === principalKey && key !== protectedKey) { + this.drop(key); + return; + } + } + } + + async put( + principalId: string, + sessionId: string, + videoRef: string, + value: VideoDrilldownPutValue, + requestOptions: { signal?: AbortSignal } = {} + ): Promise { + if (!validIdentity(principalId, sessionId, videoRef)) { + validationFailure("Drill-down cache key is invalid"); + } + this.sweepExpired(); + const signal = requestOptions.signal; + const { frames, resolution, totalBytes } = await validateFrames( + value, + this.normalizeJpeg, + signal + ); if (this.options.maxTotalBytes !== undefined && totalBytes > this.options.maxTotalBytes) { - throw new Error("Drill-down entry exceeds the cache byte budget"); + validationFailure("Drill-down entry exceeds the cache byte budget"); } - const key = cacheKey(sessionId, videoRef); + if ( + this.options.maxBytesPerPrincipal !== undefined && + totalBytes > this.options.maxBytesPerPrincipal + ) { + validationFailure("Drill-down entry exceeds the principal byte quota"); + } + const principalKey = digestKey(principalId); + const sessionKey = digestKey(principalId, sessionId); + const key = digestKey(principalId, sessionId, videoRef); + const createdAt = this.now(); + const derivation = await buildDerivationMetadata( + videoRef, + value, + frames, + resolution, + createdAt, + signal + ); + throwIfAborted(signal); this.drop(key); this.entries.set(key, { bytes: totalBytes, + derivation, durationSeconds: value.durationSeconds, - expiresAt: this.now() + this.options.ttlMs, + expiresAt: createdAt + this.options.ttlMs, frames, - sessionId, + principalKey, + sessionKey, }); this.totalBytes += totalBytes; + this.addUsage(principalKey, totalBytes); + while (this.principalExceedsQuota(principalKey)) { + const previousSize = this.entries.size; + this.evictOldestForPrincipal(principalKey, key); + if (this.entries.size === previousSize) break; + } while ( this.entries.size > this.options.maxEntries || (this.options.maxTotalBytes !== undefined && this.totalBytes > this.options.maxTotalBytes) @@ -136,11 +576,14 @@ export class VideoDrilldownCache { } get( + principalId: string, sessionId: string, videoRef: string, options: { endSeconds?: number; frameCount?: number; startSeconds?: number } = {} ): VideoDrilldownResult | null { - const key = cacheKey(sessionId, videoRef); + if (!validIdentity(principalId, sessionId, videoRef)) return null; + this.sweepExpired(); + const key = digestKey(principalId, sessionId, videoRef); const stored = this.entries.get(key); if (!stored) return null; if (stored.expiresAt <= this.now()) { @@ -178,19 +621,33 @@ export class VideoDrilldownCache { frame.timestampSeconds <= focusWindow.endSeconds) ) .slice(0, frameCount) - .map((frame) => ({ ...frame })); + .map((frame) => ({ + dataUri: `${JPEG_DATA_URI_PREFIX}${frame.data.toString("base64")}`, + height: frame.height, + timestampSeconds: frame.timestampSeconds, + width: frame.width, + })); if (frames.length === 0) return null; return { + derivation: { + ...stored.derivation, + parent: { ...stored.derivation.parent }, + resolution: { ...stored.derivation.resolution }, + }, durationSeconds: stored.durationSeconds, ...(focusWindow ? { focusWindow } : {}), frames, }; } - clearSession(sessionId: string): number { + clearSession(principalId: string, sessionId: string): number { + if (!validIdentity(principalId, sessionId)) return 0; + this.sweepExpired(); + const principalKey = digestKey(principalId); + const sessionKey = digestKey(principalId, sessionId); let removed = 0; for (const [key, entry] of this.entries.entries()) { - if (entry.sessionId === sessionId) { + if (entry.principalKey === principalKey && entry.sessionKey === sessionKey) { this.drop(key); removed += 1; } @@ -198,8 +655,27 @@ export class VideoDrilldownCache { return removed; } + getUsage(principalId: string): { + bytes: number; + entries: number; + totalBytes: number; + totalEntries: number; + } { + this.sweepExpired(); + const usage = validPrincipal(principalId) + ? this.principalUsage.get(digestKey(principalId)) + : undefined; + return { + bytes: usage?.bytes ?? 0, + entries: usage?.entries ?? 0, + totalBytes: this.totalBytes, + totalEntries: this.entries.size, + }; + } + clearAll(): void { this.entries.clear(); + this.principalUsage.clear(); this.totalBytes = 0; } } diff --git a/src/lib/guardrails/videoBridgeRuntime.ts b/src/lib/guardrails/videoBridgeRuntime.ts index fea9769381..736d240c19 100644 --- a/src/lib/guardrails/videoBridgeRuntime.ts +++ b/src/lib/guardrails/videoBridgeRuntime.ts @@ -69,6 +69,24 @@ export interface VideoSamplingDecision extends VideoSamplingMetadata { timestamps: number[]; } +export interface VideoStructuralInterval { + endSeconds: number; + startSeconds: number; +} +export interface VideoStructuralSample { + blur?: number | null; + brightness?: number | null; + sceneScore?: number | null; + spatialInformation?: number | null; + temporalInformation?: number | null; + timestampSeconds: number; +} +export interface VideoStructuralAnalysis { + freezeIntervals: VideoStructuralInterval[]; + samples: VideoStructuralSample[]; + sceneCandidates: number[]; +} + export function resolveVideoFocusWindow( durationSeconds: number, bounds: VideoFocusBounds @@ -95,6 +113,10 @@ export const VIDEO_FRAME_MAX_BYTES = 4 * 1024 * 1024; export const VIDEO_FRAMES_TOTAL_MAX_BYTES = 23 * 1024 * 1024; export const VIDEO_MAX_DIMENSION = 8_192; export const VIDEO_MAX_PIXELS = 33_554_432; +const VIDEO_STRUCTURAL_ANALYSIS_FPS = 1; +const VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES = 600; +const VIDEO_STRUCTURAL_ANALYSIS_MAX_WIDTH = 320; +const VIDEO_STRUCTURAL_SCENE_THRESHOLD = 10; const SAFE_FORMATS = new Set([ "3g2", @@ -111,7 +133,6 @@ const SAFE_FORMATS = new Set([ "webm", ]); const SAFE_FORMAT_WHITELIST = [...SAFE_FORMATS].join(","); - const defaultRunner: VideoCommandRunner = async (executable, args, options) => { const result = await execFileAsync(executable, [...args], { encoding: "utf8", @@ -122,7 +143,6 @@ const defaultRunner: VideoCommandRunner = async (executable, args, options) => { }); return { stdout: String(result.stdout), stderr: String(result.stderr) }; }; - function assertLocalPath(filePath: string): void { if (!isAbsolute(filePath) || filePath.includes("\0") || filePath.includes("://")) { throw new Error("Video runtime requires a local path"); @@ -223,7 +243,6 @@ function normalizeSceneCandidates( } return [...unique].sort((left, right) => left - right); } - export function parseSceneChangeTimestamps(output: string, durationSeconds: number): number[] { const candidates: number[] = []; const timestampPattern = /\bpts_time:([+-]?(?:\d+(?:\.\d*)?|\.\d+))\b/g; @@ -233,67 +252,256 @@ export function parseSceneChangeTimestamps(output: string, durationSeconds: numb } return normalizeSceneCandidates(durationSeconds, candidates); } - -/** Allocate midpoint samples proportionally across validated scene segments. */ +const STRUCTURAL_METRIC_FIELDS = { + "lavfi.blur": "blur", + "lavfi.scd.score": "sceneScore", + "lavfi.signalstats.YAVG": "brightness", + "lavfi.siti.si": "spatialInformation", + "lavfi.siti.ti": "temporalInformation", +} as const; +function parseStructuralSamples(output: string, durationSeconds: number): VideoStructuralSample[] { + const samples = new Map(); + const pattern = /\bpts_time:([+-]?(?:\d+(?:\.\d*)?|\.\d+))[^\n]*\r?\n([A-Za-z0-9_.]+)=([^\s]+)/g; + for (const match of output.matchAll(pattern)) { + const timestamp = Number(Number(match[1]).toFixed(3)); + const field = STRUCTURAL_METRIC_FIELDS[match[2] as keyof typeof STRUCTURAL_METRIC_FIELDS]; + const metric = Number(match[3]); + const unusable = !field || timestamp < 0 || timestamp >= durationSeconds; + if (unusable || (!Number.isFinite(metric) && !samples.has(timestamp))) continue; + if (!samples.has(timestamp)) { + if (samples.size >= VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES) continue; + samples.set(timestamp, { + timestampSeconds: timestamp, + }); + } + const sample = samples.get(timestamp); + if (sample) sample[field] = Number.isFinite(metric) ? metric : null; + } + return [...samples.values()].sort( + (left, right) => left.timestampSeconds - right.timestampSeconds + ); +} +function parseStructuralMetricEvents(output: string, metric: string): number[] { + const pattern = new RegExp(`${metric}:\\s*([+-]?(?:\\d+(?:\\.\\d*)?|\\.\\d+))`, "g"); + return [...output.matchAll(pattern)].map((match) => Number(match[1])).filter(Number.isFinite); +} +function parseFreezeIntervals(output: string, durationSeconds: number): VideoStructuralInterval[] { + const starts = parseStructuralMetricEvents(output, "freeze_start"); + const ends = parseStructuralMetricEvents(output, "freeze_end"); + const durations = parseStructuralMetricEvents(output, "freeze_duration"); + return starts + .map((start, index) => { + const startSeconds = Math.max(0, Math.min(durationSeconds, start)); + const inferredEnd = start + (durations[index] ?? durationSeconds - start); + const endSeconds = Math.max( + startSeconds, + Math.min(durationSeconds, ends[index] ?? inferredEnd) + ); + return { endSeconds, startSeconds }; + }) + .filter((interval) => interval.endSeconds - interval.startSeconds >= 1); +} +export function parseVideoStructuralAnalysis( + metadataOutput: string, + diagnosticOutput: string, + durationSeconds: number +): VideoStructuralAnalysis { + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + throw new Error("Video structural analysis requires a positive duration"); + } + const samples = parseStructuralSamples(metadataOutput, durationSeconds); + const diagnosticScenes = [ + ...diagnosticOutput.matchAll(/lavfi\.scd\.score:\s*[\d.]+,\s*lavfi\.scd\.time:\s*([\d.]+)/g), + ].map((match) => Number(match[1])); + return { + freezeIntervals: parseFreezeIntervals(diagnosticOutput, durationSeconds), + samples, + sceneCandidates: normalizeSceneCandidates(durationSeconds, [ + ...samples + .filter((sample) => (sample.sceneScore ?? 0) >= VIDEO_STRUCTURAL_SCENE_THRESHOLD) + .map((sample) => sample.timestampSeconds), + ...diagnosticScenes, + ]), + }; +} +interface StructuralSamplingSegment { + endSeconds: number; + frozen: boolean; + priority: number; + startSeconds: number; +} +function averageStructuralMetric(values: Array): number | null { + const finite = values.filter( + (value): value is number => value !== null && value !== undefined && Number.isFinite(value) + ); + return finite.length > 0 ? finite.reduce((sum, value) => sum + value, 0) / finite.length : null; +} +function normalizedStructuralMetric( + samples: readonly VideoStructuralSample[], + field: Exclude, + fallback: number, + scale: number +): number { + return Math.min( + 1, + Math.max( + 0, + (averageStructuralMetric(samples.map((sample) => sample[field])) ?? fallback) / scale + ) + ); +} +function structuralSegmentPriority( + startSeconds: number, + endSeconds: number, + analysis: VideoStructuralAnalysis +): StructuralSamplingSegment { + const length = endSeconds - startSeconds; + const samples = analysis.samples.filter( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); + const freezeCoverage = Math.min( + 1, + analysis.freezeIntervals.reduce( + (sum, interval) => + sum + + Math.max( + 0, + Math.min(endSeconds, interval.endSeconds) - Math.max(startSeconds, interval.startSeconds) + ), + 0 + ) / length + ); + const spatial = normalizedStructuralMetric(samples, "spatialInformation", 40, 100); + const temporal = normalizedStructuralMetric(samples, "temporalInformation", 10, 30); + const sharpness = 1 - normalizedStructuralMetric(samples, "blur", 10, 20); + const brightness = averageStructuralMetric(samples.map((sample) => sample.brightness)); + const exposure = brightness === null || (brightness >= 24 && brightness <= 232) ? 1 : 0.25; + const interest = exposure * (0.2 + spatial * 0.3 + temporal * 0.4 + sharpness * 0.1); + const maxTemporal = Math.max(0, ...samples.map((sample) => sample.temporalInformation ?? 0)); + return { + endSeconds, + frozen: freezeCoverage >= 0.8 && maxTemporal <= 1, + priority: length * Math.max(0.05, interest) * (1 - freezeCoverage * 0.75), + startSeconds, + }; +} +function allocateStructuralFrames( + segments: readonly StructuralSamplingSegment[], + frameCount: number +): number[] { + if (segments.length > frameCount) return segments.map(() => 0); + const allocation = segments.map(() => 1); + let remaining = frameCount - segments.length; + const totalPriority = segments.reduce( + (sum, segment) => sum + (segment.frozen ? 0 : segment.priority), + 0 + ); + if (totalPriority <= 0) return allocation; + const idealExtras = segments.map((segment) => + segment.frozen ? 0 : (segment.priority / totalPriority) * remaining + ); + const extras = idealExtras.map((value) => Math.floor(value)); + remaining -= extras.reduce((sum, value) => sum + value, 0); + const remainderOrder = idealExtras + .map((value, index) => ({ index, remainder: value - Math.floor(value) })) + .sort((left, right) => right.remainder - left.remainder || left.index - right.index); + for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1; + return allocation.map((value, index) => value + extras[index]); +} +function timestampsFromSegmentAllocation( + segments: readonly Pick[], + allocation: readonly number[] +): number[] { + return segments.flatMap((segment, segmentIndex) => + Array.from( + { length: allocation[segmentIndex] }, + (_unused, index) => + segment.startSeconds + + ((index + 0.5) * (segment.endSeconds - segment.startSeconds)) / allocation[segmentIndex] + ) + ); +} +function calculateLengthWeightedSegmentTimestamps( + startSeconds: number, + endSeconds: number, + frameCount: number, + boundaries: readonly number[] +): number[] { + const uniform = calculateFrameTimestamps(endSeconds - startSeconds, frameCount).map( + (timestamp) => timestamp + startSeconds + ); + const starts = [startSeconds, ...boundaries]; + const ends = [...boundaries, endSeconds]; + const segments = starts.map((start, index) => ({ + endSeconds: ends[index], + frozen: false, + priority: ends[index] - start, + startSeconds: start, + })); + return segments.length > frameCount + ? uniform + : timestampsFromSegmentAllocation(segments, allocateStructuralFrames(segments, frameCount)); +} +/** Allocate a bounded caption budget across validated structural segments. */ export function calculateSegmentAwareTimestamps( durationSeconds: number, requestedFrameCount: number, sceneCandidates: readonly number[], - focusWindow: VideoFocusWindow | null = null + focusWindow: VideoFocusWindow | null = null, + structuralAnalysis: VideoStructuralAnalysis | null = null ): number[] { const startSeconds = focusWindow?.startSeconds ?? 0; const endSeconds = focusWindow?.endSeconds ?? durationSeconds; const uniform = calculateFrameTimestamps(endSeconds - startSeconds, requestedFrameCount).map( (timestamp) => timestamp + startSeconds ); - const boundaries = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter( - (timestamp) => timestamp > startSeconds && timestamp < endSeconds + const structuralBoundaries = structuralAnalysis?.freezeIntervals.flatMap((interval) => [ + interval.startSeconds, + interval.endSeconds, + ]); + const sceneBoundaries = sceneCandidates.filter( + (candidate) => + !structuralBoundaries?.some( + (boundary) => Math.abs(candidate - boundary) <= 1 / VIDEO_STRUCTURAL_ANALYSIS_FPS + ) ); - if (boundaries.length === 0) return uniform; - const segmentStarts = [startSeconds, ...boundaries]; - const segmentEnds = [...boundaries, endSeconds]; - const lengths = segmentStarts.map((segmentStart, index) => segmentEnds[index] - segmentStart); - const segmentCount = lengths.length; - const frameCount = uniform.length; - if (segmentCount > frameCount) { - return [...uniform].map((timestamp, index) => { - const segmentIndex = Math.min( - segmentCount - 1, - Math.floor((index * segmentCount) / frameCount) - ); - const segmentStart = segmentStarts[segmentIndex]; - const segmentEnd = segmentEnds[segmentIndex]; - return segmentStart + (segmentEnd - segmentStart) / 2; - }); + const boundaries = normalizeSceneCandidates(durationSeconds, [ + ...sceneBoundaries, + ...(structuralBoundaries ?? []), + ]).filter((timestamp) => timestamp > startSeconds && timestamp < endSeconds); + if (!structuralAnalysis) { + return boundaries.length === 0 + ? uniform + : calculateLengthWeightedSegmentTimestamps( + startSeconds, + endSeconds, + uniform.length, + boundaries + ); } - const allocation = lengths.map(() => 1); - let remaining = frameCount - segmentCount; - const idealExtra = lengths.map((length) => (length / (endSeconds - startSeconds)) * remaining); - const extras = idealExtra.map((value) => Math.floor(value)); - remaining -= extras.reduce((sum, value) => sum + value, 0); - const remainderOrder = idealExtra - .map((value, index) => ({ index, remainder: value - Math.floor(value) })) - .sort((left, right) => right.remainder - left.remainder || left.index - right.index); - for (let index = 0; index < remaining; index++) extras[remainderOrder[index].index] += 1; - for (let index = 0; index < allocation.length; index++) allocation[index] += extras[index]; - const timestamps: number[] = []; - for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) { - const count = allocation[segmentIndex]; - const segmentStart = segmentStarts[segmentIndex]; - const segmentLength = lengths[segmentIndex]; - for (let index = 0; index < count; index++) { - timestamps.push(segmentStart + ((index + 0.5) * segmentLength) / count); - } + const starts = [startSeconds, ...boundaries]; + const ends = [...boundaries, endSeconds]; + const segments = starts.map((start, index) => + structuralSegmentPriority(start, ends[index], structuralAnalysis) + ); + const allocation = allocateStructuralFrames(segments, uniform.length); + if (segments.length > uniform.length) { + return calculateLengthWeightedSegmentTimestamps( + startSeconds, + endSeconds, + uniform.length, + boundaries + ); } - return timestamps; + return timestampsFromSegmentAllocation(segments, allocation); } - export function calculateSamplingDecision( durationSeconds: number, requestedFrameCount: number, policy: VideoSamplingPolicy, sceneCandidates: readonly number[] = [], - focusWindow: VideoFocusWindow | null = null + focusWindow: VideoFocusWindow | null = null, + structuralAnalysis: VideoStructuralAnalysis | null = null ): VideoSamplingDecision { const startSeconds = focusWindow?.startSeconds ?? 0; const endSeconds = focusWindow?.endSeconds ?? durationSeconds; @@ -309,21 +517,17 @@ export function calculateSamplingDecision( timestamps: uniform, }; } - const candidates = normalizeSceneCandidates(durationSeconds, sceneCandidates).filter( - (timestamp) => timestamp >= startSeconds && timestamp < endSeconds + (timestamp) => timestamp > startSeconds && timestamp < endSeconds ); - if (candidates.length === 0) { - return { - candidateCount: 0, - ...(focusWindow ? { focusWindow } : {}), - policyEffective: "uniform", - policyRequested: policy, - timestamps: uniform, - }; - } - - if (policy === "segment_aware") { + const focusHasSample = structuralAnalysis?.samples.some( + (sample) => sample.timestampSeconds >= startSeconds && sample.timestampSeconds < endSeconds + ); + const focusHasFreeze = structuralAnalysis?.freezeIntervals.some( + (interval) => interval.startSeconds < endSeconds && interval.endSeconds > startSeconds + ); + const hasStructuralEvidence = Boolean(focusHasSample || focusHasFreeze); + if (policy === "segment_aware" && (candidates.length > 0 || hasStructuralEvidence)) { return { candidateCount: candidates.length, ...(focusWindow ? { focusWindow } : {}), @@ -333,12 +537,30 @@ export function calculateSamplingDecision( durationSeconds, requestedFrameCount, candidates, - focusWindow + focusWindow, + structuralAnalysis ), }; } - + if (candidates.length === 0) { + return { + candidateCount: 0, + ...(focusWindow ? { focusWindow } : {}), + policyEffective: "uniform", + policyRequested: policy, + timestamps: uniform, + }; + } const frameCount = uniform.length; + if (frameCount === 1) { + return { + candidateCount: candidates.length, + ...(focusWindow ? { focusWindow } : {}), + policyEffective: "uniform", + policyRequested: "scene_aware", + timestamps: uniform, + }; + } const selected = candidates.length <= frameCount ? [...candidates] @@ -369,7 +591,6 @@ export function calculateSamplingDecision( timestamps: selected, }; } - export async function detectSceneChangeTimestamps( inputPath: string, options: { @@ -412,7 +633,72 @@ export async function detectSceneChangeTimestamps( ); return parseSceneChangeTimestamps(`${result.stdout}\n${result.stderr}`, options.durationSeconds); } - +const STRUCTURAL_ANALYSIS_FILTER = [ + `scale=w='min(${VIDEO_STRUCTURAL_ANALYSIS_MAX_WIDTH},iw)':h=-2:flags=fast_bilinear`, + `scdet=threshold=${VIDEO_STRUCTURAL_SCENE_THRESHOLD}`, + "freezedetect=n=-60dB:d=1", + `fps=${VIDEO_STRUCTURAL_ANALYSIS_FPS}`, + "siti", + "blurdetect=radius=10:block_width=32:block_height=32", + "signalstats", + ...[ + "lavfi.scd.score", + "lavfi.siti.si", + "lavfi.siti.ti", + "lavfi.blur", + "lavfi.signalstats.YAVG", + ].map((key) => `metadata=mode=print:key=${key}:file=-`), +].join(","); +export async function analyzeVideoStructure( + inputPath: string, + options: { + durationSeconds: number; + runner?: VideoCommandRunner; + signal?: AbortSignal; + streamIndex: number; + timeoutMs?: number; + } +): Promise { + assertLocalPath(inputPath); + if (!Number.isFinite(options.durationSeconds) || options.durationSeconds <= 0) { + throw new Error("Video structural analysis requires a positive duration"); + } + if (!Number.isInteger(options.streamIndex) || options.streamIndex < 0) { + throw new Error("Video stream index is invalid"); + } + const result = await (options.runner ?? defaultRunner)( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "info", + "-nostats", + "-protocol_whitelist", + "file", + "-format_whitelist", + SAFE_FORMAT_WHITELIST, + "-threads", + "1", + "-filter_threads", + "1", + "-i", + inputPath, + "-map", + `0:${options.streamIndex}`, + "-vf", + STRUCTURAL_ANALYSIS_FILTER, + "-an", + "-frames:v", + String(VIDEO_STRUCTURAL_ANALYSIS_MAX_SAMPLES), + "-f", + "null", + "-", + ], + { signal: options.signal, timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000) } + ); + return parseVideoStructuralAnalysis(result.stdout, result.stderr, options.durationSeconds); +} export async function probeLocalVideo( inputPath: string, options: { @@ -547,18 +833,31 @@ export async function extractFramesFromLocalVideo( assertLocalPath(outputDirectory); const policy = options.samplingPolicy ?? "uniform"; let sceneCandidates: number[] = []; + let structuralAnalysis: VideoStructuralAnalysis | null = null; if (policy !== "uniform") { try { - sceneCandidates = await detectSceneChangeTimestamps(inputPath, { - durationSeconds: options.durationSeconds, - runner: options.runner, - signal: options.signal, - streamIndex: options.streamIndex, - timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), - }); + if (policy === "segment_aware") { + structuralAnalysis = await analyzeVideoStructure(inputPath, { + durationSeconds: options.durationSeconds, + runner: options.runner, + signal: options.signal, + streamIndex: options.streamIndex, + timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), + }); + sceneCandidates = structuralAnalysis.sceneCandidates; + } else { + sceneCandidates = await detectSceneChangeTimestamps(inputPath, { + durationSeconds: options.durationSeconds, + runner: options.runner, + signal: options.signal, + streamIndex: options.streamIndex, + timeoutMs: Math.min(options.timeoutMs ?? 30_000, 30_000), + }); + } } catch { if (options.signal?.aborted) throw new Error("Video extraction request aborted"); sceneCandidates = []; + structuralAnalysis = null; } } const focusWindow = options.focusWindow @@ -569,7 +868,8 @@ export async function extractFramesFromLocalVideo( options.frameCount, policy, sceneCandidates, - focusWindow + focusWindow, + structuralAnalysis ); if (!Number.isInteger(options.streamIndex) || options.streamIndex < 0) { throw new Error("Video stream index is invalid"); diff --git a/src/server/authz/policies/management.ts b/src/server/authz/policies/management.ts index 772c801247..0ab352b84a 100644 --- a/src/server/authz/policies/management.ts +++ b/src/server/authz/policies/management.ts @@ -15,6 +15,7 @@ import { evaluateAccessTokenAuth } from "../accessTokenAuth"; import { isInternalServiceRequest } from "../../../lib/api/internalServiceAuth"; import { VIDEO_BRIDGE_BROKER_PATH, + VIDEO_BRIDGE_DRILLDOWN_PATH, isVideoBridgeBrokerTokenRequest, } from "../../../lib/guardrails/videoBridgeBrokerAuth"; import { CLI_TOKEN_HEADER, PEER_IP_HEADER, VIA_PROXY_HEADER } from "../headers"; @@ -246,19 +247,20 @@ export const managementPolicy: RoutePolicy = { return allow({ kind: "management_key", id: "model-sync", label: "internal-model-sync" }); } - // Exact-path, per-process authenticated self-hop used by the public Video - // Bridge guardrail. The unconditional LOCAL_ONLY gate above has already - // rejected remote peers; this carve-out is deliberately not valid for the - // adjacent runtime-status route or any future child path. + // Exact-path, per-process authenticated self-hops used by the public Video + // Bridge guardrail and its isolated drill-down lifecycle. The unconditional + // LOCAL_ONLY gate above has already rejected remote peers; this carve-out is + // deliberately not valid for runtime status or any future adjacent path. if ( - path === VIDEO_BRIDGE_BROKER_PATH && + (path === VIDEO_BRIDGE_BROKER_PATH || path === VIDEO_BRIDGE_DRILLDOWN_PATH) && isLoopbackRequest(ctx) && isVideoBridgeBrokerTokenRequest(ctx.request as unknown as Request, path) ) { + const drilldown = path === VIDEO_BRIDGE_DRILLDOWN_PATH; return allow({ kind: "management_key", - id: "video-bridge-broker", - label: "internal-video-bridge-broker", + id: drilldown ? "video-bridge-drilldown" : "video-bridge-broker", + label: drilldown ? "internal-video-bridge-drilldown" : "internal-video-bridge-broker", }); } diff --git a/tests/integration/video-bridge-sampler-ffmpeg.test.ts b/tests/integration/video-bridge-sampler-ffmpeg.test.ts new file mode 100644 index 0000000000..c33bb770d9 --- /dev/null +++ b/tests/integration/video-bridge-sampler-ffmpeg.test.ts @@ -0,0 +1,223 @@ +/** + * Real FFmpeg fixture gate for the scene-aware Video Bridge sampler. + * + * Run explicitly because FFmpeg is an optional operational dependency: + * RUN_VIDEO_BRIDGE_FFMPEG=1 node --import tsx/esm --test \ + * tests/integration/video-bridge-sampler-ffmpeg.test.ts + */ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { + extractVideoFramesFromBytes, + type VideoCommandRunner, +} from "../../src/lib/guardrails/videoBridgeRuntime.ts"; + +const execFileAsync = promisify(execFile); +const REAL_FFMPEG_ENABLED = process.env.RUN_VIDEO_BRIDGE_FFMPEG === "1"; +const REAL_FFMPEG_SKIP = REAL_FFMPEG_ENABLED + ? false + : "Set RUN_VIDEO_BRIDGE_FFMPEG=1 to run the real FFmpeg fixture matrix"; + +const realRunner: VideoCommandRunner = async (executable, args, options) => { + const result = await execFileAsync(executable, [...args], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + windowsHide: true, + }); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; +}; + +async function createFixture( + directory: string, + name: string, + inputArgs: readonly string[], + videoFilter: string +): Promise { + const outputPath = join(directory, `${name}.mkv`); + await realRunner( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + ...inputArgs, + "-vf", + videoFilter, + "-c:v", + "ffv1", + "-y", + outputPath, + ], + { timeoutMs: 30_000 } + ); + return readFile(outputPath); +} + +async function createRapidEdgeCutFixture(directory: string): Promise { + const outputPath = join(directory, "rapid-edge-cuts.mkv"); + await realRunner( + "ffmpeg", + [ + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x64:r=10:d=0.2", + "-f", + "lavfi", + "-i", + "color=c=black:s=64x64:r=10:d=2.6", + "-f", + "lavfi", + "-i", + "color=c=white:s=64x64:r=10:d=0.2", + "-filter_complex", + "[0:v][1:v][2:v]concat=n=3:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "ffv1", + "-y", + outputPath, + ], + { timeoutMs: 30_000 } + ); + return readFile(outputPath); +} + +async function sample(bytes: Buffer, frameCount: number, runner = realRunner) { + return extractVideoFramesFromBytes(bytes, { + frameCount, + maxDurationSeconds: 600, + runner, + samplingPolicy: "scene_aware", + timeoutMs: 30_000, + }); +} + +test( + "scene-aware sampling handles the canonical real FFmpeg fixture matrix", + { skip: REAL_FFMPEG_SKIP }, + async (context) => { + const directory = await mkdtemp(join(tmpdir(), "omniroute-video-sampler-fixtures-")); + context.after(async () => rm(directory, { force: true, recursive: true })); + + const rapidCuts = await createRapidEdgeCutFixture(directory); + await context.test("rapid cuts near both ends retain coverage within the cap", async () => { + const result = await sample(rapidCuts, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.2, 0.5, 2.8] + ); + assert.deepEqual(result.sampling, { + candidateCount: 2, + policyEffective: "scene_aware", + policyRequested: "scene_aware", + }); + assert.ok(result.frames.length <= 16); + }); + + await context.test("one frame falls back to the full-window midpoint", async () => { + const result = await sample(rapidCuts, 1); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [1.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 2, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + }); + + const staticVideo = await createFixture( + directory, + "static", + ["-f", "lavfi", "-i", "color=c=blue:s=64x64:r=10:d=4"], + "format=yuv420p" + ); + await context.test("a static scene falls back to uniform midpoints", async () => { + const result = await sample(staticVideo, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 0, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + }); + + const slowChange = await createFixture( + directory, + "slow-change", + ["-f", "lavfi", "-i", "nullsrc=s=64x64:r=10:d=4"], + "geq=lum='clip(16+200*T/4,16,235)':cb=128:cr=128,format=yuv420p" + ); + await context.test("a gradual luminance change does not become a false scene cut", async () => { + const result = await sample(slowChange, 4); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.equal(result.sampling.candidateCount, 0); + assert.equal(result.sampling.policyEffective, "uniform"); + }); + + const shortVideo = await createFixture( + directory, + "short", + ["-f", "lavfi", "-i", "color=c=yellow:s=64x64:r=10:d=0.4"], + "format=yuv420p" + ); + await context.test("a sub-second clip remains deterministic and bounded", async () => { + const result = await sample(shortVideo, 8); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.2] + ); + assert.equal(result.sampling.policyEffective, "uniform"); + }); + + await context.test( + "a detector failure falls back while real frame extraction continues", + async () => { + const detectorFailureRunner: VideoCommandRunner = async (executable, args, options) => { + if (args.some((arg) => arg.includes("showinfo"))) { + throw new Error("fixture scene detector failure"); + } + return realRunner(executable, args, options); + }; + const result = await sample(staticVideo, 4, detectorFailureRunner); + + assert.deepEqual( + result.frames.map((frame) => frame.timestampSeconds), + [0.5, 1.5, 2.5, 3.5] + ); + assert.deepEqual(result.sampling, { + candidateCount: 0, + policyEffective: "uniform", + policyRequested: "scene_aware", + }); + } + ); + } +); diff --git a/tests/unit/guardrails/videoBridgeContactSheet.test.ts b/tests/unit/guardrails/videoBridgeContactSheet.test.ts index 578baccefb..b8c41c83a0 100644 --- a/tests/unit/guardrails/videoBridgeContactSheet.test.ts +++ b/tests/unit/guardrails/videoBridgeContactSheet.test.ts @@ -15,6 +15,12 @@ async function frame(color: string, timestampSeconds: number) { return { dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, timestampSeconds }; } +function decodeJpegDataUri(dataUri: string): Buffer { + const prefix = "data:image/jpeg;base64,"; + assert.ok(dataUri.toLowerCase().startsWith(prefix), "expected a JPEG data URI"); + return Buffer.from(dataUri.slice(prefix.length), "base64"); +} + test("builds a bounded contact sheet and preserves timestamp labels", async () => { const result = await buildVideoContactSheet([ await frame("red", 1), @@ -28,6 +34,61 @@ test("builds a bounded contact sheet and preserves timestamp labels", async () = assert.equal(result.frames.length, 3); }); +test("renders a high-contrast timestamp label inside every contact-sheet cell", async () => { + const result = await buildVideoContactSheet([ + await frame("white", 1), + await frame("white", 65.25), + await frame("white", 130.5), + await frame("white", 600), + ]); + + assert.equal(result.used, true); + assert.equal(result.width, 1024); + assert.equal(result.height, 1024); + const { data, info } = await sharp(decodeJpegDataUri(result.dataUri ?? "")) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + assert.equal(info.channels, 3); + + const tileSize = 512; + const labelTop = 448; + const labelBottom = 512; + const labelFingerprints: string[] = []; + for (let index = 0; index < 4; index++) { + const tileLeft = (index % 2) * tileSize; + const tileTop = Math.floor(index / 2) * tileSize; + let darkPixels = 0; + let lightPixels = 0; + let contentLightPixels = 0; + const labelBytes: number[] = []; + + for (let y = labelTop; y < labelBottom; y++) { + for (let x = 0; x < tileSize; x++) { + const offset = ((tileTop + y) * info.width + tileLeft + x) * info.channels; + const luminance = (data[offset] + data[offset + 1] + data[offset + 2]) / 3; + if (luminance < 48) darkPixels += 1; + if (luminance > 208) lightPixels += 1; + labelBytes.push(Math.round(luminance)); + } + } + for (let y = 128; y < 384; y++) { + for (let x = 64; x < 448; x++) { + const offset = ((tileTop + y) * info.width + tileLeft + x) * info.channels; + const luminance = (data[offset] + data[offset + 1] + data[offset + 2]) / 3; + if (luminance > 208) contentLightPixels += 1; + } + } + + assert.ok(darkPixels > tileSize * 48, `cell ${index} should have a dark label band`); + assert.ok(lightPixels > 40, `cell ${index} should have light timestamp glyphs`); + assert.ok(contentLightPixels > 90_000, `cell ${index} should preserve visible frame content`); + labelFingerprints.push(Buffer.from(labelBytes).toString("base64")); + } + + assert.equal(new Set(labelFingerprints).size, 4, "each timestamp should render a distinct label"); +}); + test("contact sheet falls back to individual frames when decoding fails", async () => { const frames = [{ dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 2 }]; const result = await buildVideoContactSheet(frames); diff --git a/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts b/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts new file mode 100644 index 0000000000..9872df41e3 --- /dev/null +++ b/tests/unit/guardrails/videoBridgeContactSheetEval.test.ts @@ -0,0 +1,177 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import sharp from "sharp"; + +import { + assessVideoContactSheetPromotion, + createVideoContactSheetEvalHoldReport, + runVideoContactSheetEval, +} from "../../../scripts/perf/video-bridge-contact-sheet-eval.ts"; + +async function evalFrame(color: string, timestampSeconds: number) { + const bytes = await sharp({ + create: { background: color, channels: 3, height: 32, width: 32 }, + }) + .jpeg() + .toBuffer(); + return { + dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, + timestampSeconds, + }; +} + +test("contact-sheet A/B eval remains HOLD when real-model configuration is missing", () => { + const report = createVideoContactSheetEvalHoldReport({ + caseCount: 0, + configurationState: "not-configured", + missingConfiguration: ["OMNIROUTE_API_KEY", "--model"], + }); + + assert.equal(report.schemaVersion, 1); + assert.equal(report.kind, "video-contact-sheet-ab-eval"); + assert.deepEqual(report.execution, { + realModel: false, + state: "not-configured", + }); + assert.deepEqual(report.promotion, { + reasons: ["REAL_MODEL_CONFIGURATION_MISSING"], + status: "HOLD", + }); + assert.deepEqual(report.missingConfiguration, ["OMNIROUTE_API_KEY", "--model"]); + assert.deepEqual(report.results, []); + assert.equal(report.summary, null); +}); + +test("contact-sheet A/B eval becomes eligible only with measured cost gains and retained quality", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 0.9, totalTokens: 1_000 }, + sheet: { latencyMs: 600, qualityScore: 0.9, totalTokens: 600 }, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 1, + minQualityScore: 0.8, + minTokenReductionRatio: 0.01, + }, + }); + + assert.deepEqual(decision, { + metrics: { + latencyReductionRatio: 0.4, + qualityRetention: 1, + tokenReductionRatio: 0.4, + }, + reasons: [], + status: "ELIGIBLE", + }); +}); + +test("contact-sheet A/B promotion remains HOLD for quality loss or absent token evidence", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + sheet: { latencyMs: 500, qualityScore: 0.7, totalTokens: null }, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 0.95, + minQualityScore: 0.8, + minTokenReductionRatio: 0.01, + }, + }); + + assert.equal(decision.status, "HOLD"); + assert.deepEqual(decision.reasons, [ + "QUALITY_SCORE_BELOW_THRESHOLD", + "QUALITY_RETENTION_BELOW_THRESHOLD", + "TOKEN_USAGE_UNAVAILABLE", + ]); + assert.equal(decision.metrics.tokenReductionRatio, null); +}); + +test("contact-sheet A/B promotion rejects zero cost gain even with permissive thresholds", () => { + const decision = assessVideoContactSheetPromotion({ + individual: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + sheet: { latencyMs: 1_000, qualityScore: 1, totalTokens: 1_000 }, + thresholds: { + minLatencyReductionRatio: 0, + minQualityRetention: 1, + minQualityScore: 1, + minTokenReductionRatio: 0, + }, + }); + + assert.equal(decision.status, "HOLD"); + assert.deepEqual(decision.reasons, [ + "LATENCY_REDUCTION_BELOW_THRESHOLD", + "TOKEN_REDUCTION_BELOW_THRESHOLD", + ]); +}); + +test("contact-sheet A/B harness measures real-model calls without storing raw responses", async () => { + const responses = [ + "At 00:01.000 there is a red square.", + "At 00:05.000 there is a blue circle.", + "At 00:01.000 there is a red square; at 00:05.000 there is a blue circle.", + ]; + let requestCount = 0; + const report = await runVideoContactSheetEval({ + config: { + apiKey: "test-only-key", + endpoint: "https://eval.invalid/v1/chat/completions", + model: "vision-eval-model", + }, + fetchImpl: async () => { + const content = responses[requestCount]; + requestCount += 1; + return new Response( + JSON.stringify({ + choices: [{ message: { content } }], + usage: { completion_tokens: 20, prompt_tokens: 80, total_tokens: 100 }, + }), + { headers: { "content-type": "application/json" }, status: 200 } + ); + }, + manifest: { + cases: [ + { + expectedFacts: [ + { + id: "red-square", + requiredTerms: ["red", "square"], + timestampSeconds: 1, + }, + { + id: "blue-circle", + requiredTerms: ["blue", "circle"], + timestampSeconds: 5, + }, + ], + frames: [await evalFrame("red", 1), await evalFrame("blue", 5)], + id: "two-scenes", + prompt: "Describe the visible shape and color at each timestamp.", + }, + ], + id: "contact-sheet-fixture-v1", + schemaVersion: 1, + thresholds: { + minLatencyReductionRatio: 0.01, + minQualityRetention: 1, + minQualityScore: 1, + minTokenReductionRatio: 0.01, + }, + }, + }); + + assert.equal(requestCount, 3); + assert.deepEqual(report.execution, { realModel: true, state: "executed" }); + assert.equal(report.results[0].individual.modelCalls, 2); + assert.equal(report.results[0].individual.totalTokens, 200); + assert.equal(report.results[0].individual.qualityScore, 1); + assert.equal(report.results[0].sheet.modelCalls, 1); + assert.equal(report.results[0].sheet.totalTokens, 100); + assert.equal(report.results[0].sheet.qualityScore, 1); + assert.equal("response" in report.results[0].individual, false); + assert.equal("response" in report.results[0].sheet, false); + assert.match(report.manifestDigest, /^[a-f0-9]{64}$/); + assert.match(report.results[0].individual.responseDigest, /^[a-f0-9]{64}$/); + assert.match(report.results[0].sheet.responseDigest, /^[a-f0-9]{64}$/); +}); diff --git a/tests/unit/guardrails/videoBridgeDrilldown.test.ts b/tests/unit/guardrails/videoBridgeDrilldown.test.ts index 054447d9cb..d0ffa931a5 100644 --- a/tests/unit/guardrails/videoBridgeDrilldown.test.ts +++ b/tests/unit/guardrails/videoBridgeDrilldown.test.ts @@ -1,34 +1,120 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import test from "node:test"; +import sharp from "sharp"; + import { + VideoDrilldownAbortedError, VideoDrilldownCache, type VideoDrilldownFrame, } from "../../../src/lib/guardrails/videoBridgeDrilldown"; -const frames: VideoDrilldownFrame[] = [ - { dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 1 }, - { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, - { dataUri: "data:image/jpeg;base64,Qw==", timestampSeconds: 9 }, -]; - -test("drill-down cache isolates sessions and returns bounded focus slices", () => { - const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); - cache.put("session-a", "video-a", { durationSeconds: 10, frames }); - cache.put("session-b", "video-a", { durationSeconds: 10, frames: [frames[0]] }); - - assert.deepEqual( - cache.get("session-a", "video-a", { endSeconds: 6, frameCount: 2 })?.frames, - frames.slice(0, 2) +const validJpegs = new Map(); +for (const [width, height] of [ + [320, 180], + [640, 360], +] as const) { + validJpegs.set( + `${width}x${height}`, + await sharp({ + create: { width, height, channels: 3, background: { r: 1, g: 1, b: 1 } }, + }) + .jpeg({ progressive: false }) + .toBuffer() ); - assert.equal(cache.get("session-a", "video-b"), null); - assert.equal(cache.get("session-b", "video-a")?.frames.length, 1); +} +const noisyPixels = Buffer.alloc(128 * 128 * 3); +let noiseState = 1; +for (let index = 0; index < noisyPixels.length; index += 1) { + noiseState = (noiseState * 1_664_525 + 1_013_904_223) >>> 0; + noisyPixels[index] = noiseState >>> 24; +} +const noisyJpeg = await sharp(noisyPixels, { + raw: { width: 128, height: 128, channels: 3 }, +}) + .jpeg({ progressive: false, quality: 90 }) + .toBuffer(); + +const frames: VideoDrilldownFrame[] = [ + { dataUri: jpegDataUri(320, 180, 0, 1), height: 180, timestampSeconds: 1, width: 320 }, + { dataUri: jpegDataUri(320, 180, 0, 2), height: 180, timestampSeconds: 5, width: 320 }, + { dataUri: jpegDataUri(320, 180, 0, 3), height: 180, timestampSeconds: 9, width: 320 }, +]; +const derivation = { + parentContentHash: `sha256:${"a".repeat(64)}`, + policy: "focused-window", + version: "video-drilldown/v1", +} as const; + +function jpegDataUri(width: number, height: number, payloadBytes = 0, fill = 0): string { + const base = validJpegs.get(`${width}x${height}`); + if (!base) throw new Error(`Missing valid JPEG fixture for ${width}x${height}`); + if (payloadBytes > 65_531) throw new Error("JPEG fixture comment is too large"); + const bytes = + payloadBytes === 0 + ? base + : Buffer.concat([ + base.subarray(0, -2), + Buffer.from([0xff, 0xfe, (payloadBytes + 2) >> 8, (payloadBytes + 2) & 0xff]), + Buffer.alloc(payloadBytes, fill), + base.subarray(-2), + ]); + return `data:image/jpeg;base64,${bytes.toString("base64")}`; +} + +function retainedBytes(dataUri: string): number { + return Buffer.from(dataUri.slice(dataUri.indexOf(",") + 1), "base64").byteLength; +} + +function retainFixtureJpeg(data: Buffer): Promise<{ data: Buffer; height: number; width: number }> { + return Promise.resolve({ data: Buffer.from(data), height: 180, width: 320 }); +} + +function drilldownValue(inputFrames: readonly VideoDrilldownFrame[]) { + return { derivation, durationSeconds: 10, frames: inputFrames }; +} + +test("drill-down cache denies cross-principal reads and deletes", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + await cache.put("principal-a", "session", "video", drilldownValue(frames)); + + assert.equal(cache.get("principal-b", "session", "video"), null); + assert.equal(cache.clearSession("principal-b", "session"), 0); + assert.equal(cache.get("principal-a", "session", "video")?.frames.length, 3); + assert.equal(cache.clearSession("principal-a", "session"), 1); + assert.equal(cache.get("principal-a", "session", "video"), null); }); -test("drill-down cache clamps a valid focus and preserves timeline metadata", () => { +test("drill-down cache isolates sessions and returns bounded focus slices", async () => { const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); - cache.put("session", "video", { durationSeconds: 10, frames }); - const result = cache.get("session", "video", { + await cache.put("principal", "session-a", "video-a", drilldownValue(frames)); + await cache.put("principal", "session-b", "video-a", drilldownValue([frames[0]])); + + const slice = cache.get("principal", "session-a", "video-a", { + endSeconds: 6, + frameCount: 2, + }); + assert.deepEqual( + slice?.frames.map(({ height, timestampSeconds, width }) => ({ + height, + timestampSeconds, + width, + })), + frames.slice(0, 2).map(({ height, timestampSeconds, width }) => ({ + height, + timestampSeconds, + width, + })) + ); + assert.equal(cache.get("principal", "session-a", "video-b"), null); + assert.equal(cache.get("principal", "session-b", "video-a")?.frames.length, 1); +}); + +test("drill-down cache clamps a valid focus and preserves timeline metadata", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + await cache.put("principal", "session", "video", drilldownValue(frames)); + const result = cache.get("principal", "session", "video", { endSeconds: 100, startSeconds: -4, frameCount: 16, @@ -38,72 +124,419 @@ test("drill-down cache clamps a valid focus and preserves timeline metadata", () assert.equal(result?.frames.length, 3); }); -test("drill-down cache rejects invalid and oversized frame payloads", () => { +test("drill-down cache rejects invalid and oversized frame payloads", async () => { const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); - assert.throws(() => cache.put("session", "video", { durationSeconds: 10, frames: [] }), /frame/i); - assert.throws( - () => - cache.put("session", "video", { - durationSeconds: 10, - frames: [{ dataUri: "data:image/png;base64,QQ==", timestampSeconds: 1 }], - }), + await assert.rejects(cache.put("principal", "session", "video", drilldownValue([])), /frame/i); + await assert.rejects( + cache.put( + "principal", + "session", + "video", + drilldownValue([ + { + dataUri: "data:image/png;base64,QQ==", + height: 180, + timestampSeconds: 1, + width: 320, + }, + ]) + ), /JPEG/i ); }); -test("drill-down cache expires entries and evicts the least recently used key", () => { - let now = 1000; - const cache = new VideoDrilldownCache({ now: () => now, ttlMs: 5000, maxEntries: 1 }); - cache.put("session-a", "video", { durationSeconds: 10, frames }); - cache.put("session-b", "video", { durationSeconds: 10, frames }); - assert.equal(cache.get("session-a", "video"), null); - now = 7000; - assert.equal(cache.get("session-b", "video"), null); +test("drill-down cache rejects non-canonical Base64 before quota accounting", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const padded = `${jpegDataUri(320, 180)}${"=".repeat(1024 * 1024)}`; + + await assert.rejects( + cache.put( + "principal", + "session", + "video", + drilldownValue([{ dataUri: padded, height: 180, timestampSeconds: 1, width: 320 }]) + ), + /canonical Base64/i + ); + assert.deepEqual(cache.getUsage("principal"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); }); -test("drill-down cache enforces a global byte budget with LRU eviction", () => { - const bigFrame = (fill: string): VideoDrilldownFrame => ({ - dataUri: `data:image/jpeg;base64,${fill.repeat(4000)}`, - timestampSeconds: 1, +test("drill-down cache rejects non-JPEG bytes disguised by a JPEG data URI", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const mp4 = Buffer.concat([ + Buffer.from([0, 0, 0, 24]), + Buffer.from("ftypisom", "ascii"), + ]).toString("base64"); + + await assert.rejects( + cache.put( + "principal", + "session", + "video", + drilldownValue([ + { + dataUri: `data:image/jpeg;base64,${mp4}`, + height: 180, + timestampSeconds: 1, + width: 320, + }, + ]) + ), + /JPEG/i + ); +}); + +test("drill-down cache canonicalizes JPEG bytes without retaining a disguised media tail", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const jpeg = validJpegs.get("320x180"); + if (!jpeg) throw new Error("Missing valid JPEG fixture for 320x180"); + const marker = Buffer.from("ftypisom", "ascii"); + const tainted = Buffer.concat([ + jpeg, + Buffer.from([0, 0, 1, 16]), + marker, + Buffer.alloc(256, 0x41), + Buffer.from([0xff, 0xd9]), + ]); + + await cache.put( + "principal", + "session", + "video", + drilldownValue([ + { + dataUri: `data:image/jpeg;base64,${tainted.toString("base64")}`, + height: 180, + timestampSeconds: 1, + width: 320, + }, + ]) + ); + + const result = cache.get("principal", "session", "video"); + assert.equal(result?.frames.length, 1); + const retained = Buffer.from(result?.frames[0].dataUri.split(",", 2)[1] ?? "", "base64"); + assert.equal(retained.includes(marker), false); + assert.ok(retained.byteLength < tainted.byteLength); + assert.equal(retained.subarray(-2).toString("hex"), "ffd9"); + assert.deepEqual( + await sharp(retained) + .metadata() + .then(({ height, width }) => ({ height, width })), + { + height: 180, + width: 320, + } + ); + assert.deepEqual(cache.getUsage("principal"), { + bytes: retained.byteLength, + entries: 1, + totalBytes: retained.byteLength, + totalEntries: 1, }); - // Each entry is ~3000 decoded bytes; the budget fits two entries. +}); + +test("drill-down cache rejects a forged SOI/SOF header without a valid scan and EOI", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const forged = "data:image/jpeg;base64,/9hBQkP/wAAHCAABAAE="; + + await assert.rejects( + cache.put( + "principal", + "session", + "video", + drilldownValue([{ dataUri: forged, height: 1, timestampSeconds: 1, width: 1 }]) + ), + /JPEG/i + ); + assert.deepEqual(cache.getUsage("principal"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down cache rejects a truncated entropy scan even when EOI is reattached", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const truncated = Buffer.concat([ + noisyJpeg.subarray(0, noisyJpeg.byteLength - 34), + Buffer.from([0xff, 0xd9]), + ]); + + await assert.rejects( + cache.put( + "principal", + "session", + "video", + drilldownValue([ + { + dataUri: `data:image/jpeg;base64,${truncated.toString("base64")}`, + height: 128, + timestampSeconds: 1, + width: 128, + }, + ]) + ), + /JPEG/i + ); + assert.deepEqual(cache.getUsage("principal"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down cache derives resolution from JPEG bytes instead of caller metadata", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + await cache.put( + "principal", + "session", + "video", + drilldownValue([{ dataUri: jpegDataUri(640, 360), height: 1, timestampSeconds: 1, width: 1 }]) + ); + + const result = cache.get("principal", "session", "video"); + assert.deepEqual(result?.derivation.resolution, { height: 360, width: 640 }); + assert.deepEqual( + result?.frames.map(({ height, width }) => ({ height, width })), + [{ height: 360, width: 640 }] + ); +}); + +test("drill-down cache expires entries and evicts the least recently used key", async () => { + let now = 1000; + const cache = new VideoDrilldownCache({ now: () => now, ttlMs: 5000, maxEntries: 1 }); + await cache.put("principal", "session-a", "video", drilldownValue(frames)); + await cache.put("principal", "session-b", "video", drilldownValue(frames)); + assert.equal(cache.get("principal", "session-a", "video"), null); + now = 7000; + assert.equal(cache.get("principal", "session-b", "video"), null); +}); + +test("drill-down cache sweeps all expired entries from principal and global usage", async () => { + let now = 1000; + const cache = new VideoDrilldownCache({ now: () => now, ttlMs: 5000, maxEntries: 4 }); + await cache.put("principal-a", "session", "video", drilldownValue(frames)); + await cache.put("principal-b", "session", "video", drilldownValue([frames[0]])); + const principalABytes = + cache + .get("principal-a", "session", "video") + ?.frames.reduce((total, frame) => total + retainedBytes(frame.dataUri), 0) ?? 0; + const principalBBytes = + cache + .get("principal-b", "session", "video") + ?.frames.reduce((total, frame) => total + retainedBytes(frame.dataUri), 0) ?? 0; + assert.deepEqual(cache.getUsage("principal-a"), { + bytes: principalABytes, + entries: 1, + totalBytes: principalABytes + principalBBytes, + totalEntries: 2, + }); + + now = 7000; + + assert.deepEqual(cache.getUsage("principal-a"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); + assert.equal(cache.clearSession("principal-b", "session"), 0); +}); + +test("drill-down cache enforces a global byte budget with LRU eviction", async () => { + const bigFrame = (fill: string): VideoDrilldownFrame => ({ + dataUri: jpegDataUri(320, 180, 3000, fill.charCodeAt(0)), + height: 180, + timestampSeconds: 1, + width: 320, + }); + const bigFrameBytes = retainedBytes(bigFrame("A").dataUri); const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 10, - maxTotalBytes: 7000, + maxTotalBytes: bigFrameBytes * 2, + normalizeJpeg: retainFixtureJpeg, }); - cache.put("s", "v1", { durationSeconds: 10, frames: [bigFrame("A")] }); - cache.put("s", "v2", { durationSeconds: 10, frames: [bigFrame("B")] }); - assert.ok(cache.get("s", "v1")); - assert.ok(cache.get("s", "v2")); - cache.put("s", "v3", { durationSeconds: 10, frames: [bigFrame("C")] }); - assert.equal(cache.get("s", "v1"), null, "the least recently used entry must be evicted"); - assert.ok(cache.get("s", "v2")); - assert.ok(cache.get("s", "v3")); - assert.ok(cache.get("s", "v2")); - cache.put("s", "v4", { durationSeconds: 10, frames: [bigFrame("D")] }); - assert.equal(cache.get("s", "v3"), null, "eviction must follow recency, not insertion order"); - assert.ok(cache.get("s", "v2")); - assert.ok(cache.get("s", "v4")); + await cache.put("principal", "s", "v1", drilldownValue([bigFrame("A")])); + await cache.put("principal", "s", "v2", drilldownValue([bigFrame("B")])); + assert.ok(cache.get("principal", "s", "v1")); + assert.ok(cache.get("principal", "s", "v2")); + await cache.put("principal", "s", "v3", drilldownValue([bigFrame("C")])); + assert.equal( + cache.get("principal", "s", "v1"), + null, + "the least recently used entry must be evicted" + ); + assert.ok(cache.get("principal", "s", "v2")); + assert.ok(cache.get("principal", "s", "v3")); + assert.ok(cache.get("principal", "s", "v2")); + await cache.put("principal", "s", "v4", drilldownValue([bigFrame("D")])); + assert.equal( + cache.get("principal", "s", "v3"), + null, + "eviction must follow recency, not insertion order" + ); + assert.ok(cache.get("principal", "s", "v2")); + assert.ok(cache.get("principal", "s", "v4")); }); -test("drill-down cache rejects an entry larger than the whole byte budget", () => { +test("drill-down cache enforces each principal quota without charging another principal", async () => { + const bigFrame = (fill: string): VideoDrilldownFrame => ({ + dataUri: jpegDataUri(320, 180, 3000, fill.charCodeAt(0)), + height: 180, + timestampSeconds: 1, + width: 320, + }); + const bigFrameBytes = retainedBytes(bigFrame("A").dataUri); + const cache = new VideoDrilldownCache({ + now: () => 1000, + ttlMs: 5000, + maxEntries: 10, + maxTotalBytes: bigFrameBytes * 6, + maxBytesPerPrincipal: bigFrameBytes * 2, + maxEntriesPerPrincipal: 2, + normalizeJpeg: retainFixtureJpeg, + }); + await cache.put("principal-a", "s", "v1", drilldownValue([bigFrame("A")])); + await cache.put("principal-a", "s", "v2", drilldownValue([bigFrame("B")])); + await cache.put("principal-b", "s", "v1", drilldownValue([bigFrame("C")])); + await cache.put("principal-b", "s", "v2", drilldownValue([bigFrame("D")])); + assert.ok(cache.get("principal-a", "s", "v1")); + + await cache.put("principal-a", "s", "v3", drilldownValue([bigFrame("E")])); + + assert.equal(cache.get("principal-a", "s", "v2"), null, "principal A must evict its own LRU"); + assert.ok(cache.get("principal-a", "s", "v1")); + assert.ok(cache.get("principal-a", "s", "v3")); + assert.ok(cache.get("principal-b", "s", "v1"), "principal B must keep its independent quota"); + assert.ok(cache.get("principal-b", "s", "v2")); +}); + +test("drill-down cache returns server-derived audit metadata without retaining the raw parent", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + const parentContentHash = `sha256:${"a".repeat(64)}`; + await cache.put("principal", "session", "sensitive-parent-ref", { + derivation: { + parentContentHash, + policy: "focused-window", + version: "video-drilldown/v1", + }, + durationSeconds: 10, + frames: [{ ...frames[0], height: 180, width: 320 }], + }); + + const result = cache.get("principal", "session", "sensitive-parent-ref"); + assert.deepEqual(result?.derivation, { + contentHash: result?.derivation.contentHash, + createdAt: 1000, + format: "image/jpeg", + parent: { + contentHash: parentContentHash, + referenceHash: `sha256:${createHash("sha256").update("sensitive-parent-ref").digest("hex")}`, + }, + policy: "focused-window", + resolution: { height: 180, width: 320 }, + version: "video-drilldown/v1", + }); + assert.match(result?.derivation.contentHash ?? "", /^sha256:[a-f0-9]{64}$/); + assert.equal(JSON.stringify(result).includes("sensitive-parent-ref"), false); +}); + +test("drill-down cache preserves the prior derivation when a replacement fails validation", async () => { + const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4 }); + await cache.put("principal", "session", "video", drilldownValue(frames)); + const before = cache.get("principal", "session", "video"); + const beforeBytes = + before?.frames.reduce((total, frame) => total + retainedBytes(frame.dataUri), 0) ?? 0; + + await assert.rejects( + cache.put("principal", "session", "video", { + derivation: { ...derivation, parentContentHash: "not-a-content-hash" }, + durationSeconds: 10, + frames, + }), + /derivation metadata/i + ); + + assert.deepEqual(cache.get("principal", "session", "video"), before); + assert.deepEqual(cache.getUsage("principal"), { + bytes: beforeBytes, + entries: 1, + totalBytes: beforeBytes, + totalEntries: 1, + }); +}); + +test("drill-down cache aborts during JPEG validation without committing quota", async () => { + let markValidationStarted: () => void = () => {}; + let releaseValidation: () => void = () => {}; + const validationStarted = new Promise((resolve) => { + markValidationStarted = resolve; + }); + const validationRelease = new Promise((resolve) => { + releaseValidation = resolve; + }); + const cache = new VideoDrilldownCache({ + maxEntries: 4, + now: () => 1000, + ttlMs: 5000, + normalizeJpeg: async (data) => { + markValidationStarted(); + await validationRelease; + return { data, height: 180, width: 320 }; + }, + }); + const controller = new AbortController(); + const pending = cache.put("principal", "session", "video", drilldownValue([frames[0]]), { + signal: controller.signal, + }); + + await validationStarted; + controller.abort(); + releaseValidation(); + + await assert.rejects(pending, VideoDrilldownAbortedError); + assert.deepEqual(cache.getUsage("principal"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down cache rejects an entry larger than the whole byte budget", async () => { const cache = new VideoDrilldownCache({ now: () => 1000, ttlMs: 5000, maxEntries: 4, maxTotalBytes: 1000, + normalizeJpeg: retainFixtureJpeg, }); - assert.throws( - () => - cache.put("s", "v1", { - durationSeconds: 10, - frames: [{ dataUri: `data:image/jpeg;base64,${"A".repeat(4000)}`, timestampSeconds: 1 }], - }), + await assert.rejects( + cache.put("principal", "s", "v1", { + derivation, + durationSeconds: 10, + frames: [ + { + dataUri: jpegDataUri(320, 180, 4000, 65), + height: 180, + timestampSeconds: 1, + width: 320, + }, + ], + }), /byte budget/i ); - assert.equal(cache.get("s", "v1"), null); + assert.equal(cache.get("principal", "s", "v1"), null); assert.throws( () => new VideoDrilldownCache({ now: () => 0, ttlMs: 1, maxEntries: 1, maxTotalBytes: 0 }), /byte budget/i diff --git a/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts b/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts new file mode 100644 index 0000000000..5c5695f018 --- /dev/null +++ b/tests/unit/guardrails/videoBridgeFu07StructuralSampling.test.ts @@ -0,0 +1,486 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { + analyzeVideoStructure, + calculateSamplingDecision, + extractFramesFromLocalVideo, + extractVideoFramesFromBytes, + parseVideoStructuralAnalysis, + type VideoCommandRunner, + type VideoStructuralAnalysis, +} from "../../../src/lib/guardrails/videoBridgeRuntime.ts"; + +const execFileAsync = promisify(execFile); + +async function writeFrozenThenMotionFixture(fixturePath: string): Promise { + await execFileAsync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=black:s=320x180:d=6:r=12", + "-f", + "lavfi", + "-i", + "testsrc2=s=320x180:d=4:r=12", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0,format=yuv420p[v]", + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-threads", + "1", + "-y", + fixturePath, + ], + { timeout: 30_000 } + ); +} + +const realRunner: VideoCommandRunner = async (executable, args, options) => { + const result = await execFileAsync(executable, [...args], { + encoding: "utf8", + maxBuffer: 1024 * 1024, + signal: options.signal, + timeout: options.timeoutMs, + }); + return { stderr: String(result.stderr), stdout: String(result.stdout) }; +}; + +function structuralAnalysis( + overrides: Partial = {} +): VideoStructuralAnalysis { + return { + freezeIntervals: [{ endSeconds: 6, startSeconds: 0 }], + samples: [ + { + blur: null, + brightness: 16, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 1, + }, + { + blur: 4.8, + brightness: 121, + sceneScore: 42, + spatialInformation: 120, + temporalInformation: 32, + timestampSeconds: 6, + }, + { + blur: 4.9, + brightness: 122, + sceneScore: 0, + spatialInformation: 118, + temporalInformation: 28, + timestampSeconds: 8, + }, + ], + sceneCandidates: [6], + ...overrides, + }; +} + +test("parses scene, freeze, blur, exposure, and spatial-temporal evidence", () => { + const metadata = [ + "frame:0 pts:0 pts_time:0", + "lavfi.scd.score=0.000", + "frame:0 pts:0 pts_time:0", + "lavfi.siti.si=0.00", + "frame:0 pts:0 pts_time:0", + "lavfi.siti.ti=0.00", + "frame:0 pts:0 pts_time:0", + "lavfi.blur=-nan", + "frame:0 pts:0 pts_time:0", + "lavfi.signalstats.YAVG=16", + "frame:6 pts:6 pts_time:6", + "lavfi.scd.score=41.013", + "frame:6 pts:6 pts_time:6", + "lavfi.siti.si=108.50", + "frame:6 pts:6 pts_time:6", + "lavfi.siti.ti=66.51", + "frame:6 pts:6 pts_time:6", + "lavfi.blur=4.75", + "frame:6 pts:6 pts_time:6", + "lavfi.signalstats.YAVG=121.5", + ].join("\n"); + const stderr = [ + "lavfi.freezedetect.freeze_start: 0", + "lavfi.freezedetect.freeze_duration: 6", + "lavfi.freezedetect.freeze_end: 6", + ].join("\n"); + + const analysis = parseVideoStructuralAnalysis(metadata, stderr, 10); + + assert.deepEqual(analysis.sceneCandidates, [6]); + assert.deepEqual(analysis.freezeIntervals, [{ endSeconds: 6, startSeconds: 0 }]); + assert.deepEqual(analysis.samples, [ + { + blur: null, + brightness: 16, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 0, + }, + { + blur: 4.75, + brightness: 121.5, + sceneScore: 41.013, + spatialInformation: 108.5, + temporalInformation: 66.51, + timestampSeconds: 6, + }, + ]); +}); + +test("runs all structural filters in one fixed, local-only, bounded FFmpeg pass", async () => { + const calls: Array<{ args: string[]; timeoutMs: number }> = []; + const runner: VideoCommandRunner = async (executable, args, options) => { + assert.equal(executable, "ffmpeg"); + calls.push({ args: [...args], timeoutMs: options.timeoutMs }); + return { + stderr: "lavfi.freezedetect.freeze_start: 0\nlavfi.freezedetect.freeze_end: 2", + stdout: "frame:0 pts:0 pts_time:0\nlavfi.scd.score=0", + }; + }; + + await analyzeVideoStructure("/tmp/input.mp4", { + durationSeconds: 8, + runner, + streamIndex: 2, + timeoutMs: 4_000, + }); + + assert.equal(calls.length, 1, "structural analysis must decode the video exactly once"); + assert.equal(calls[0].timeoutMs, 4_000); + assert.ok(calls[0].args.includes("-nostdin")); + assert.deepEqual(calls[0].args.slice(calls[0].args.indexOf("-map"), -1), [ + "-map", + "0:2", + "-vf", + calls[0].args[calls[0].args.indexOf("-vf") + 1], + "-an", + "-frames:v", + "600", + "-f", + "null", + ]); + const filter = calls[0].args[calls[0].args.indexOf("-vf") + 1]; + for (const expected of ["scdet", "freezedetect", "blurdetect", "signalstats", "siti"]) { + assert.match(filter, new RegExp(expected)); + } + assert.equal( + calls[0].args.some((argument) => argument.includes("://")), + false + ); +}); + +test("spends one frame on a frozen segment and reallocates the budget to dense motion", () => { + const analysis = structuralAnalysis(); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.equal(decision.timestamps.length, 4); + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 6).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 6).length, 3); +}); + +test("avoids redundant caption work for an entirely frozen video", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [{ endSeconds: 8, startSeconds: 0 }], + samples: [ + { + blur: null, + brightness: 81, + sceneScore: 0, + spatialInformation: 0, + temporalInformation: 0, + timestampSeconds: 4, + }, + ], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision(8, 8, "segment_aware", [], null, analysis); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.equal(decision.timestamps.length, 1); + assert.deepEqual(decision.timestamps, [4]); +}); + +test("does not prune a moving clip when freeze evidence is absent", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [], + samples: [ + { + blur: 4.8, + brightness: 120, + sceneScore: 0, + spatialInformation: 100, + temporalInformation: 30, + timestampSeconds: 1, + }, + { + blur: 4.9, + brightness: 122, + sceneScore: 0, + spatialInformation: 105, + temporalInformation: 32, + timestampSeconds: 7, + }, + ], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision(8, 4, "segment_aware", [], null, analysis); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.deepEqual(decision.timestamps, [1, 3, 5, 7]); +}); + +test("uses lower FFmpeg blur scores as sharper evidence for the extra frame", () => { + const common = { + brightness: 120, + sceneScore: 0, + spatialInformation: 50, + temporalInformation: 10, + }; + const analysis = structuralAnalysis({ + freezeIntervals: [], + samples: [ + { ...common, blur: 17, timestampSeconds: 1 }, + { ...common, blur: 4, timestampSeconds: 5 }, + ], + sceneCandidates: [4], + }); + const decision = calculateSamplingDecision(8, 3, "segment_aware", [4], null, analysis); + + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 4).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 4).length, 2); +}); + +test("malformed-only structural metadata fails open to uniform sampling", () => { + const analysis = parseVideoStructuralAnalysis("frame:0 pts:0 pts_time:0\nlavfi.blur=-nan", "", 8); + const decision = calculateSamplingDecision(8, 4, "segment_aware", [], null, analysis); + + assert.deepEqual(analysis.samples, []); + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.timestamps, [1, 3, 5, 7]); +}); + +test("keeps the long trailing segment when scene boundaries outnumber the frame budget", () => { + const decision = calculateSamplingDecision(20, 4, "segment_aware", [1, 2, 3, 4]); + + assert.equal(decision.timestamps.length, 4); + assert.ok( + decision.timestamps.some((timestamp) => timestamp > 4), + "the 16-second tail must not be dropped by early short cuts" + ); +}); + +test("preserves the legacy length-weighted allocation without structural evidence", () => { + const decision = calculateSamplingDecision(10, 8, "segment_aware", [2]); + + assert.equal(decision.policyEffective, "segment_aware"); + assert.deepEqual( + decision.timestamps.map((timestamp) => Number(timestamp.toFixed(3))), + [0.5, 1.5, 2.667, 4, 5.333, 6.667, 8, 9.333] + ); +}); + +test("does not report a focus-window boundary as usable segment evidence", () => { + const decision = calculateSamplingDecision(10, 4, "segment_aware", [2], { + endSeconds: 8, + startSeconds: 2, + }); + + assert.equal(decision.policyEffective, "uniform"); + assert.equal(decision.candidateCount, 0); + assert.deepEqual(decision.timestamps, [2.75, 4.25, 5.75, 7.25]); +}); + +test("does not claim segment-aware evidence that falls outside the focus window", () => { + const analysis = structuralAnalysis({ + freezeIntervals: [{ endSeconds: 10, startSeconds: 8 }], + samples: [{ timestampSeconds: 9, temporalInformation: 0 }], + sceneCandidates: [], + }); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + [], + { endSeconds: 8, startSeconds: 2 }, + analysis + ); + + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.timestamps, [2.75, 4.25, 5.75, 7.25]); +}); + +test("structural timeout fails open to uniform while an abort stops extraction", async () => { + let analysisCalls = 0; + const timeoutRunner: VideoCommandRunner = async (_executable, args) => { + if (args.some((argument) => argument.includes("freezedetect"))) { + analysisCalls += 1; + throw new Error("structural deadline exceeded"); + } + return { stderr: "", stdout: "" }; + }; + + const frames = await extractFramesFromLocalVideo("/tmp/input.mp4", "/tmp/frames", { + durationSeconds: 8, + frameCount: 4, + runner: timeoutRunner, + samplingPolicy: "segment_aware", + streamIndex: 0, + timeoutMs: 250, + }); + assert.equal(analysisCalls, 1); + assert.equal(frames.sampling.policyEffective, "uniform"); + assert.deepEqual( + frames.map((frame) => frame.timestampSeconds), + [1, 3, 5, 7] + ); + + const controller = new AbortController(); + let frameExtractionCalls = 0; + const abortRunner: VideoCommandRunner = async (_executable, args, options) => { + if (args.some((argument) => argument.includes("freezedetect"))) { + assert.equal(options.signal, controller.signal); + controller.abort(); + throw new Error("aborted inside structural analysis"); + } + frameExtractionCalls += 1; + return { stderr: "", stdout: "" }; + }; + await assert.rejects( + () => + extractFramesFromLocalVideo("/tmp/input.mp4", "/tmp/frames", { + durationSeconds: 8, + frameCount: 4, + runner: abortRunner, + samplingPolicy: "segment_aware", + signal: controller.signal, + streamIndex: 0, + timeoutMs: 250, + }), + /aborted/ + ); + assert.equal(frameExtractionCalls, 0); +}); + +test("real FFmpeg evidence distinguishes a frozen dark segment from dense motion", async (t) => { + try { + await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + } catch { + t.skip("FFmpeg is an optional runtime dependency"); + return; + } + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-real-")); + const fixturePath = join(directory, "frozen-then-motion.mp4"); + try { + await writeFrozenThenMotionFixture(fixturePath); + + const analysis = await analyzeVideoStructure(fixturePath, { + durationSeconds: 10, + streamIndex: 0, + timeoutMs: 30_000, + }); + const decision = calculateSamplingDecision( + 10, + 4, + "segment_aware", + analysis.sceneCandidates, + null, + analysis + ); + + assert.ok(analysis.samples.length >= 8); + assert.ok(analysis.sceneCandidates.some((timestamp) => Math.abs(timestamp - 6) <= 1)); + assert.ok( + analysis.freezeIntervals.some( + (interval) => interval.startSeconds <= 1 && interval.endSeconds >= 5 + ) + ); + assert.ok(analysis.samples.some((sample) => (sample.spatialInformation ?? 0) > 20)); + assert.ok(analysis.samples.some((sample) => (sample.temporalInformation ?? 0) > 5)); + assert.ok(analysis.samples.some((sample) => (sample.blur ?? 0) > 0)); + assert.ok(analysis.samples.some((sample) => (sample.brightness ?? 255) < 24)); + assert.equal(decision.timestamps.filter((timestamp) => timestamp < 6).length, 1); + assert.equal(decision.timestamps.filter((timestamp) => timestamp > 6).length, 3); + } finally { + await rm(directory, { force: true, recursive: true }); + } +}); + +test("real FFmpeg abort stops preanalysis, skips frame extraction, and cleans the private tree", async (t) => { + try { + await execFileAsync("ffmpeg", ["-version"], { timeout: 5_000 }); + } catch { + t.skip("FFmpeg is an optional runtime dependency"); + return; + } + + const directory = await mkdtemp(join(tmpdir(), "video-fu07-abort-")); + const fixturePath = join(directory, "abort.mp4"); + const controller = new AbortController(); + let privateInputPath = ""; + let analysisStarted = false; + let frameExtractionCalls = 0; + try { + await writeFrozenThenMotionFixture(fixturePath); + const bytes = await readFile(fixturePath); + const runner: VideoCommandRunner = async (executable, args, options) => { + if (executable === "ffprobe") privateInputPath = args.at(-1) ?? ""; + if (args.some((argument) => argument.includes("freezedetect"))) { + analysisStarted = true; + setTimeout(() => controller.abort(), 25); + } else if (executable === "ffmpeg") { + frameExtractionCalls += 1; + } + return realRunner(executable, args, options); + }; + + await assert.rejects( + () => + extractVideoFramesFromBytes(bytes, { + frameCount: 4, + maxDurationSeconds: 600, + runner, + samplingPolicy: "segment_aware", + signal: controller.signal, + timeoutMs: 30_000, + }), + /aborted/ + ); + assert.equal(analysisStarted, true); + assert.equal(frameExtractionCalls, 0); + assert.notEqual(privateInputPath, ""); + await assert.rejects(() => access(privateInputPath)); + } finally { + await rm(directory, { force: true, recursive: true }); + } +}); diff --git a/tests/unit/guardrails/videoBridgeSampler.test.ts b/tests/unit/guardrails/videoBridgeSampler.test.ts index b8d391a1dc..ec7304655a 100644 --- a/tests/unit/guardrails/videoBridgeSampler.test.ts +++ b/tests/unit/guardrails/videoBridgeSampler.test.ts @@ -29,6 +29,26 @@ test("scene-aware sampling falls back to deterministic uniform midpoints for a s assert.equal(decision.candidateCount, 0); }); +test("scene-aware sampling falls back to the midpoint when one frame cannot cover both ends", () => { + const decision = calculateSamplingDecision(8, 1, "scene_aware", [0.25, 7.75]); + + assert.deepEqual(decision.timestamps, [4]); + assert.equal(decision.policyRequested, "scene_aware"); + assert.equal(decision.policyEffective, "uniform"); + assert.equal(decision.candidateCount, 2); +}); + +test("one-frame scene-aware fallback uses the active focus-window midpoint", () => { + const decision = calculateSamplingDecision(10, 1, "scene_aware", [2.25, 7.75], { + endSeconds: 8, + startSeconds: 2, + }); + + assert.deepEqual(decision.timestamps, [5]); + assert.equal(decision.policyEffective, "uniform"); + assert.deepEqual(decision.focusWindow, { endSeconds: 8, startSeconds: 2 }); +}); + test("scene candidates are parsed from showinfo output and malformed values are ignored", () => { const output = [ "[Parsed_showinfo_0 @ 0x1] n:1 pts_time:1.250", diff --git a/tests/unit/video-bridge-drilldown-authz.test.ts b/tests/unit/video-bridge-drilldown-authz.test.ts new file mode 100644 index 0000000000..59d4c20204 --- /dev/null +++ b/tests/unit/video-bridge-drilldown-authz.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildVideoBridgeDrilldownHeaders, + VIDEO_BRIDGE_DRILLDOWN_PATH, +} from "../../src/lib/guardrails/videoBridgeBrokerAuth.ts"; +import { managementPolicy } from "../../src/server/authz/policies/management.ts"; + +function policyContext(path: string, ip = "127.0.0.1") { + return { + request: { + method: "GET", + headers: new Headers(buildVideoBridgeDrilldownHeaders("principal-a")), + ip, + url: `http://localhost${path}`, + nextUrl: { pathname: path }, + }, + classification: { + routeClass: "MANAGEMENT" as const, + normalizedPath: path, + reason: "management_api", + }, + requestId: "req_video_drilldown_authz", + }; +} + +test("drill-down principal is canonical visible ASCII and is never silently trimmed", () => { + assert.throws(() => buildVideoBridgeDrilldownHeaders(" principal-a "), /principal/i); + assert.throws(() => buildVideoBridgeDrilldownHeaders("principal-á"), /principal/i); + assert.doesNotThrow(() => buildVideoBridgeDrilldownHeaders("tenant:principal-a")); +}); + +test("management policy carries the token-bound drill-down self-hop to the route", async () => { + const outcome = await managementPolicy.evaluate(policyContext(VIDEO_BRIDGE_DRILLDOWN_PATH)); + + assert.equal(outcome.allow, true); + if (outcome.allow) { + assert.equal(outcome.subject.id, "video-bridge-drilldown"); + assert.equal(outcome.subject.label, "internal-video-bridge-drilldown"); + } + + const adjacent = await managementPolicy.evaluate( + policyContext("/api/modality-bridge/video/runtime") + ); + assert.notEqual( + adjacent.allow ? adjacent.subject.label : "rejected", + "internal-video-bridge-drilldown", + "the broker token must not authenticate an adjacent Video Bridge path" + ); + + const remote = await managementPolicy.evaluate( + policyContext(VIDEO_BRIDGE_DRILLDOWN_PATH, "203.0.113.10") + ); + assert.equal(remote.allow, false); + if (!remote.allow) assert.equal(remote.code, "LOCAL_ONLY"); +}); diff --git a/tests/unit/video-bridge-drilldown-route.test.ts b/tests/unit/video-bridge-drilldown-route.test.ts index 6ee9372752..1e92c94abc 100644 --- a/tests/unit/video-bridge-drilldown-route.test.ts +++ b/tests/unit/video-bridge-drilldown-route.test.ts @@ -1,26 +1,92 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { handleVideoDrilldownRequest } from "../../src/app/api/modality-bridge/video/drilldown/route"; -import { buildVideoBridgeBrokerHeaders } from "../../src/lib/guardrails/videoBridgeBrokerAuth"; -import { VideoDrilldownCache } from "../../src/lib/guardrails/videoBridgeDrilldown"; +import sharp from "sharp"; + +import { + handleVideoDrilldownRequest, + VIDEO_DRILLDOWN_MAX_BODY_BYTES, +} from "../../src/app/api/modality-bridge/video/drilldown/route"; +import { + buildVideoBridgeBrokerHeaders, + buildVideoBridgeDrilldownHeaders, + VIDEO_BRIDGE_DRILLDOWN_PRINCIPAL_HEADER, +} from "../../src/lib/guardrails/videoBridgeBrokerAuth"; +import { + VideoDrilldownCache, + VIDEO_DRILLDOWN_MAX_ENTRY_BYTES, +} from "../../src/lib/guardrails/videoBridgeDrilldown"; import { AUTHZ_HEADER_PEER_LOCALITY } from "../../src/server/authz/headers"; import { isLocalOnlyPath } from "../../src/server/authz/routeGuard"; -function headers(contentType?: string): Headers { +const derivation = { + parentContentHash: `sha256:${"a".repeat(64)}`, + policy: "focused-window", + version: "video-drilldown/v1", +}; + +const validJpegs = new Map(); +for (const [width, height] of [ + [320, 180], + [640, 360], +] as const) { + validJpegs.set( + `${width}x${height}`, + await sharp({ + create: { width, height, channels: 3, background: { r: 1, g: 1, b: 1 } }, + }) + .jpeg({ progressive: false }) + .toBuffer() + ); +} + +function jpegDataUri(width: number, height: number, payloadBytes = 0, fill = 0): string { + const base = validJpegs.get(`${width}x${height}`); + if (!base) throw new Error(`Missing valid JPEG fixture for ${width}x${height}`); + if (payloadBytes > 65_531) throw new Error("JPEG fixture comment is too large"); + const bytes = + payloadBytes === 0 + ? base + : Buffer.concat([ + base.subarray(0, -2), + Buffer.from([0xff, 0xfe, (payloadBytes + 2) >> 8, (payloadBytes + 2) & 0xff]), + Buffer.alloc(payloadBytes, fill), + base.subarray(-2), + ]); + return `data:image/jpeg;base64,${bytes.toString("base64")}`; +} + +function headers(principalId: string, contentType?: string): Headers { return new Headers({ - ...buildVideoBridgeBrokerHeaders(), + ...buildVideoBridgeDrilldownHeaders(principalId), [AUTHZ_HEADER_PEER_LOCALITY]: "loopback", ...(contentType ? { "Content-Type": contentType } : {}), }); } +test("drill-down JSON body budget can carry the documented decoded entry ceiling", () => { + const encodedEntryBytes = Math.ceil(VIDEO_DRILLDOWN_MAX_ENTRY_BYTES / 3) * 4; + assert.ok(VIDEO_DRILLDOWN_MAX_BODY_BYTES >= encodedEntryBytes + 64 * 1024); +}); + test("drill-down route is loopback/token protected and has no public fallback", async () => { assert.equal(isLocalOnlyPath("/api/modality-bridge/video/drilldown", "GET"), true); const response = await handleVideoDrilldownRequest( new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=s&videoRef=v") ); assert.equal(response.status, 403); + + const missingPrincipal = new Headers({ + ...buildVideoBridgeBrokerHeaders(), + [AUTHZ_HEADER_PEER_LOCALITY]: "loopback", + }); + assert.equal(missingPrincipal.has(VIDEO_BRIDGE_DRILLDOWN_PRINCIPAL_HEADER), false); + const missingPrincipalResponse = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=s&videoRef=v", { + headers: missingPrincipal, + }) + ); + assert.equal(missingPrincipalResponse.status, 403); }); test("drill-down route stores, slices, and deletes an isolated session result", async () => { @@ -28,15 +94,22 @@ test("drill-down route stores, slices, and deletes an isolated session result", const post = await handleVideoDrilldownRequest( new Request("http://localhost/api/modality-bridge/video/drilldown", { body: JSON.stringify({ + derivation, durationSeconds: 10, frames: [ - { dataUri: "data:image/jpeg;base64,QQ==", timestampSeconds: 1 }, - { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, + { + dataUri: jpegDataUri(320, 180, 1, 1), + timestampSeconds: 1, + }, + { + dataUri: jpegDataUri(320, 180, 1, 2), + timestampSeconds: 5, + }, ], sessionId: "session-a", videoRef: "video-a", }), - headers: headers("application/json"), + headers: headers("principal-a", "application/json"), method: "POST", }), { cache } @@ -46,21 +119,318 @@ test("drill-down route stores, slices, and deletes an isolated session result", const get = await handleVideoDrilldownRequest( new Request( "http://localhost/api/modality-bridge/video/drilldown?sessionId=session-a&videoRef=video-a&start=2&end=6&frames=1", - { headers: headers() } + { headers: headers("principal-a") } ), { cache } ); assert.equal(get.status, 200); - assert.deepEqual((await get.json()).frames, [ - { dataUri: "data:image/jpeg;base64,Qg==", timestampSeconds: 5 }, - ]); + const getBody = await get.json(); + assert.equal(getBody.frames.length, 1); + assert.deepEqual( + getBody.frames.map( + ({ + height, + timestampSeconds, + width, + }: { + height: number; + timestampSeconds: number; + width: number; + }) => ({ + height, + timestampSeconds, + width, + }) + ), + [{ height: 180, timestampSeconds: 5, width: 320 }] + ); + assert.match(getBody.frames[0].dataUri, /^data:image\/jpeg;base64,/); + const returnedJpeg = Buffer.from(getBody.frames[0].dataUri.split(",", 2)[1], "base64"); + assert.deepEqual( + await sharp(returnedJpeg) + .metadata() + .then(({ height, width }) => ({ height, width })), + { height: 180, width: 320 } + ); + assert.equal(getBody.derivation.createdAt, 1000); + assert.equal(getBody.derivation.format, "image/jpeg"); + assert.equal(getBody.derivation.parent.contentHash, derivation.parentContentHash); + assert.deepEqual(getBody.derivation.resolution, { height: 180, width: 320 }); + assert.match(getBody.derivation.contentHash, /^sha256:[a-f0-9]{64}$/); const deleted = await handleVideoDrilldownRequest( new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=session-a", { - headers: headers(), + headers: headers("principal-a"), method: "DELETE", }), { cache } ); assert.deepEqual(await deleted.json(), { removed: 1 }); }); + +test("drill-down route denies cross-principal reads and deletes without enumerating", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const body = JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [ + { + dataUri: jpegDataUri(320, 180), + timestampSeconds: 1, + }, + ], + sessionId: "shared-session", + videoRef: "shared-video", + }); + const stored = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body, + headers: headers("principal-a", "application/json"), + method: "POST", + }), + { cache } + ); + assert.equal(stored.status, 201); + + const deniedRead = await handleVideoDrilldownRequest( + new Request( + "http://localhost/api/modality-bridge/video/drilldown?sessionId=shared-session&videoRef=shared-video", + { headers: headers("principal-b") } + ), + { cache } + ); + assert.equal(deniedRead.status, 404); + + const deniedDelete = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=shared-session", { + headers: headers("principal-b"), + method: "DELETE", + }), + { cache } + ); + assert.deepEqual(await deniedDelete.json(), { removed: 0 }); + + const ownerRead = await handleVideoDrilldownRequest( + new Request( + "http://localhost/api/modality-bridge/video/drilldown?sessionId=shared-session&videoRef=shared-video", + { headers: headers("principal-a") } + ), + { cache } + ); + assert.equal(ownerRead.status, 200); +}); + +test("drill-down route does not retain a cancelled derivation", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const controller = new AbortController(); + controller.abort(); + const response = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [ + { + dataUri: jpegDataUri(320, 180), + timestampSeconds: 1, + }, + ], + sessionId: "cancelled-session", + videoRef: "cancelled-video", + }), + headers: headers("principal-a", "application/json"), + method: "POST", + signal: controller.signal, + }), + { cache } + ); + + assert.equal(response.status, 499); + assert.deepEqual(cache.getUsage("principal-a"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down route cancels an in-flight JPEG validation before cache commit", async () => { + let markValidationStarted: () => void = () => {}; + let releaseValidation: () => void = () => {}; + const validationStarted = new Promise((resolve) => { + markValidationStarted = resolve; + }); + const validationRelease = new Promise((resolve) => { + releaseValidation = resolve; + }); + const cache = new VideoDrilldownCache({ + maxEntries: 4, + now: () => 1000, + ttlMs: 5000, + normalizeJpeg: async (data) => { + markValidationStarted(); + await validationRelease; + return { data, height: 180, width: 320 }; + }, + }); + const controller = new AbortController(); + const pending = handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [{ dataUri: jpegDataUri(320, 180), timestampSeconds: 1 }], + sessionId: "cancelled-session", + videoRef: "cancelled-video", + }), + headers: headers("principal-a", "application/json"), + method: "POST", + signal: controller.signal, + }), + { cache } + ); + + await validationStarted; + controller.abort(); + releaseValidation(); + + const response = await pending; + assert.equal(response.status, 499); + assert.deepEqual(cache.getUsage("principal-a"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down route rejects raw media instead of silently retaining it", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const response = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [ + { + dataUri: jpegDataUri(320, 180), + timestampSeconds: 1, + }, + ], + rawMedia: "data:video/mp4;base64,AAAA", + sessionId: "raw-session", + videoRef: "raw-video", + }), + headers: headers("principal-a", "application/json"), + method: "POST", + }), + { cache } + ); + + assert.equal(response.status, 400); + assert.equal(cache.getUsage("principal-a").entries, 0); +}); + +test("drill-down route rejects padded Base64, disguised media, and caller dimensions", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const mp4 = Buffer.concat([ + Buffer.from([0, 0, 0, 24]), + Buffer.from("ftypisom", "ascii"), + ]).toString("base64"); + const invalidFrames: Array> = [ + { dataUri: `${jpegDataUri(320, 180)}${"=".repeat(1024 * 1024)}`, timestampSeconds: 1 }, + { dataUri: `data:image/jpeg;base64,${mp4}`, timestampSeconds: 1 }, + { dataUri: "data:image/jpeg;base64,/9hBQkP/wAAHCAABAAE=", timestampSeconds: 1 }, + { dataUri: jpegDataUri(320, 180), height: 1, timestampSeconds: 1, width: 1 }, + ]; + + for (const [index, frame] of invalidFrames.entries()) { + const response = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [frame], + sessionId: `invalid-session-${index}`, + videoRef: `invalid-video-${index}`, + }), + headers: headers("principal-a", "application/json"), + method: "POST", + }), + { cache } + ); + assert.equal(response.status, 400); + } + + assert.equal(cache.getUsage("principal-a").entries, 0); +}); + +test("drill-down route rejects non-canonical session and video identifiers consistently", async () => { + const cache = new VideoDrilldownCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const post = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [{ dataUri: jpegDataUri(320, 180), timestampSeconds: 1 }], + sessionId: " session-a ", + videoRef: " video-a ", + }), + headers: headers("principal-a", "application/json"), + method: "POST", + }), + { cache } + ); + assert.equal(post.status, 400); + + const get = await handleVideoDrilldownRequest( + new Request( + "http://localhost/api/modality-bridge/video/drilldown?sessionId=%20session-a%20&videoRef=%20video-a%20", + { headers: headers("principal-a") } + ), + { cache } + ); + assert.equal(get.status, 400); + + const deleted = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown?sessionId=%20session-a%20", { + headers: headers("principal-a"), + method: "DELETE", + }), + { cache } + ); + assert.equal(deleted.status, 400); + assert.deepEqual(cache.getUsage("principal-a"), { + bytes: 0, + entries: 0, + totalBytes: 0, + totalEntries: 0, + }); +}); + +test("drill-down route maps unexpected cache failures to a sanitized 500", async () => { + class FailingCache extends VideoDrilldownCache { + override async put(..._args: Parameters): Promise { + throw new Error("secret failure at /tmp/internal/drilldown.ts:42"); + } + } + const cache = new FailingCache({ maxEntries: 4, now: () => 1000, ttlMs: 5000 }); + const response = await handleVideoDrilldownRequest( + new Request("http://localhost/api/modality-bridge/video/drilldown", { + body: JSON.stringify({ + derivation, + durationSeconds: 10, + frames: [{ dataUri: jpegDataUri(320, 180), timestampSeconds: 1 }], + sessionId: "session-a", + videoRef: "video-a", + }), + headers: headers("principal-a", "application/json"), + method: "POST", + }), + { cache } + ); + + assert.equal(response.status, 500); + const text = await response.text(); + assert.doesNotMatch(text, /secret failure|\/tmp\/internal|drilldown\.ts/i); +});