From 2eab74b039a24cb2e5b3dfae7446efa99e348aca Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Sat, 29 Aug 2026 07:05:04 -0300 Subject: [PATCH] feat(guardrails): enforce video transcript provenance, budgets and reconciliation (#11652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FU-05 core: caller-supplied Video Bridge transcripts can no longer self-assert "embedded"/"audio-bridge" provenance — normalizeVideoTranscript now reclassifies any caller-declared value to "client" by default, and only a server-owned adapter passing the code-only trustedSource option (never reachable from request-body JSON) can assign the trusted values. The dedicated audioTranscript fusion field is always labeled "audio-bridge" by the server regardless of what a cue declares, structurally separating it from the generic transcript field. Adds the missing FU-05 budgets (256 cues, 4096 input code units/cue, 4 KiB UTF-8/cue, 64 KiB total text), malformed-Unicode rejection, focus-window scoping, and deterministic cross-source cue reconciliation that preserves contributing-source metadata instead of silently dropping it on exact-match dedup. New logic lives in videoBridgeTranscriptContract.ts so the existing helper file only grows by a thin delegation. Bumps the Video Bridge result-cache contract version (v4 -> v5) so a cache entry computed under the old, less-restrictive normalization can never be served once the new contract is deployed. 3 of the 12 pre-existing provenance tests asserted the exact forged-source acceptance this ticket closes; updated in place with inline rationale. --- ...652-video-transcript-provenance-budgets.md | 1 + src/lib/guardrails/videoBridge.ts | 6 +- src/lib/guardrails/videoBridgeHelpers.ts | 143 ++------- .../videoBridgeTranscriptContract.ts | 297 ++++++++++++++++++ .../guardrails/videoBridgeResultCache.test.ts | 10 +- .../videoBridgeTranscriptBudgets.test.ts | 211 +++++++++++++ ...videoBridgeTranscriptCacheIdentity.test.ts | 145 +++++++++ .../videoBridgeTranscriptProvenance.test.ts | 63 +++- 8 files changed, 742 insertions(+), 134 deletions(-) create mode 100644 changelog.d/features/11652-video-transcript-provenance-budgets.md create mode 100644 src/lib/guardrails/videoBridgeTranscriptContract.ts create mode 100644 tests/unit/guardrails/videoBridgeTranscriptBudgets.test.ts create mode 100644 tests/unit/guardrails/videoBridgeTranscriptCacheIdentity.test.ts diff --git a/changelog.d/features/11652-video-transcript-provenance-budgets.md b/changelog.d/features/11652-video-transcript-provenance-budgets.md new file mode 100644 index 0000000000..de4e67a095 --- /dev/null +++ b/changelog.d/features/11652-video-transcript-provenance-budgets.md @@ -0,0 +1 @@ +- **feat(guardrails):** enforce a bounded, deterministic contract for Video Bridge transcripts — 256 cues, 4096 input code units and 4 KiB UTF-8 per cue, 64 KiB total text, malformed-Unicode rejection, focus-window scoping, cross-source reconciliation with contributing-source metadata, and a structural provenance trust boundary so caller JSON can never self-assert `embedded`/`audio-bridge` provenance ([#11652](https://github.com/diegosouzapw/OmniRoute/issues/11652)) diff --git a/src/lib/guardrails/videoBridge.ts b/src/lib/guardrails/videoBridge.ts index 4672a7dc3b..1ae3ee95b7 100644 --- a/src/lib/guardrails/videoBridge.ts +++ b/src/lib/guardrails/videoBridge.ts @@ -102,7 +102,11 @@ function waitForVideoBridgePromise(promise: Promise, signal: AbortSignal): }); } -const VIDEO_BRIDGE_RESULT_CACHE_VERSION = "v4"; +// v5 (#11652): the normalized transcript contract changed (provenance trust +// boundary, budgets, cross-source reconciliation, focus scoping) — bump so a +// cache entry computed under the old, less-restrictive normalization can +// never be served for a request processed under the new contract. +const VIDEO_BRIDGE_RESULT_CACHE_VERSION = "v5"; const VIDEO_BRIDGE_RESULT_CACHE_POLICY = "sampling-then-dedup-v2"; const VIDEO_BRIDGE_RESULT_CACHE_KEY_KIND = "video-result-v4"; const VIDEO_BRIDGE_DOWNLOAD_FLIGHT_VERSION = "v1"; diff --git a/src/lib/guardrails/videoBridgeHelpers.ts b/src/lib/guardrails/videoBridgeHelpers.ts index 4673191bae..c4bb21a4f9 100644 --- a/src/lib/guardrails/videoBridgeHelpers.ts +++ b/src/lib/guardrails/videoBridgeHelpers.ts @@ -16,6 +16,18 @@ import { type VideoSamplingMetadata, type VideoSamplingPolicy, } from "./videoBridgeRuntime"; +import { + buildNormalizedVideoTranscript, + reconcileVideoTranscriptCues, + type NormalizeVideoTranscriptOptions, + type VideoTranscriptCue, +} from "./videoBridgeTranscriptContract"; + +export type { + NormalizeVideoTranscriptOptions, + VideoTranscriptCue, + VideoTranscriptSource, +} from "./videoBridgeTranscriptContract"; export const VIDEO_BRIDGE_MAX_BYTES = 50 * 1024 * 1024; // Inline base64 shares the public 50 MiB JSON admission budget with model, @@ -91,92 +103,18 @@ export interface VideoPart { contactSheet?: boolean; } -export type VideoTranscriptSource = "audio-bridge" | "client" | "embedded"; - -export interface VideoTranscriptCue { - confidence: number; - endSeconds: number; - source: VideoTranscriptSource; - startSeconds: number; - text: string; -} - -const VIDEO_TRANSCRIPT_SOURCES: ReadonlySet = new Set([ - "audio-bridge", - "client", - "embedded", -]); - -/** Validate optional transcript metadata without ever invoking a transcription provider. */ +/** + * Validate optional transcript metadata without ever invoking a transcription + * provider. Delegates the full contract (budgets, the provenance trust + * boundary, reconciliation, and focus scoping) to videoBridgeTranscriptContract.ts + * — see that module for the security rationale. + */ export function normalizeVideoTranscript( value: unknown, - durationSeconds: number + durationSeconds: number, + options?: NormalizeVideoTranscriptOptions ): VideoTranscriptCue[] { - if (value === undefined || value === null) return []; - const rawCues = Array.isArray(value) - ? value - : value && typeof value === "object" && Array.isArray((value as Record).cues) - ? (value as Record).cues - : null; - if (!rawCues) throw new Error("Invalid video transcript: expected a cues array"); - if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { - throw new Error("Invalid video transcript duration"); - } - const seen = new Set(); - const normalized: VideoTranscriptCue[] = []; - for (const cue of rawCues) { - if (!cue || typeof cue !== "object") throw new Error("Invalid video transcript cue"); - const record = cue as Record; - const text = typeof record.text === "string" ? record.text.trim() : ""; - const source = record.source; - const startSeconds = - typeof record.startSeconds === "number" - ? record.startSeconds - : typeof record.start === "number" - ? record.start - : Number.NaN; - const endSeconds = - typeof record.endSeconds === "number" - ? record.endSeconds - : typeof record.end === "number" - ? record.end - : Number.NaN; - const confidence = record.confidence === undefined ? 1 : record.confidence; - if ( - !text || - typeof source !== "string" || - !VIDEO_TRANSCRIPT_SOURCES.has(source as VideoTranscriptSource) - ) { - throw new Error("Invalid video transcript source or provenance"); - } - if ( - !Number.isFinite(startSeconds) || - !Number.isFinite(endSeconds) || - !Number.isFinite(confidence) || - confidence < 0 || - confidence > 1 || - startSeconds < 0 || - endSeconds > durationSeconds || - endSeconds <= startSeconds - ) { - throw new Error("Invalid video transcript timestamp or confidence range"); - } - const normalizedCue = { - confidence, - endSeconds, - source: source as VideoTranscriptSource, - startSeconds, - text, - } satisfies VideoTranscriptCue; - const key = JSON.stringify(normalizedCue); - if (!seen.has(key)) { - seen.add(key); - normalized.push(normalizedCue); - } - } - return normalized.sort( - (left, right) => left.startSeconds - right.startSeconds || left.endSeconds - right.endSeconds - ); + return buildNormalizedVideoTranscript(value, durationSeconds, options); } const REPLACEABLE_VIDEO_SHAPES: ReadonlySet = new Set([ @@ -551,31 +489,6 @@ function formatTranscriptCue(cue: VideoTranscriptCue): string { return `transcript[source=${cue.source};confidence=${cue.confidence.toFixed(2)};interval=${formatVideoTimestamp(cue.startSeconds)}-${formatVideoTimestamp(cue.endSeconds)}] ${cue.text}`; } -function deduplicateVideoTranscriptCues(cues: readonly VideoTranscriptCue[]): VideoTranscriptCue[] { - const seen = new Set(); - return cues - .filter((cue) => { - const key = JSON.stringify([ - cue.source, - cue.confidence, - cue.startSeconds, - cue.endSeconds, - cue.text, - ]); - if (seen.has(key)) return false; - seen.add(key); - return true; - }) - .sort( - (left, right) => - left.startSeconds - right.startSeconds || - left.endSeconds - right.endSeconds || - left.source.localeCompare(right.source) || - left.text.localeCompare(right.text) || - left.confidence - right.confidence - ); -} - export async function describeVideoPart( part: VideoPart, options: DescribeVideoOptions, @@ -644,7 +557,8 @@ export async function describeVideoPart( : null; const separatelyRenderedTranscriptCues = normalizeVideoTranscript( part.transcript, - extracted.durationSeconds + extracted.durationSeconds, + { focusWindow } ); let transcriptCues = [...separatelyRenderedTranscriptCues]; let appendedTranscriptCues = separatelyRenderedTranscriptCues; @@ -698,7 +612,14 @@ export async function describeVideoPart( audio: async () => { normalizedFusionTranscriptCues = normalizeVideoTranscript( part.audioTranscript, - extracted.durationSeconds + extracted.durationSeconds, + // Structural trust seam: whatever the caller supplies in the + // dedicated audioTranscript field is always labeled "audio-bridge" + // by this fusion channel, regardless of any per-cue `source` the + // caller declared. This is not an authenticity claim about the + // caller's own transcription — only that it arrived through the + // audio-bridge fusion field rather than the generic transcript. + { trustedSource: "audio-bridge", focusWindow } ); return { observations: normalizedFusionTranscriptCues.map((cue) => ({ @@ -726,7 +647,7 @@ export async function describeVideoPart( ...(fused.failures ? { failures: fused.failures } : {}), }; const fusedAudioCues = fused.audioAvailable ? normalizedFusionTranscriptCues : []; - transcriptCues = deduplicateVideoTranscriptCues([...transcriptCues, ...fusedAudioCues]); + transcriptCues = reconcileVideoTranscriptCues([...transcriptCues, ...fusedAudioCues]); const fusedVideoTimeline = fused.observations.flatMap((observation) => observation.source === "video" ? [ diff --git a/src/lib/guardrails/videoBridgeTranscriptContract.ts b/src/lib/guardrails/videoBridgeTranscriptContract.ts new file mode 100644 index 0000000000..a5655da585 --- /dev/null +++ b/src/lib/guardrails/videoBridgeTranscriptContract.ts @@ -0,0 +1,297 @@ +/** + * FU-05 transcript contract: deterministic budgets, a structural provenance + * trust boundary, and cross-source reconciliation for Video Bridge + * transcripts. Extracted from videoBridgeHelpers.ts so the frozen-adjacent + * helper file only grows by a thin delegation (see #11652). + * + * Trust boundary: `normalizeVideoTranscript`'s only public entry point is + * `buildNormalizedVideoTranscript`. A caller-supplied cue can only ever + * declare `source: "client"` — declaring "embedded" or "audio-bridge" is + * self-asserted provenance and is silently reclassified to "client" because + * there is no way for the server to verify it came from an embedded-subtitle + * extractor or the audio-bridge pipeline. The ONLY way to obtain a trusted + * "embedded"/"audio-bridge" cue is for server-owned code to pass + * `options.trustedSource` explicitly — that option can never be reached by + * deserializing request-body JSON (it is a second, code-only argument), so + * the trust decision is structural rather than a flag the caller can flip. + */ + +export type VideoTranscriptSource = "audio-bridge" | "client" | "embedded"; + +export interface VideoTranscriptCue { + confidence: number; + endSeconds: number; + source: VideoTranscriptSource; + startSeconds: number; + text: string; + /** + * Present only when reconciliation merged cues declaring 2+ distinct + * sources into one cue (see `reconcileVideoTranscriptCues`). Absent for + * the common single-source case so existing exact-shape assertions are + * unaffected. + */ + contributingSources?: VideoTranscriptSource[]; +} + +export interface VideoTranscriptFocusWindow { + endSeconds: number; + startSeconds: number; +} + +export interface NormalizeVideoTranscriptOptions { + /** Structural trust seam — server-owned adapters only, never derived from request JSON. */ + trustedSource?: VideoTranscriptSource; + /** Scope (filter + clip) the resulting cues to this window; `null`/omitted keeps everything. */ + focusWindow?: VideoTranscriptFocusWindow | null; +} + +const VIDEO_TRANSCRIPT_SOURCES: ReadonlySet = new Set([ + "audio-bridge", + "client", + "embedded", +]); + +export const VIDEO_TRANSCRIPT_MAX_CUES = 256; +export const VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS = 4096; +export const VIDEO_TRANSCRIPT_MAX_CUE_UTF8_BYTES = 4 * 1024; +export const VIDEO_TRANSCRIPT_MAX_TOTAL_UTF8_BYTES = 64 * 1024; + +// Bounded (no unbounded quantifiers) — matches an unpaired high surrogate not +// followed by a low surrogate, or an unpaired low surrogate not preceded by a +// high surrogate. Safe against catastrophic backtracking per AGENTS.md ReDoS rule. +const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/; + +const SOURCE_PRIORITY: Record = { + client: 0, + "audio-bridge": 1, + embedded: 2, +}; + +function extractRawCues(value: unknown): unknown[] { + if (Array.isArray(value)) return value; + if ( + value && + typeof value === "object" && + Array.isArray((value as Record).cues) + ) { + return (value as Record).cues as unknown[]; + } + throw new Error("Invalid video transcript: expected a cues array"); +} + +function extractCueText(record: Record): string { + const raw = record.text; + if (typeof raw !== "string") throw new Error("Invalid video transcript cue text"); + if (raw.length > VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS) { + throw new Error( + `Video transcript cue exceeds the maximum of ${VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS} input code units` + ); + } + if (LONE_SURROGATE.test(raw)) { + throw new Error("Invalid video transcript cue text encoding"); + } + const text = raw.normalize("NFC").trim(); + if (!text) throw new Error("Invalid video transcript cue text"); + if (Buffer.byteLength(text, "utf8") > VIDEO_TRANSCRIPT_MAX_CUE_UTF8_BYTES) { + throw new Error( + `Video transcript cue exceeds the maximum of ${VIDEO_TRANSCRIPT_MAX_CUE_UTF8_BYTES} UTF-8 bytes` + ); + } + return text; +} + +function resolveCueSource( + record: Record, + trustedSource: VideoTranscriptSource | undefined +): VideoTranscriptSource { + if (trustedSource) return trustedSource; + const declared = record.source; + if ( + typeof declared !== "string" || + !VIDEO_TRANSCRIPT_SOURCES.has(declared as VideoTranscriptSource) + ) { + throw new Error("Invalid video transcript source or provenance"); + } + // Only a server-owned adapter may assign "embedded"/"audio-bridge" via the + // trustedSource seam above; any caller-declared value collapses to "client". + return "client"; +} + +function resolveCueTimestamps(record: Record): { + startSeconds: number; + endSeconds: number; +} { + const startSeconds = + typeof record.startSeconds === "number" + ? record.startSeconds + : typeof record.start === "number" + ? record.start + : Number.NaN; + const endSeconds = + typeof record.endSeconds === "number" + ? record.endSeconds + : typeof record.end === "number" + ? record.end + : Number.NaN; + return { startSeconds, endSeconds }; +} + +function assertValidCueBounds( + startSeconds: number, + endSeconds: number, + confidence: unknown, + durationSeconds: number +): void { + if ( + !Number.isFinite(startSeconds) || + !Number.isFinite(endSeconds) || + !Number.isFinite(confidence as number) || + (confidence as number) < 0 || + (confidence as number) > 1 || + startSeconds < 0 || + endSeconds > durationSeconds || + endSeconds <= startSeconds + ) { + throw new Error("Invalid video transcript timestamp or confidence range"); + } +} + +function parseVideoTranscriptCue( + cue: unknown, + durationSeconds: number, + trustedSource: VideoTranscriptSource | undefined +): VideoTranscriptCue { + if (!cue || typeof cue !== "object") throw new Error("Invalid video transcript cue"); + const record = cue as Record; + const text = extractCueText(record); + const source = resolveCueSource(record, trustedSource); + const { startSeconds, endSeconds } = resolveCueTimestamps(record); + const confidence = record.confidence === undefined ? 1 : record.confidence; + assertValidCueBounds(startSeconds, endSeconds, confidence, durationSeconds); + return { confidence: confidence as number, endSeconds, source, startSeconds, text }; +} + +interface VideoTranscriptCluster { + startSeconds: number; + endSeconds: number; + text: string; + members: VideoTranscriptCue[]; +} + +function findOverlappingCluster( + clusters: readonly VideoTranscriptCluster[], + cue: VideoTranscriptCue +): VideoTranscriptCluster | undefined { + return clusters.find( + (cluster) => + cluster.text === cue.text && + cue.startSeconds < cluster.endSeconds && + cue.endSeconds > cluster.startSeconds + ); +} + +function mergeVideoTranscriptCluster(cluster: VideoTranscriptCluster): VideoTranscriptCue { + const { members } = cluster; + const distinctSources = Array.from(new Set(members.map((member) => member.source))).sort( + (left, right) => SOURCE_PRIORITY[left] - SOURCE_PRIORITY[right] + ); + const winningSource = distinctSources[distinctSources.length - 1]; + return { + confidence: Math.max(...members.map((member) => member.confidence)), + endSeconds: Math.max(...members.map((member) => member.endSeconds)), + source: winningSource, + startSeconds: Math.min(...members.map((member) => member.startSeconds)), + text: cluster.text, + ...(distinctSources.length > 1 ? { contributingSources: distinctSources } : {}), + }; +} + +/** + * Reconcile duplicate and time-overlapping cues sharing identical text — + * possibly contributed by different sources/calls — into one deterministic + * cue per cluster, preserving which sources contributed when more than one + * did. Replaces naive exact-match deduplication, which silently discarded + * the losing source's identity. + */ +export function reconcileVideoTranscriptCues( + cues: readonly VideoTranscriptCue[] +): VideoTranscriptCue[] { + const sorted = [...cues].sort( + (left, right) => + left.startSeconds - right.startSeconds || + left.endSeconds - right.endSeconds || + left.text.localeCompare(right.text) || + SOURCE_PRIORITY[left.source] - SOURCE_PRIORITY[right.source] + ); + const clusters: VideoTranscriptCluster[] = []; + for (const cue of sorted) { + const cluster = findOverlappingCluster(clusters, cue); + if (cluster) { + cluster.members.push(cue); + cluster.startSeconds = Math.min(cluster.startSeconds, cue.startSeconds); + cluster.endSeconds = Math.max(cluster.endSeconds, cue.endSeconds); + } else { + clusters.push({ + endSeconds: cue.endSeconds, + members: [cue], + startSeconds: cue.startSeconds, + text: cue.text, + }); + } + } + return clusters + .map(mergeVideoTranscriptCluster) + .sort((left, right) => left.startSeconds - right.startSeconds || left.endSeconds - right.endSeconds); +} + +/** Filter cues to those overlapping `focusWindow`, clipping their bounds to it. */ +export function scopeVideoTranscriptCuesToFocusWindow( + cues: readonly VideoTranscriptCue[], + focusWindow: VideoTranscriptFocusWindow | null | undefined +): VideoTranscriptCue[] { + if (!focusWindow) return [...cues]; + return cues + .filter( + (cue) => cue.startSeconds < focusWindow.endSeconds && cue.endSeconds > focusWindow.startSeconds + ) + .map((cue) => ({ + ...cue, + endSeconds: Math.min(cue.endSeconds, focusWindow.endSeconds), + startSeconds: Math.max(cue.startSeconds, focusWindow.startSeconds), + })); +} + +/** + * Validate optional transcript metadata without ever invoking a transcription + * provider. Enforces the FU-05 budgets (256 cues, 4096 input code units/cue, + * 4 KiB UTF-8/cue, 64 KiB total), the provenance trust boundary described at + * the top of this file, cross-source reconciliation, and focus-window scoping. + */ +export function buildNormalizedVideoTranscript( + value: unknown, + durationSeconds: number, + options: NormalizeVideoTranscriptOptions = {} +): VideoTranscriptCue[] { + if (value === undefined || value === null) return []; + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + throw new Error("Invalid video transcript duration"); + } + const rawCues = extractRawCues(value); + if (rawCues.length > VIDEO_TRANSCRIPT_MAX_CUES) { + throw new Error(`Video transcript exceeds the maximum of ${VIDEO_TRANSCRIPT_MAX_CUES} cues`); + } + const parsed: VideoTranscriptCue[] = []; + let totalBytes = 0; + for (const rawCue of rawCues) { + const cue = parseVideoTranscriptCue(rawCue, durationSeconds, options.trustedSource); + totalBytes += Buffer.byteLength(cue.text, "utf8"); + if (totalBytes > VIDEO_TRANSCRIPT_MAX_TOTAL_UTF8_BYTES) { + throw new Error( + `Video transcript exceeds the maximum total of ${VIDEO_TRANSCRIPT_MAX_TOTAL_UTF8_BYTES} UTF-8 bytes` + ); + } + parsed.push(cue); + } + const reconciled = reconcileVideoTranscriptCues(parsed); + return scopeVideoTranscriptCuesToFocusWindow(reconciled, options.focusWindow ?? null); +} diff --git a/tests/unit/guardrails/videoBridgeResultCache.test.ts b/tests/unit/guardrails/videoBridgeResultCache.test.ts index fb7c50ec8c..146af87380 100644 --- a/tests/unit/guardrails/videoBridgeResultCache.test.ts +++ b/tests/unit/guardrails/videoBridgeResultCache.test.ts @@ -430,7 +430,7 @@ test("result-cache metadata carries the exact visual dedup policy identity", asy ); assert.ok(storedMetadata); - assert.equal(storedMetadata.cacheVersion, "v4"); + assert.equal(storedMetadata.cacheVersion, "v5"); assert.equal(storedMetadata.policyVersion, "sampling-then-dedup-v2"); assert.equal(storedMetadata.dedupPolicyVersion, "grayscale-16x16-mean-cells-v2"); assert.equal(storedMetadata.dedupThreshold, 0.04); @@ -446,12 +446,12 @@ test("a corrupt result-cache payload is discarded and recomputed", async () => { producerModel: "openai/gpt-4o-mini", metadata: { analysisMode: "full", - cacheVersion: "v4", + cacheVersion: "v5", dedupCandidateFrameCount: 16, dedupPolicyVersion: "grayscale-16x16-mean-cells-v2", dedupThreshold: 0.04, policyVersion: "sampling-then-dedup-v2", - extractorVersion: "v4", + extractorVersion: "v5", strategy: "uniform", model: "openai/gpt-4o-mini", prompt: "FU-01 corrupt cache", @@ -510,12 +510,12 @@ test("invalid numeric result-cache metadata is deleted and recomputed", async (t const cachedValue = "[Video description: cached numeric metadata]"; const validMetadata = (): Record => ({ analysisMode: "full", - cacheVersion: "v4", + cacheVersion: "v5", dedupCandidateFrameCount: 16, dedupPolicyVersion: "grayscale-16x16-mean-cells-v2", dedupThreshold: 0.04, policyVersion: "sampling-then-dedup-v2", - extractorVersion: "v4", + extractorVersion: "v5", strategy: "uniform", model: "openai/gpt-4o-mini", prompt: "FU-01 numeric cache validation", diff --git a/tests/unit/guardrails/videoBridgeTranscriptBudgets.test.ts b/tests/unit/guardrails/videoBridgeTranscriptBudgets.test.ts new file mode 100644 index 0000000000..1b776484fd --- /dev/null +++ b/tests/unit/guardrails/videoBridgeTranscriptBudgets.test.ts @@ -0,0 +1,211 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + describeVideoPart, + normalizeVideoTranscript, + type VideoCaptionFrame, +} from "../../../src/lib/guardrails/videoBridgeHelpers"; +import { + VIDEO_TRANSCRIPT_MAX_CUES, + VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS, + VIDEO_TRANSCRIPT_MAX_CUE_UTF8_BYTES, + VIDEO_TRANSCRIPT_MAX_TOTAL_UTF8_BYTES, +} from "../../../src/lib/guardrails/videoBridgeTranscriptContract"; + +function cue(overrides: Partial> = {}): Record { + return { text: "cue", start: 0, end: 1, source: "client", ...overrides }; +} + +// --------------------------------------------------------------------------- +// Budgets (#11652 scope): 256 cues, 4096 input code units/cue, 4 KiB UTF-8/cue, +// 64 KiB total text. All enforced deterministically with a clear error. +// --------------------------------------------------------------------------- + +test("accepts exactly the maximum cue count and rejects one more", () => { + const atLimit = Array.from({ length: VIDEO_TRANSCRIPT_MAX_CUES }, (_unused, index) => + cue({ text: `cue-${index}`, start: index, end: index + 0.5 }) + ); + assert.equal(normalizeVideoTranscript({ cues: atLimit }, VIDEO_TRANSCRIPT_MAX_CUES + 1).length, VIDEO_TRANSCRIPT_MAX_CUES); + + const overLimit = [...atLimit, cue({ text: "one-too-many", start: VIDEO_TRANSCRIPT_MAX_CUES })]; + assert.throws( + () => normalizeVideoTranscript({ cues: overLimit }, VIDEO_TRANSCRIPT_MAX_CUES + 2), + /256 cues/ + ); +}); + +test("rejects a cue whose raw text exceeds the maximum input code units", () => { + const tooLong = "a".repeat(VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS + 1); + assert.throws( + () => normalizeVideoTranscript({ cues: [cue({ text: tooLong })] }, 10), + /input code units/ + ); + // Exactly at the limit is accepted. + const atLimit = "a".repeat(VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS); + assert.equal(normalizeVideoTranscript({ cues: [cue({ text: atLimit })] }, 10)[0]?.text.length, VIDEO_TRANSCRIPT_MAX_CUE_CODE_UNITS); +}); + +test("rejects a cue whose UTF-8 encoding exceeds the maximum per-cue size", () => { + // Each "é" (U+00E9) is 1 code unit but 2 UTF-8 bytes, so this trips the byte + // budget while staying well under the code-unit budget. + const wideText = "é".repeat(VIDEO_TRANSCRIPT_MAX_CUE_UTF8_BYTES / 2 + 1); + assert.throws( + () => normalizeVideoTranscript({ cues: [cue({ text: wideText })] }, 10), + /UTF-8/ + ); +}); + +test("rejects a transcript whose combined UTF-8 text exceeds the total budget", () => { + const perCueBytes = 1024; + const cueCount = Math.ceil(VIDEO_TRANSCRIPT_MAX_TOTAL_UTF8_BYTES / perCueBytes) + 1; + const cues = Array.from({ length: cueCount }, (_unused, index) => + cue({ text: "b".repeat(perCueBytes), start: index, end: index + 0.5 }) + ); + assert.throws( + () => normalizeVideoTranscript({ cues }, cueCount + 1), + /total.*UTF-8|maximum total/i + ); +}); + +// --------------------------------------------------------------------------- +// Malformed Unicode +// --------------------------------------------------------------------------- + +test("rejects cue text containing an unpaired surrogate", () => { + assert.throws( + () => normalizeVideoTranscript({ cues: [cue({ text: "abc\uD800def" })] }, 10), + /encoding/i + ); + assert.throws( + () => normalizeVideoTranscript({ cues: [cue({ text: "abc\uDC00def" })] }, 10), + /encoding/i + ); +}); + +test("accepts well-formed surrogate pairs (astral text)", () => { + const cues = normalizeVideoTranscript({ cues: [cue({ text: "hello \u{1F600}" })] }, 10); + assert.equal(cues[0]?.text, "hello \u{1F600}"); +}); + +// --------------------------------------------------------------------------- +// Provenance forgery (structural trust boundary) +// --------------------------------------------------------------------------- + +test("a forged embedded source from request-body JSON is reclassified to client", () => { + const cues = normalizeVideoTranscript({ cues: [cue({ source: "embedded" })] }, 10); + assert.equal(cues[0]?.source, "client"); +}); + +test("a forged audio-bridge source from request-body JSON is reclassified to client", () => { + const cues = normalizeVideoTranscript({ cues: [cue({ source: "audio-bridge" })] }, 10); + assert.equal(cues[0]?.source, "client"); +}); + +test("the trustedSource option is a code-only seam that overrides any caller-declared source", () => { + const embeddedCues = normalizeVideoTranscript( + { cues: [cue({ source: "client" })] }, + 10, + { trustedSource: "embedded" } + ); + assert.equal(embeddedCues[0]?.source, "embedded"); + + const audioBridgeCues = normalizeVideoTranscript( + { cues: [cue({ source: "unknown-junk" })] }, + 10, + { trustedSource: "audio-bridge" } + ); + assert.equal(audioBridgeCues[0]?.source, "audio-bridge"); +}); + +// --------------------------------------------------------------------------- +// Focus scoping +// --------------------------------------------------------------------------- + +test("normalizeVideoTranscript scopes cues to the focus window, dropping non-overlapping cues", () => { + const cues = normalizeVideoTranscript( + { + cues: [ + cue({ text: "before", start: 0, end: 1 }), + cue({ text: "inside", start: 4, end: 6 }), + cue({ text: "spanning", start: 7, end: 12 }), + cue({ text: "after", start: 20, end: 21 }), + ], + }, + 30, + { focusWindow: { startSeconds: 3, endSeconds: 10 } } + ); + + assert.deepEqual( + cues.map((entry) => entry.text), + ["inside", "spanning"] + ); + const spanning = cues.find((entry) => entry.text === "spanning"); + assert.equal(spanning?.startSeconds, 7); + assert.equal(spanning?.endSeconds, 10, "clipped to the focus window end"); +}); + +test("describeVideoPart scopes transcript cues to the effective focus window end-to-end", async () => { + const frames: VideoCaptionFrame[] = [ + { dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 5 }, + ]; + const described = await describeVideoPart( + { + container: "messages", + messageIndex: 0, + partIndex: 0, + ref: "data:video/mp4;base64,AA==", + shape: "data_uri_string", + transcript: { + cues: [ + { text: "outside focus", start: 0, end: 1, source: "client" }, + { text: "inside focus", start: 4, end: 6, source: "client" }, + ], + }, + }, + { frameCount: 1, focusWindow: { startSeconds: 3, endSeconds: 10 }, timeoutMs: 1000 }, + async () => "a scene", + { extractFrames: async () => ({ durationSeconds: 20, frames }) } + ); + + assert.deepEqual( + described.transcriptCues?.map((entry) => entry.text), + ["inside focus"] + ); + assert.doesNotMatch(described.description, /outside focus/); +}); + +// --------------------------------------------------------------------------- +// Cross-source / within-call reconciliation +// --------------------------------------------------------------------------- + +test("reconciles overlapping same-text cues within a single normalizeVideoTranscript call", () => { + const cues = normalizeVideoTranscript( + { + cues: [ + cue({ text: "same words", start: 1, end: 3, confidence: 0.4 }), + cue({ text: "same words", start: 2, end: 4, confidence: 0.9 }), + ], + }, + 10 + ); + + assert.equal(cues.length, 1); + assert.equal(cues[0]?.startSeconds, 1); + assert.equal(cues[0]?.endSeconds, 4); + assert.equal(cues[0]?.confidence, 0.9, "keeps the higher-confidence reading"); + assert.equal(cues[0]?.contributingSources, undefined, "single-source merges add no metadata"); +}); + +test("non-overlapping cues with identical text are kept distinct", () => { + const cues = normalizeVideoTranscript( + { + cues: [ + cue({ text: "repeated line", start: 1, end: 2 }), + cue({ text: "repeated line", start: 8, end: 9 }), + ], + }, + 10 + ); + assert.equal(cues.length, 2); +}); diff --git a/tests/unit/guardrails/videoBridgeTranscriptCacheIdentity.test.ts b/tests/unit/guardrails/videoBridgeTranscriptCacheIdentity.test.ts new file mode 100644 index 0000000000..7ec1235017 --- /dev/null +++ b/tests/unit/guardrails/videoBridgeTranscriptCacheIdentity.test.ts @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts"; +import type { BridgeCacheEntry } from "../../../src/lib/guardrails/modalityBridge/bridgeCache.ts"; + +// FU-05 (#11652): "the normalized transcript contract must be part of cache +// identity ... cache hits cannot cross transcript identity or provenance +// changes." These tests exercise the third named seam alongside +// normalizeVideoTranscript/describeVideoPart: the guardrail's result-cache +// key, which already folds in a fingerprint of the raw transcript/ +// audioTranscript payload (videoBridge.ts::buildVideoResultCacheKey). + +function payloadWithTranscript(transcript: unknown): Record { + return { + model: "example/text-only", + messages: [ + { + role: "user", + content: [ + { + type: "input_video", + video_url: "data:video/mp4;base64,VFJBTlNDUklQVC1DQUNIRQ==", + transcript, + }, + ], + }, + ], + }; +} + +function makeBridge(onDescribe: () => Promise<{ description: string; durationSeconds: number; framesRequested: number; framesUsed: number }>) { + return new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeCacheEnabled: true, + modalityBridgeCacheMaxEntries: 11, + modalityBridgeCacheTtlMinutes: 5, + modalityBridgeVideoEnabled: true, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + modalityBridgeVisionPrompt: "FU-05 transcript cache identity", + }), + getCapabilities: () => ({ supportsVideo: false }), + selectVisionModel: async () => "openai/gpt-4o-mini", + describePart: onDescribe, + }, + }); +} + +test("a cache hit does not cross a transcript provenance change on an otherwise identical video", async () => { + let describeCalls = 0; + const bridge = makeBridge(async () => { + describeCalls += 1; + return { + description: "[Video description: cache identity probe]", + durationSeconds: 5, + framesRequested: 1, + framesUsed: 1, + }; + }); + + const clientDeclared = { + cues: [{ text: "same words", start: 1, end: 2, source: "client" }], + }; + const audioBridgeDeclared = { + cues: [{ text: "same words", start: 1, end: 2, source: "audio-bridge" }], + }; + + await bridge.preCall(payloadWithTranscript(clientDeclared), {}); + await bridge.preCall(payloadWithTranscript(clientDeclared), {}); + await bridge.preCall(payloadWithTranscript(audioBridgeDeclared), {}); + + assert.equal( + describeCalls, + 2, + "the repeated identical (video, transcript) pair must reuse the cached result, " + + "but the raw provenance change must force a miss even though both sources are " + + "reclassified to client after normalization" + ); +}); + +test("a cache hit does not cross a transcript identity change (different cues, same video)", async () => { + let describeCalls = 0; + const bridge = makeBridge(async () => { + describeCalls += 1; + return { + description: "[Video description: cache identity probe]", + durationSeconds: 5, + framesRequested: 1, + framesUsed: 1, + }; + }); + + await bridge.preCall( + payloadWithTranscript({ cues: [{ text: "first cue", start: 1, end: 2, source: "client" }] }), + {} + ); + await bridge.preCall( + payloadWithTranscript({ cues: [{ text: "second cue", start: 1, end: 2, source: "client" }] }), + {} + ); + + assert.equal(describeCalls, 2, "different transcript content must never share a cache entry"); +}); + +test("the result-cache contract version was bumped for the FU-05 normalization change", async () => { + let storedMetadata: Record | undefined; + const bridge = new VideoBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeCacheEnabled: true, + modalityBridgeVideoEnabled: true, + modalityBridgeVideoModel: "openai/gpt-4o-mini", + modalityBridgeVisionPrompt: "FU-05 cache version", + }), + getCapabilities: () => ({ supportsVideo: false }), + selectVisionModel: async () => "openai/gpt-4o-mini", + resultCache: { + delete: () => undefined, + getEntry: () => undefined, + setEntry: (_key: string, entry: BridgeCacheEntry) => { + storedMetadata = entry.metadata as Record; + }, + }, + describePart: async () => ({ + description: "[Video description: version probe]", + durationSeconds: 1, + framesRequested: 1, + framesUsed: 1, + }), + }, + }); + + await bridge.preCall( + payloadWithTranscript({ cues: [{ text: "cue", start: 0, end: 1, source: "client" }] }), + {} + ); + + assert.ok(storedMetadata); + assert.notEqual( + storedMetadata?.cacheVersion, + "v4", + "a cache entry computed under the pre-FU-05 normalization contract must never match" + ); +}); diff --git a/tests/unit/guardrails/videoBridgeTranscriptProvenance.test.ts b/tests/unit/guardrails/videoBridgeTranscriptProvenance.test.ts index 22425ef26a..82d274bb2a 100644 --- a/tests/unit/guardrails/videoBridgeTranscriptProvenance.test.ts +++ b/tests/unit/guardrails/videoBridgeTranscriptProvenance.test.ts @@ -18,7 +18,14 @@ async function jpegFrame(color: string, timestampSeconds: number): Promise