mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 10:52:17 +03:00
3b00535d04
FU-05 core (closes #11652): caller-supplied Video Bridge transcripts had no bounded, deterministic contract — a client could self-assert source: "embedded"/"audio-bridge" and it was accepted verbatim. normalizeVideoTranscript gained a code-only trustedSource seam unreachable from request-body JSON; without it, any cue declaring embedded/audio-bridge is reclassified to client. Added budgets (256 cues, 4096 code units/cue, 4KiB/cue, 64KiB total), malformed-surrogate rejection, focus-window scoping, deterministic cross-source reconciliation, and bumped the result-cache version v4→v5 so old-contract cache entries can never serve new-contract requests. All 187 videoBridge* tests pass (185 pass, 2 unrelated pre-existing skips).
456 lines
16 KiB
TypeScript
456 lines
16 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import sharp from "sharp";
|
|
|
|
import {
|
|
describeVideoPart,
|
|
normalizeVideoTranscript,
|
|
type VideoCaptionFrame,
|
|
} from "../../../src/lib/guardrails/videoBridgeHelpers";
|
|
|
|
async function jpegFrame(color: string, timestampSeconds: number): Promise<VideoCaptionFrame> {
|
|
const bytes = await sharp({
|
|
create: { background: color, channels: 3, height: 24, width: 32 },
|
|
})
|
|
.jpeg()
|
|
.toBuffer();
|
|
return { dataUri: `data:image/jpeg;base64,${bytes.toString("base64")}`, timestampSeconds };
|
|
}
|
|
|
|
// #11652: the untrusted (default, no `trustedSource` option) path is the
|
|
// ONLY entry point request-body JSON can reach. A caller cannot verify their
|
|
// own claim of "audio-bridge"/"embedded" provenance, so both self-asserted
|
|
// values are reclassified to "client" — only a server-owned adapter passing
|
|
// `trustedSource` explicitly (a seam request JSON cannot reach) can produce
|
|
// them. This intentionally changes the pre-#11652 behavior, which accepted
|
|
// a caller-declared "audio-bridge" source verbatim.
|
|
test("accepts only provenance-bearing transcript cues, reclassifies forged provenance, and deduplicates exact repeats", () => {
|
|
const cues = normalizeVideoTranscript(
|
|
{
|
|
cues: [
|
|
{ text: "hello", start: 1, end: 3, source: "client", confidence: 0.8 },
|
|
{ text: "hello", start: 1, end: 3, source: "client", confidence: 0.8 },
|
|
{ text: "world", startSeconds: 3, endSeconds: 5, source: "audio-bridge" },
|
|
],
|
|
},
|
|
10
|
|
);
|
|
|
|
assert.deepEqual(cues, [
|
|
{ text: "hello", startSeconds: 1, endSeconds: 3, source: "client", confidence: 0.8 },
|
|
{ text: "world", startSeconds: 3, endSeconds: 5, source: "client", confidence: 1 },
|
|
]);
|
|
});
|
|
|
|
test("rejects untrusted sources, malformed cues, and out-of-range timestamps", () => {
|
|
assert.throws(
|
|
() =>
|
|
normalizeVideoTranscript({ cues: [{ text: "x", start: 1, end: 2, source: "unknown" }] }, 10),
|
|
/source/i
|
|
);
|
|
assert.throws(
|
|
() =>
|
|
normalizeVideoTranscript({ cues: [{ text: "x", start: -1, end: 2, source: "client" }] }, 10),
|
|
/timestamp|range/i
|
|
);
|
|
assert.throws(
|
|
() =>
|
|
normalizeVideoTranscript({ cues: [{ text: "x", start: 4, end: 4, source: "embedded" }] }, 10),
|
|
/timestamp|range/i
|
|
);
|
|
assert.throws(
|
|
() =>
|
|
normalizeVideoTranscript(
|
|
{ cues: [{ text: "x", start: 9, end: 11, source: "embedded" }] },
|
|
10
|
|
),
|
|
/timestamp|range/i
|
|
);
|
|
});
|
|
|
|
// #11652: `part.transcript` is the generic, fully caller-controlled field —
|
|
// a cue declaring source: "audio-bridge" there is forged provenance (that
|
|
// label is reserved for the dedicated audioTranscript fusion field) and is
|
|
// reclassified to "client". Pre-#11652 this asserted the forged label was
|
|
// preserved verbatim; that was the exact bug this ticket closes.
|
|
test("keeps transcript metadata attached and reclassifies a forged source on the described video output", async () => {
|
|
const frames: VideoCaptionFrame[] = [
|
|
{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 2 },
|
|
{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 8 },
|
|
];
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
transcript: {
|
|
cues: [{ text: "spoken words", start: 1, end: 3, source: "audio-bridge", confidence: 0.9 }],
|
|
},
|
|
},
|
|
{ frameCount: 2, timeoutMs: 1000 },
|
|
async () => "a scene",
|
|
{
|
|
extractFrames: async () => ({ durationSeconds: 10, frames }),
|
|
}
|
|
);
|
|
|
|
assert.equal(described.transcriptCues?.length, 1);
|
|
assert.equal(described.transcriptCues?.[0]?.source, "client");
|
|
assert.match(described.description, /transcript\[source=client;confidence=0\.90/);
|
|
assert.match(described.description, /spoken words/);
|
|
});
|
|
|
|
test("fuses an explicitly supplied audio-bridge track without starting STT", async () => {
|
|
let captionCalls = 0;
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "audio cue", start: 1, end: 3, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 1, timeoutMs: 1000 },
|
|
async () => {
|
|
captionCalls += 1;
|
|
return "visual cue";
|
|
},
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 5,
|
|
frames: [{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 2 }],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.equal(captionCalls, 1);
|
|
assert.equal(described.transcriptCues?.[0]?.source, "audio-bridge");
|
|
assert.match(described.description, /audio cue/);
|
|
assert.deepEqual(described.fusion, {
|
|
audioAvailable: true,
|
|
videoAvailable: true,
|
|
partial: false,
|
|
});
|
|
});
|
|
|
|
test("renders fused video and audio observations in chronological order", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "middle audio", start: 3, end: 4, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 2, timeoutMs: 1000 },
|
|
async (_frame, timestampSeconds) => `visual at ${timestampSeconds}`,
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 6,
|
|
frames: [
|
|
{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 1 },
|
|
{ dataUri: "data:image/jpeg;base64,AQ==", timestampSeconds: 5 },
|
|
],
|
|
}),
|
|
}
|
|
);
|
|
|
|
const firstVisual = described.description.indexOf("frame@t=00:01.000 visual at 1");
|
|
const audio = described.description.indexOf("middle audio");
|
|
const secondVisual = described.description.indexOf("frame@t=00:05.000 visual at 5");
|
|
assert.ok(firstVisual >= 0);
|
|
assert.ok(audio > firstVisual);
|
|
assert.ok(secondVisual > audio);
|
|
assert.equal(described.transcriptCues?.[0]?.text, "middle audio");
|
|
assert.deepEqual(described.fusion, {
|
|
audioAvailable: true,
|
|
videoAvailable: true,
|
|
partial: false,
|
|
});
|
|
});
|
|
|
|
test("preserves provided and fused transcript cues without rendering either twice", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
transcript: {
|
|
cues: [
|
|
{ text: "provided cue", start: 0.25, end: 0.75, source: "client" },
|
|
{ text: "late client cue", start: 4, end: 4.5, source: "client" },
|
|
],
|
|
},
|
|
audioTranscript: {
|
|
cues: [{ text: "fused cue", start: 2, end: 3, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 1, timeoutMs: 1000 },
|
|
async () => "visual cue",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 5,
|
|
frames: [{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 1 }],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.deepEqual(
|
|
described.transcriptCues?.map((cue) => cue.text),
|
|
["provided cue", "fused cue", "late client cue"]
|
|
);
|
|
assert.equal(described.description.split("provided cue").length - 1, 1);
|
|
assert.equal(described.description.split("fused cue").length - 1, 1);
|
|
assert.equal(described.description.split("late client cue").length - 1, 1);
|
|
const provided = described.description.indexOf("provided cue");
|
|
const visual = described.description.indexOf("frame@t=00:01.000 visual cue");
|
|
const fused = described.description.indexOf("fused cue");
|
|
const lateProvided = described.description.indexOf("late client cue");
|
|
assert.ok(provided >= 0);
|
|
assert.ok(visual > provided);
|
|
assert.ok(fused > visual);
|
|
assert.ok(lateProvided > fused);
|
|
assert.deepEqual(described.fusion, {
|
|
audioAvailable: true,
|
|
videoAvailable: true,
|
|
partial: false,
|
|
});
|
|
});
|
|
|
|
test("deduplicates an exact cue shared by provided and fused transcript tracks", async () => {
|
|
const sharedCue = {
|
|
confidence: 0.8,
|
|
end: 3,
|
|
source: "audio-bridge" as const,
|
|
start: 2,
|
|
text: "shared audio cue",
|
|
};
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
transcript: { cues: [sharedCue] },
|
|
audioTranscript: { cues: [sharedCue] },
|
|
},
|
|
{ frameCount: 1, timeoutMs: 1000 },
|
|
async () => "visual cue",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 5,
|
|
frames: [{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 1 }],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.equal(described.transcriptCues?.length, 1);
|
|
assert.equal(described.description.split("shared audio cue").length - 1, 1);
|
|
});
|
|
|
|
// #11652: pre-#11652 this test proved a caller-declared "embedded" source
|
|
// survived verbatim from `part.transcript` — exactly the forgery this ticket
|
|
// closes. Rewritten to prove the new contract instead: the generic
|
|
// `transcript` field always reclassifies a declared "embedded"/"audio-bridge"
|
|
// source to "client" (no way to verify the claim), the dedicated
|
|
// `audioTranscript` fusion field always forces "audio-bridge" regardless of
|
|
// what the caller declared there, and cues that end up overlapping in time
|
|
// with identical text across the two channels are reconciled into one cue
|
|
// that keeps every contributing source instead of silently dropping one.
|
|
test("labels transcript cues by channel and reconciles overlapping cross-channel duplicates with contributing-source metadata", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
transcript: {
|
|
cues: [
|
|
{ confidence: 0.8, end: 2, source: "client" as const, start: 1, text: "client-only cue" },
|
|
{ confidence: 0.7, end: 4, source: "embedded" as const, start: 3, text: "shared cue" },
|
|
],
|
|
},
|
|
audioTranscript: {
|
|
cues: [{ confidence: 0.9, end: 4, source: "client" as const, start: 3, text: "shared cue" }],
|
|
},
|
|
},
|
|
{ frameCount: 1, timeoutMs: 1000 },
|
|
async () => "visual cue",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 5,
|
|
frames: [{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 0.5 }],
|
|
}),
|
|
}
|
|
);
|
|
|
|
const cues = described.transcriptCues ?? [];
|
|
const clientOnly = cues.find((cue) => cue.text === "client-only cue");
|
|
const shared = cues.find((cue) => cue.text === "shared cue");
|
|
|
|
assert.equal(cues.length, 2);
|
|
assert.equal(clientOnly?.source, "client");
|
|
assert.equal(clientOnly?.contributingSources, undefined);
|
|
assert.equal(shared?.source, "audio-bridge");
|
|
assert.deepEqual(shared?.contributingSources, ["client", "audio-bridge"]);
|
|
assert.equal(described.description.split("shared cue").length - 1, 1);
|
|
});
|
|
|
|
test("keeps each successful caption attached to its source-frame timestamp", async (t) => {
|
|
for (const omittedCaption of ["failed", "empty"] as const) {
|
|
await t.test(omittedCaption, async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "audio before last frame", start: 4, end: 4.5, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 3, timeoutMs: 20_000 },
|
|
async (_frame, timestampSeconds) => {
|
|
if (timestampSeconds === 3) {
|
|
if (omittedCaption === "failed") throw new Error("caption unavailable");
|
|
return " ";
|
|
}
|
|
return timestampSeconds === 1 ? "first visual" : "last visual";
|
|
},
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 6,
|
|
frames: [
|
|
{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 1 },
|
|
{ dataUri: "data:image/jpeg;base64,AQ==", timestampSeconds: 3 },
|
|
{ dataUri: "data:image/jpeg;base64,Ag==", timestampSeconds: 5 },
|
|
],
|
|
}),
|
|
}
|
|
);
|
|
|
|
const firstVisual = described.description.indexOf("frame@t=00:01.000 first visual");
|
|
const audio = described.description.indexOf("audio before last frame");
|
|
const lastVisual = described.description.indexOf("frame@t=00:05.000 last visual");
|
|
assert.ok(firstVisual >= 0);
|
|
assert.ok(audio > firstVisual);
|
|
assert.ok(lastVisual > audio);
|
|
assert.equal(described.framesUsed, 2);
|
|
});
|
|
}
|
|
});
|
|
|
|
test("uses the full contact-sheet timestamp range for fusion ordering", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
contactSheet: true,
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "shorter audio", start: 1, end: 7, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 3, timeoutMs: 20_000 },
|
|
async () => "whole contact sheet",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 10,
|
|
frames: [
|
|
await jpegFrame("red", 1),
|
|
await jpegFrame("green", 5),
|
|
await jpegFrame("blue", 9),
|
|
],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.equal(described.contactSheetUsed, true);
|
|
const audio = described.description.indexOf("shorter audio");
|
|
const contactSheet = described.description.indexOf("whole contact sheet");
|
|
assert.ok(audio >= 0);
|
|
assert.ok(contactSheet > audio);
|
|
});
|
|
|
|
test("derives the contact-sheet interval from minimum and maximum timestamps", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
contactSheet: true,
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "late audio", start: 8, end: 8.5, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 3, timeoutMs: 20_000 },
|
|
async () => "unordered contact sheet",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 10,
|
|
frames: [
|
|
await jpegFrame("blue", 9),
|
|
await jpegFrame("red", 1),
|
|
await jpegFrame("green", 5),
|
|
],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.equal(described.contactSheetUsed, true);
|
|
const contactSheet = described.description.indexOf("unordered contact sheet");
|
|
const audio = described.description.indexOf("late audio");
|
|
assert.ok(contactSheet >= 0);
|
|
assert.ok(audio > contactSheet);
|
|
});
|
|
|
|
test("an invalid audioTranscript degrades to a partial fusion and keeps the visual description", async () => {
|
|
const described = await describeVideoPart(
|
|
{
|
|
container: "messages",
|
|
messageIndex: 0,
|
|
partIndex: 0,
|
|
ref: "data:video/mp4;base64,AA==",
|
|
shape: "data_uri_string",
|
|
audioTranscript: {
|
|
cues: [{ text: "late cue", start: 1, end: 99, source: "audio-bridge" }],
|
|
},
|
|
},
|
|
{ frameCount: 1, timeoutMs: 1000 },
|
|
async () => "visual cue",
|
|
{
|
|
extractFrames: async () => ({
|
|
durationSeconds: 5,
|
|
frames: [{ dataUri: "data:image/jpeg;base64,AA==", timestampSeconds: 2 }],
|
|
}),
|
|
}
|
|
);
|
|
|
|
assert.match(described.description, /visual cue/);
|
|
assert.equal(described.transcriptCues, undefined, "invalid audio must not add transcript cues");
|
|
assert.deepEqual(described.fusion, {
|
|
audioAvailable: false,
|
|
videoAvailable: true,
|
|
partial: true,
|
|
failures: { audio: "FAILED" },
|
|
});
|
|
});
|