mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-19 05:32:19 +03:00
533e5c6ec7
The fusion result's availability, partial and failure fields now reach DescribedVideo.fusion, the guardrail meta (audioFusionRuns/Partials/ FailureCodes), the result-cache metadata and bridge stats. Audio transcript validation moved inside the fusion's audio branch, so an invalid audioTranscript records failures.audio and keeps the visual description instead of failing the whole video.
717 lines
26 KiB
TypeScript
717 lines
26 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import { VideoBridgeGuardrail } from "../../../src/lib/guardrails/videoBridge.ts";
|
|
import { callVisionModel } from "../../../src/lib/guardrails/visionBridgeHelpers.ts";
|
|
import {
|
|
buildModalityBridgeHeader,
|
|
getBridgeStats,
|
|
} from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts";
|
|
import {
|
|
registerDefaultGuardrails,
|
|
resetGuardrailsForTests,
|
|
} from "../../../src/lib/guardrails/registry.ts";
|
|
|
|
const payload = () => ({
|
|
model: "example/text-only",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "input_video", video_url: "data:video/mp4;base64,QUJD" },
|
|
{ type: "text", text: "What happens?" },
|
|
],
|
|
},
|
|
],
|
|
});
|
|
|
|
function guardrail(options: { capability?: boolean | null; fail?: boolean } = {}) {
|
|
return new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
modalityBridgeCacheEnabled: false,
|
|
}),
|
|
getCapabilities: () => ({
|
|
supportsVideo: options.capability === undefined ? false : options.capability,
|
|
}),
|
|
describePart: async () => {
|
|
if (options.fail) throw new Error("private ffmpeg failure");
|
|
return {
|
|
description: "[Video description: frame@t=00:01.000 a person waves]",
|
|
durationSeconds: 2,
|
|
framesRequested: 1,
|
|
framesUsed: 1,
|
|
};
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
test("VideoBridgeGuardrail has priority 7 and native video targets bypass conversion", async () => {
|
|
let calls = 0;
|
|
const native = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({ modalityBridgeVideoEnabled: true }),
|
|
getCapabilities: () => ({ supportsVideo: true }),
|
|
describePart: async () => {
|
|
calls += 1;
|
|
throw new Error("should not run");
|
|
},
|
|
},
|
|
});
|
|
assert.equal(native.name, "video-bridge");
|
|
assert.equal(native.priority, 7);
|
|
assert.equal((await native.preCall(payload(), {})).modifiedPayload, undefined);
|
|
assert.equal(calls, 0);
|
|
});
|
|
|
|
test("converts Chat video to timestamped text and emits telemetry/header metadata", async () => {
|
|
const before = getBridgeStats().video;
|
|
const result = await guardrail().preCall(payload(), {});
|
|
const modified = result.modifiedPayload as ReturnType<typeof payload>;
|
|
assert.deepEqual(modified.messages[0].content[0], {
|
|
type: "text",
|
|
text: "[Video description: frame@t=00:01.000 a person waves]",
|
|
});
|
|
assert.equal(result.meta?.videosProcessed, 1);
|
|
assert.equal(result.meta?.framesUsed, 1);
|
|
assert.equal(result.meta?.videoModel, "openai/gpt-4o-mini");
|
|
assert.equal(
|
|
buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]),
|
|
"video->text;model=openai/gpt-4o-mini;parts=1"
|
|
);
|
|
assert.ok(getBridgeStats().video.bridged >= before.bridged + 1);
|
|
});
|
|
|
|
test("preserves scene-aware sampler metadata in guardrail meta and the transparency header", async () => {
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
modalityBridgeVideoSamplingPolicy: "scene_aware",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async () => ({
|
|
description: "[Video description: untrusted media-derived observation: a cut]",
|
|
durationSeconds: 12,
|
|
framesRequested: 4,
|
|
framesExtracted: 4,
|
|
framesUsed: 4,
|
|
dedupDropped: 1,
|
|
sampling: {
|
|
candidateCount: 3,
|
|
policyEffective: "scene_aware",
|
|
policyRequested: "scene_aware",
|
|
},
|
|
}),
|
|
},
|
|
});
|
|
|
|
const result = await bridge.preCall(payload(), {});
|
|
assert.equal(result.meta?.samplingPolicyRequested, "scene_aware");
|
|
assert.equal(result.meta?.samplingPolicyEffective, "scene_aware");
|
|
assert.equal(result.meta?.samplingCandidateCount, 3);
|
|
assert.equal(result.meta?.dedupDropped, 1);
|
|
assert.equal(
|
|
buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]),
|
|
"video->text;model=openai/gpt-4o-mini;parts=1;sampling=scene_aware;candidates=3"
|
|
);
|
|
});
|
|
|
|
test("reports only validated transcript provenance in guardrail metadata", async () => {
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async (part) => {
|
|
assert.deepEqual(part.transcript, {
|
|
cues: [{ text: "spoken words", start: 1, end: 2, source: "client" }],
|
|
});
|
|
return {
|
|
description: "[Video description: caption; transcript[source=client] spoken words]",
|
|
durationSeconds: 2,
|
|
framesRequested: 1,
|
|
framesUsed: 1,
|
|
transcriptCues: [
|
|
{
|
|
confidence: 1,
|
|
endSeconds: 2,
|
|
source: "client",
|
|
startSeconds: 1,
|
|
text: "spoken words",
|
|
},
|
|
],
|
|
};
|
|
},
|
|
},
|
|
});
|
|
const result = await bridge.preCall(
|
|
{
|
|
...payload(),
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{
|
|
type: "input_video",
|
|
video_url: "data:video/mp4;base64,QUJD",
|
|
transcript: { cues: [{ text: "spoken words", start: 1, end: 2, source: "client" }] },
|
|
},
|
|
],
|
|
},
|
|
],
|
|
},
|
|
{}
|
|
);
|
|
assert.equal(result.meta?.transcriptCuesApplied, 1);
|
|
});
|
|
|
|
test("converts Responses input using input_text while preserving sibling order", async () => {
|
|
const body = {
|
|
model: "example/text-only",
|
|
input: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "input_text", text: "before" },
|
|
{ type: "video_url", video_url: { url: "https://example.test/video.mp4" } },
|
|
{ type: "input_text", text: "after" },
|
|
],
|
|
},
|
|
],
|
|
};
|
|
const result = await guardrail().preCall(body, {});
|
|
assert.deepEqual((result.modifiedPayload as typeof body).input[0].content, [
|
|
{ type: "input_text", text: "before" },
|
|
{ type: "input_text", text: "[Video description: frame@t=00:01.000 a person waves]" },
|
|
{ type: "input_text", text: "after" },
|
|
]);
|
|
});
|
|
|
|
test("preserves unknown-capability video on total failure but stubs proven text-only input", async () => {
|
|
const original = payload();
|
|
const snapshot = structuredClone(original);
|
|
const unknown = await guardrail({ capability: null, fail: true }).preCall(original, {});
|
|
assert.equal(unknown.modifiedPayload, undefined);
|
|
assert.deepEqual(original, snapshot);
|
|
|
|
const knownFalse = await guardrail({ capability: false, fail: true }).preCall(payload(), {});
|
|
const modified = knownFalse.modifiedPayload as ReturnType<typeof payload>;
|
|
assert.deepEqual(modified.messages[0].content[0], {
|
|
type: "text",
|
|
text: "[Video 1]: (unavailable — video could not be described)",
|
|
});
|
|
assert.equal(String(knownFalse.meta?.failures).includes("private"), false);
|
|
});
|
|
|
|
test("reports cache hits per converted video without carrying a previous hit forward", async () => {
|
|
const body = payload();
|
|
body.messages[0].content.splice(1, 0, {
|
|
type: "video_url",
|
|
video_url: "data:video/mp4;base64,REVG",
|
|
});
|
|
const before = getBridgeStats().video;
|
|
let described = 0;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoMaxVideos: 2,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async () => {
|
|
described += 1;
|
|
return {
|
|
cacheHits: described === 1 ? 1 : 0,
|
|
description: `[Video description: frame@t=00:0${described}.000 frame ${described}]`,
|
|
durationSeconds: 2,
|
|
framesRequested: 1,
|
|
framesUsed: 1,
|
|
};
|
|
},
|
|
},
|
|
});
|
|
|
|
const result = await bridge.preCall(body, {});
|
|
const after = getBridgeStats().video;
|
|
assert.equal(result.meta?.cacheHits, 1);
|
|
assert.equal(after.bridged - before.bridged, 2);
|
|
assert.equal(after.cacheHits - before.cacheHits, 1);
|
|
});
|
|
|
|
test("default registry includes Video Bridge after Vision and Audio", () => {
|
|
resetGuardrailsForTests({ registerDefaults: false });
|
|
const names = registerDefaultGuardrails()
|
|
.list()
|
|
.filter((entry) => entry.name.endsWith("-bridge"))
|
|
.map((entry) => `${entry.priority}:${entry.name}`);
|
|
assert.deepEqual(names, ["5:vision-bridge", "6:audio-bridge", "7:video-bridge"]);
|
|
resetGuardrailsForTests();
|
|
});
|
|
|
|
test("maxVideos describes only the first video and removes every excess raw video for text-only targets", async () => {
|
|
const body = payload();
|
|
body.messages[0].content.splice(1, 0, {
|
|
type: "video_url",
|
|
video_url: "data:video/mp4;base64,REVG",
|
|
});
|
|
let calls = 0;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoMaxVideos: 1,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async () => {
|
|
calls += 1;
|
|
return {
|
|
description: "[Video description: untrusted media-derived observation: first]",
|
|
durationSeconds: 1,
|
|
framesRequested: 1,
|
|
framesExtracted: 1,
|
|
framesUsed: 1,
|
|
};
|
|
},
|
|
},
|
|
});
|
|
const result = await bridge.preCall(body, {});
|
|
const content = (result.modifiedPayload as typeof body).messages[0].content;
|
|
assert.equal(calls, 1);
|
|
assert.equal(
|
|
content.some((part) => "video_url" in part),
|
|
false
|
|
);
|
|
assert.match(String((content[1] as { text?: string }).text), /not processed.*limit/i);
|
|
assert.equal(result.meta?.attempts, 1);
|
|
assert.equal(result.meta?.videosProcessed, 1);
|
|
assert.equal(result.meta?.videosReplaced, 2);
|
|
});
|
|
|
|
test("maxVideos preserves excess raw video only when target video support is unknown", async () => {
|
|
const body = payload();
|
|
body.messages[0].content.splice(1, 0, {
|
|
type: "video_url",
|
|
video_url: "data:video/mp4;base64,REVG",
|
|
});
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoMaxVideos: 1,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: null }),
|
|
describePart: async () => ({
|
|
description: "[Video description: untrusted media-derived observation: first]",
|
|
durationSeconds: 1,
|
|
framesRequested: 1,
|
|
framesExtracted: 1,
|
|
framesUsed: 1,
|
|
}),
|
|
},
|
|
});
|
|
const result = await bridge.preCall(body, {});
|
|
const content = (result.modifiedPayload as typeof body).messages[0].content;
|
|
assert.equal("video_url" in content[1], true);
|
|
});
|
|
|
|
test("empty Video and Vision model settings use the Vision auto-router and report the effective model", async () => {
|
|
let selectedFixedModel: string | undefined;
|
|
let calledModel = "";
|
|
let routedThroughOmniRoute = false;
|
|
let injectedFetch = false;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "",
|
|
modalityBridgeVisionModel: "",
|
|
modalityBridgeCacheEnabled: false,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async (fixedModel) => {
|
|
selectedFixedModel = fixedModel;
|
|
return "google/gemini-2.5-flash";
|
|
},
|
|
extractFrames: async () => ({
|
|
durationSeconds: 1,
|
|
frames: [{ timestampSeconds: 0.5, dataUri: "data:image/jpeg;base64,AUTO9760" }],
|
|
}),
|
|
callVisionModel: async (_image, config) => {
|
|
calledModel = config.model;
|
|
routedThroughOmniRoute = config.routeThroughOmniRoute === true;
|
|
injectedFetch = typeof config.fetchImpl === "function";
|
|
return "a safe observation";
|
|
},
|
|
},
|
|
});
|
|
const result = await bridge.preCall(payload(), {});
|
|
assert.equal(selectedFixedModel, undefined);
|
|
assert.equal(calledModel, "google/gemini-2.5-flash");
|
|
assert.equal(routedThroughOmniRoute, true);
|
|
assert.equal(injectedFetch, true);
|
|
assert.equal(result.meta?.videoModel, "google/gemini-2.5-flash");
|
|
assert.ok(result.modifiedPayload);
|
|
});
|
|
|
|
test("client abort between videos stops processing and never stubs or falls back", async () => {
|
|
const body = payload();
|
|
body.messages[0].content.splice(1, 0, {
|
|
type: "video_url",
|
|
video_url: "data:video/mp4;base64,REVG",
|
|
});
|
|
const controller = new AbortController();
|
|
let calls = 0;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoMaxVideos: 2,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async () => {
|
|
calls += 1;
|
|
controller.abort();
|
|
return {
|
|
description: "[Video description: untrusted media-derived observation: first]",
|
|
durationSeconds: 1,
|
|
framesRequested: 1,
|
|
framesExtracted: 1,
|
|
framesUsed: 1,
|
|
};
|
|
},
|
|
},
|
|
});
|
|
await assert.rejects(() => bridge.preCall(body, { signal: controller.signal }), /aborted/);
|
|
assert.equal(calls, 1);
|
|
assert.equal(
|
|
body.messages[0].content.some(
|
|
(part) => "text" in part && /unavailable/.test(String(part.text))
|
|
),
|
|
false
|
|
);
|
|
});
|
|
|
|
test("real Video Bridge cache hit avoids a second model call and records the hit", async () => {
|
|
let modelCalls = 0;
|
|
const beforeStats = getBridgeStats().video;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
modalityBridgeVisionPrompt: "cache integration 9760",
|
|
modalityBridgeCacheEnabled: true,
|
|
modalityBridgeCacheTtlMinutes: 60,
|
|
modalityBridgeCacheMaxEntries: 50,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async () => "openai/gpt-4o-mini",
|
|
extractFrames: async () => ({
|
|
durationSeconds: 1,
|
|
frames: [{ timestampSeconds: 0.5, dataUri: "data:image/jpeg;base64,CACHE9760" }],
|
|
}),
|
|
callVisionModel: async () => {
|
|
modelCalls += 1;
|
|
return "cached observation";
|
|
},
|
|
},
|
|
});
|
|
const first = await bridge.preCall(payload(), {});
|
|
const second = await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 1);
|
|
assert.equal(first.meta?.cacheHits, 0);
|
|
assert.equal(second.meta?.cacheHits, 0);
|
|
const afterStats = getBridgeStats().video;
|
|
const firstTextPart = (first.modifiedPayload as ReturnType<typeof payload>).messages[0]
|
|
.content[0];
|
|
assert.equal(afterStats.resultCacheHits - beforeStats.resultCacheHits, 1);
|
|
assert.equal(
|
|
afterStats.resultCacheBytes - beforeStats.resultCacheBytes,
|
|
Buffer.byteLength(String((firstTextPart as { text: string }).text), "utf8")
|
|
);
|
|
assert.equal(afterStats.resultCacheLatencyMs - beforeStats.resultCacheLatencyMs >= 0, true);
|
|
});
|
|
|
|
test("real primary failure reports and caches the successful fallback model identity", async () => {
|
|
const primary = "openai/gpt-4o-mini";
|
|
const fallback = "anthropic/claude-fable-5";
|
|
const beforeStats = getBridgeStats().video;
|
|
const attemptedModels: string[] = [];
|
|
const fetchImpl: typeof fetch = async (_input, init) => {
|
|
const body = JSON.parse(String(init?.body)) as { model: string };
|
|
attemptedModels.push(body.model);
|
|
if (body.model === primary) {
|
|
return new Response("primary unavailable", { status: 503 });
|
|
}
|
|
return Response.json({ choices: [{ message: { content: "fallback observation" } }] });
|
|
};
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: primary,
|
|
modalityBridgeVisionPrompt: "fallback identity integration 9760",
|
|
modalityBridgeCacheEnabled: true,
|
|
modalityBridgeCacheTtlMinutes: 62,
|
|
modalityBridgeCacheMaxEntries: 52,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async () => primary,
|
|
extractFrames: async () => ({
|
|
durationSeconds: 1,
|
|
frames: [{ timestampSeconds: 0.5, dataUri: "data:image/jpeg;base64,FALLBACK9760" }],
|
|
}),
|
|
callVisionModel: (image, config) =>
|
|
callVisionModel(
|
|
image,
|
|
{ ...config, fetchImpl },
|
|
"sk-fallback-test",
|
|
{ maxFallbackAttempts: 2 },
|
|
{
|
|
hasUsableCredentials: async (model) => model === primary || model === fallback,
|
|
}
|
|
),
|
|
},
|
|
});
|
|
|
|
const first = await bridge.preCall(payload(), {});
|
|
const second = await bridge.preCall(payload(), {});
|
|
|
|
assert.deepEqual(attemptedModels, [primary, fallback]);
|
|
assert.equal(first.meta?.videoModel, fallback, "meta must name the successful fallback");
|
|
assert.equal(second.meta?.videoModel, fallback, "cache hit must retain the producer identity");
|
|
assert.equal(second.meta?.cacheHits, 0);
|
|
const deltaResultCacheHits = getBridgeStats().video.resultCacheHits - beforeStats.resultCacheHits;
|
|
assert.equal(deltaResultCacheHits >= 1, true);
|
|
assert.equal(
|
|
buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: second.meta }]),
|
|
`video->text;model=${fallback};parts=1`
|
|
);
|
|
});
|
|
|
|
test("cache keys miss on prompt and effective model changes; failures are not cached", async () => {
|
|
let prompt = "prompt-a-9760";
|
|
let selectedModel = "openai/gpt-4o-mini";
|
|
let modelCalls = 0;
|
|
let fail = true;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: selectedModel,
|
|
modalityBridgeVisionPrompt: prompt,
|
|
modalityBridgeCacheEnabled: true,
|
|
modalityBridgeCacheTtlMinutes: 61,
|
|
modalityBridgeCacheMaxEntries: 51,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async () => selectedModel,
|
|
extractFrames: async () => ({
|
|
durationSeconds: 1,
|
|
frames: [{ timestampSeconds: 0.25, dataUri: "data:image/jpeg;base64,MISS9760" }],
|
|
}),
|
|
callVisionModel: async () => {
|
|
modelCalls += 1;
|
|
if (fail) throw new Error("model failure");
|
|
return "observation";
|
|
},
|
|
},
|
|
});
|
|
|
|
await bridge.preCall(payload(), {});
|
|
fail = false;
|
|
await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 2, "failed captions must not be cached");
|
|
const hitWithSameSettings = await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 2, "result cache must reuse after a success");
|
|
assert.equal(hitWithSameSettings.meta?.cacheHits, 0);
|
|
await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 2, "frame extraction options did not change on this path");
|
|
prompt = "prompt-b-9760";
|
|
await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 3, "prompt changes must invalidate result cache");
|
|
selectedModel = "google/gemini-2.5-flash";
|
|
await bridge.preCall(payload(), {});
|
|
assert.equal(modelCalls, 4, "effective model changes must invalidate result cache");
|
|
});
|
|
|
|
test("FFmpeg ENOENT is sanitized and counts only as a failed attempt, never a bridged success", async () => {
|
|
const before = getBridgeStats().video;
|
|
const warnings: Array<{ message: string; meta?: Record<string, unknown> }> = [];
|
|
const error = Object.assign(new Error("spawn /private/operator/ffmpeg ENOENT"), {
|
|
code: "ENOENT",
|
|
});
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
describePart: async () => {
|
|
throw error;
|
|
},
|
|
},
|
|
});
|
|
const result = await bridge.preCall(payload(), {
|
|
log: { warn: (_tag, message, meta) => warnings.push({ message, meta }) },
|
|
});
|
|
const after = getBridgeStats().video;
|
|
assert.equal(after.attempts - before.attempts, 1);
|
|
assert.equal(after.successes - before.successes, 0);
|
|
assert.equal(after.bridged - before.bridged, 0);
|
|
assert.equal(after.failures - before.failures, 1);
|
|
assert.equal(result.meta?.videosProcessed, 0);
|
|
assert.ok(result.modifiedPayload, "proven text-only input still needs a safe stub");
|
|
assert.equal(JSON.stringify(warnings).includes("/private/operator"), false);
|
|
assert.equal(buildModalityBridgeHeader([{ guardrail: "video-bridge", meta: result.meta }]), null);
|
|
});
|
|
|
|
function cachedBridgeWithCounter(counter: { calls: number }, cacheSalt: string) {
|
|
return new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
modalityBridgeVisionPrompt: `cache dimensions ${cacheSalt}`,
|
|
modalityBridgeCacheEnabled: true,
|
|
modalityBridgeCacheTtlMinutes: 60,
|
|
modalityBridgeCacheMaxEntries: 50,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async () => "openai/gpt-4o-mini",
|
|
describePart: async () => {
|
|
counter.calls += 1;
|
|
return {
|
|
description: `[Video description: observation ${counter.calls}]`,
|
|
durationSeconds: 4,
|
|
framesRequested: 1,
|
|
framesUsed: 1,
|
|
};
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
test("result cache misses when audioTranscript is added, changes, and hits when equivalent", async () => {
|
|
const counter = { calls: 0 };
|
|
const bridge = cachedBridgeWithCounter(counter, "audio-transcript");
|
|
const withAudio = (audioTranscript?: unknown) => ({
|
|
model: "example/text-only",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{
|
|
type: "input_video",
|
|
video_url: "data:video/mp4;base64,QUJD",
|
|
...(audioTranscript === undefined ? {} : { audioTranscript }),
|
|
},
|
|
],
|
|
},
|
|
],
|
|
});
|
|
const cuesA = { cues: [{ text: "hello", start: 0, end: 1, source: "client" }] };
|
|
const cuesB = { cues: [{ text: "different", start: 1, end: 2, source: "client" }] };
|
|
|
|
await bridge.preCall(withAudio(), {});
|
|
assert.equal(counter.calls, 1);
|
|
await bridge.preCall(withAudio(cuesA), {});
|
|
assert.equal(counter.calls, 2, "adding an audioTranscript must invalidate the result cache");
|
|
await bridge.preCall(withAudio(structuredClone(cuesA)), {});
|
|
assert.equal(counter.calls, 2, "an equivalent audioTranscript must reuse the cached result");
|
|
await bridge.preCall(withAudio(cuesB), {});
|
|
assert.equal(counter.calls, 3, "a different audioTranscript must invalidate the result cache");
|
|
await bridge.preCall(withAudio(), {});
|
|
assert.equal(counter.calls, 3, "removing the audioTranscript must reuse the first cached result");
|
|
});
|
|
|
|
test("result cache misses when the focus window is added or changed", async () => {
|
|
const counter = { calls: 0 };
|
|
const bridge = cachedBridgeWithCounter(counter, "focus-window");
|
|
const withFocus = (bounds?: { start?: number; end?: number }) => ({
|
|
model: "example/text-only",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{
|
|
type: "input_video",
|
|
video_url: "data:video/mp4;base64,QUJD",
|
|
...(bounds ?? {}),
|
|
},
|
|
],
|
|
},
|
|
],
|
|
});
|
|
|
|
await bridge.preCall(withFocus(), {});
|
|
assert.equal(counter.calls, 1);
|
|
await bridge.preCall(withFocus({ start: 0, end: 1 }), {});
|
|
assert.equal(counter.calls, 2, "adding a focus window must invalidate the result cache");
|
|
await bridge.preCall(withFocus({ start: 0, end: 1 }), {});
|
|
assert.equal(counter.calls, 2, "an identical focus window must reuse the cached result");
|
|
await bridge.preCall(withFocus({ start: 1, end: 2 }), {});
|
|
assert.equal(counter.calls, 3, "a different focus window must invalidate the result cache");
|
|
});
|
|
|
|
test("audio/video fusion telemetry reaches guardrail meta, bridge stats, and cache hits", async () => {
|
|
const before = getBridgeStats().video;
|
|
let describeCalls = 0;
|
|
const bridge = new VideoBridgeGuardrail({
|
|
deps: {
|
|
getSettings: async () => ({
|
|
modalityBridgeVideoEnabled: true,
|
|
modalityBridgeVideoModel: "openai/gpt-4o-mini",
|
|
modalityBridgeVisionPrompt: "fusion telemetry",
|
|
modalityBridgeCacheEnabled: true,
|
|
modalityBridgeCacheTtlMinutes: 60,
|
|
modalityBridgeCacheMaxEntries: 50,
|
|
}),
|
|
getCapabilities: () => ({ supportsVideo: false }),
|
|
selectVisionModel: async () => "openai/gpt-4o-mini",
|
|
describePart: async () => {
|
|
describeCalls += 1;
|
|
return {
|
|
description: "[Video description: partial fusion observation]",
|
|
durationSeconds: 4,
|
|
framesRequested: 1,
|
|
framesUsed: 1,
|
|
fusion: {
|
|
audioAvailable: false,
|
|
videoAvailable: true,
|
|
partial: true,
|
|
failures: { audio: "FAILED" as const },
|
|
},
|
|
};
|
|
},
|
|
},
|
|
});
|
|
|
|
const first = await bridge.preCall(payload(), {});
|
|
assert.equal(first.meta?.audioFusionRuns, 1);
|
|
assert.equal(first.meta?.audioFusionPartials, 1);
|
|
assert.deepEqual(first.meta?.audioFusionFailureCodes, ["audio:FAILED"]);
|
|
|
|
const second = await bridge.preCall(payload(), {});
|
|
assert.equal(describeCalls, 1, "the second call must be a result cache hit");
|
|
assert.equal(second.meta?.audioFusionRuns, 1, "cache hits must restore fusion telemetry");
|
|
assert.equal(second.meta?.audioFusionPartials, 1);
|
|
assert.deepEqual(second.meta?.audioFusionFailureCodes, ["audio:FAILED"]);
|
|
|
|
const after = getBridgeStats().video;
|
|
assert.equal(after.fusionRuns - before.fusionRuns, 2);
|
|
assert.equal(after.fusionPartials - before.fusionPartials, 2);
|
|
});
|