fix(video-bridge): harden bounded extraction contracts

This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-08-15 04:26:31 -03:00
committed by Xiangzhe
parent e30f42ccb8
commit 67fa0d0286
18 changed files with 531 additions and 102 deletions

View File

@@ -175,18 +175,18 @@ function inspectVideoShapes(
}
}
const source = obj.source as Record<string, unknown> | undefined;
if (
(type === "video_source" ||
(typeof mediaType === "string" && mediaType.toLowerCase().startsWith("video/"))) &&
source
) {
if (typeof source.data === "string") {
const mime = typeof mediaType === "string" ? mediaType : "video/mp4";
pushPart(ctx, "video", `data:${mime};base64,${source.data}`, "video_source", depth);
if (source) {
const videoMediaType =
typeof mediaType === "string" && mediaType.toLowerCase().startsWith("video/");
// Base64 must carry an explicit video MIME. This prevents a type:video wrapper
// from relabelling arbitrary base64 content as MP4.
if (videoMediaType && typeof source.data === "string") {
pushPart(ctx, "video", `data:${mediaType};base64,${source.data}`, "video_source", depth);
return true;
}
const ref = urlFrom(source.url);
if (ref) {
const explicitAnthropicUrl = type === "video" && source.type === "url";
if (ref && (explicitAnthropicUrl || type === "video_source" || videoMediaType)) {
pushPart(ctx, "video", ref, "video_source", depth);
return true;
}

View File

@@ -7,6 +7,8 @@ import { Card, ModelSelectField, Toggle } from "@/shared/components";
import type { ApiModel } from "@/shared/components/ModelSelectField";
import {
MODALITY_BRIDGE_DEFAULTS,
VIDEO_BRIDGE_TIMEOUT_MAX_MS,
VIDEO_BRIDGE_TIMEOUT_MIN_MS,
resolveVideoBridgeRuntimeSettings,
} from "@/shared/constants/modalityBridgeDefaults";
@@ -236,16 +238,16 @@ export default function ModalityBridgeVideoTab() {
<NumberField
testId="modality-bridge-video-timeout"
label={t("modalityBridgeTimeoutMs")}
min={1000}
max={300000}
min={VIDEO_BRIDGE_TIMEOUT_MIN_MS}
max={VIDEO_BRIDGE_TIMEOUT_MAX_MS}
value={settings.modalityBridgeVideoTimeout}
onChange={(value) => setLocal({ modalityBridgeVideoTimeout: value })}
onBlur={(raw) =>
commitNumber(
"modalityBridgeVideoTimeout",
raw,
1000,
300000,
VIDEO_BRIDGE_TIMEOUT_MIN_MS,
VIDEO_BRIDGE_TIMEOUT_MAX_MS,
MODALITY_BRIDGE_DEFAULTS.videoTimeoutMs
)
}

View File

@@ -3,24 +3,31 @@ import {
VIDEO_BRIDGE_BROKER_PATH,
isVideoBridgeBrokerInternalRequest,
} from "@/lib/guardrails/videoBridgeBrokerAuth";
import { createVideoExtractionQueue } from "@/lib/guardrails/videoBridgeBrokerQueue";
import {
createVideoExtractionQueue,
type VideoExtractionQueue,
VideoExtractionQueueError,
} from "@/lib/guardrails/videoBridgeBrokerQueue";
import { extractVideoFramesFromBytes } from "@/lib/guardrails/videoBridgeRuntime";
import { resolveModelSyncInternalBaseUrl } from "@/shared/services/modelSyncScheduler";
import { VIDEO_BRIDGE_TIMEOUT_MAX_MS } from "@/shared/constants/modalityBridgeDefaults";
export const dynamic = "force-dynamic";
export const revalidate = 0;
const MAX_INPUT_BYTES = 50 * 1024 * 1024;
const MAX_DURATION_SECONDS = 600;
const BROKER_TIMEOUT_MS = 120_000;
export const BROKER_TIMEOUT_MS = VIDEO_BRIDGE_TIMEOUT_MAX_MS;
const extractionQueue = createVideoExtractionQueue({
concurrency: 1,
maxPending: 4,
maxQueuedBytes: 100 * 1024 * 1024,
});
function invalid(message: string, status = 400): Response {
return createErrorResponse({ status, message, type: "invalid_request" });
function invalid(message: string, status = 400, headers?: Record<string, string>): Response {
const response = createErrorResponse({ status, message, type: "invalid_request" });
for (const [name, value] of Object.entries(headers ?? {})) response.headers.set(name, value);
return response;
}
function parseFrameCount(url: URL): number | null {
@@ -60,7 +67,16 @@ export async function readBoundedVideoBrokerBody(
);
}
export async function POST(request: Request): Promise<Response> {
interface VideoExtractionBrokerRouteDependencies {
deadlineSignal?: AbortSignal;
extractFrames?: typeof extractVideoFramesFromBytes;
queue?: VideoExtractionQueue;
}
export async function handleVideoExtractionBrokerRequest(
request: Request,
dependencies: VideoExtractionBrokerRouteDependencies = {}
): Promise<Response> {
const url = new URL(request.url);
if (request.method !== "POST" || url.pathname !== expectedBrokerPath()) {
return invalid("Invalid Video Bridge broker request", 404);
@@ -100,13 +116,15 @@ export async function POST(request: Request): Promise<Response> {
return invalid("Video Bridge input exceeds the byte limit", 413);
}
const deadline = AbortSignal.timeout(BROKER_TIMEOUT_MS);
const deadline = dependencies.deadlineSignal ?? AbortSignal.timeout(BROKER_TIMEOUT_MS);
const signal = AbortSignal.any([request.signal, deadline]);
const queue = dependencies.queue ?? extractionQueue;
const extractFrames = dependencies.extractFrames ?? extractVideoFramesFromBytes;
try {
const result = await extractionQueue.run(
const result = await queue.run(
bytes.byteLength,
() =>
extractVideoFramesFromBytes(bytes, {
extractFrames(bytes, {
frameCount,
maxDurationSeconds: MAX_DURATION_SECONDS,
signal,
@@ -118,14 +136,36 @@ export async function POST(request: Request): Promise<Response> {
} catch (error) {
const unavailable =
error && typeof error === "object" && "code" in error && error.code === "ENOENT";
const queueCapacity =
error instanceof VideoExtractionQueueError && error.code === "QUEUE_CAPACITY";
const clientAborted = request.signal.aborted;
const deadlineExceeded = !clientAborted && deadline.aborted;
console.warn("[VideoBridgeBroker] extraction failed", {
aborted: signal.aborted,
code: unavailable ? "RUNTIME_UNAVAILABLE" : "EXTRACTION_FAILED",
aborted: clientAborted,
code: clientAborted
? "CLIENT_ABORTED"
: queueCapacity
? "QUEUE_CAPACITY"
: deadlineExceeded
? "DEADLINE_EXCEEDED"
: unavailable
? "RUNTIME_UNAVAILABLE"
: "EXTRACTION_FAILED",
frameCount,
inputBytes: bytes.byteLength,
});
if (signal.aborted) return invalid("Video extraction was aborted", 499);
if (clientAborted) return invalid("Video extraction was aborted", 499);
if (deadlineExceeded) return invalid("Video extraction deadline exceeded", 504);
if (queueCapacity) {
return invalid("Video extraction capacity is temporarily unavailable", 503, {
"Retry-After": "1",
});
}
if (unavailable) return invalid("Video extraction runtime is unavailable", 503);
return invalid("Video extraction failed", 422);
}
}
export async function POST(request: Request): Promise<Response> {
return handleVideoExtractionBrokerRequest(request);
}

View File

@@ -1,15 +1,34 @@
import { NextResponse } from "next/server";
import { createErrorResponse } from "@/lib/api/errorResponse";
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
import { probeVideoRuntime } from "@/lib/guardrails/videoBridgeRuntime";
import { AUTHZ_HEADER_PEER_LOCALITY } from "@/server/authz/headers";
export const dynamic = "force-dynamic";
export const revalidate = 0;
export async function GET(request: Request) {
interface VideoRuntimeStatusDependencies {
probe?: typeof probeVideoRuntime;
}
export async function handleVideoRuntimeStatus(
request: Request,
dependencies: VideoRuntimeStatusDependencies = {}
): Promise<Response> {
if (request.headers.get(AUTHZ_HEADER_PEER_LOCALITY) !== "loopback") {
return createErrorResponse({
status: 403,
message: "This endpoint is available only to trusted loopback requests",
});
}
const authError = await requireManagementAuth(request);
if (authError) return authError;
const status = await probeVideoRuntime();
const status = await (dependencies.probe ?? probeVideoRuntime)();
return NextResponse.json(status, { headers: { "Cache-Control": "no-store" } });
}
export async function GET(request: Request): Promise<Response> {
return handleVideoRuntimeStatus(request);
}

View File

@@ -14,6 +14,7 @@ export interface GuardrailContext {
method?: string | null;
model?: string | null;
provider?: string | null;
/** Caller lifecycle signal; media bridges treat request abort as a deliberate fail-open exception. */
signal?: AbortSignal;
sourceFormat?: string | null;
stream?: boolean;

View File

@@ -15,8 +15,20 @@ export interface VideoExtractionQueue {
): Promise<T>;
}
function abortError(): Error {
return new Error("Video extraction request aborted");
export type VideoExtractionQueueErrorCode = "CLIENT_ABORTED" | "QUEUE_CAPACITY";
export class VideoExtractionQueueError extends Error {
constructor(
readonly code: VideoExtractionQueueErrorCode,
message: string
) {
super(message);
this.name = "VideoExtractionQueueError";
}
}
function abortError(): VideoExtractionQueueError {
return new VideoExtractionQueueError("CLIENT_ABORTED", "Video extraction request aborted");
}
export function createVideoExtractionQueue(options: {
@@ -59,7 +71,12 @@ export function createVideoExtractionQueue(options: {
}
if (signal?.aborted) return Promise.reject(abortError());
if (pending.length >= options.maxPending || queuedBytes + byteSize > options.maxQueuedBytes) {
return Promise.reject(new Error("Video extraction queue capacity exceeded"));
return Promise.reject(
new VideoExtractionQueueError(
"QUEUE_CAPACITY",
"Video extraction queue capacity exceeded"
)
);
}
return new Promise<T>((resolve, reject) => {
const item: QueueItem<T> = { byteSize, execute, reject, resolve, signal };

View File

@@ -9,6 +9,10 @@ import {
} from "./videoBridgeBrokerClient";
export const VIDEO_BRIDGE_MAX_BYTES = 50 * 1024 * 1024;
// Inline base64 shares the public 50 MiB JSON admission budget with model,
// messages and framing. Reserve 14 MiB for that envelope; remote downloads and
// the loopback broker retain the independent 50 MiB binary limit.
export const VIDEO_BRIDGE_INLINE_MAX_BYTES = 36 * 1024 * 1024;
export const VIDEO_BRIDGE_MAX_DURATION_SECONDS = 600;
type VideoContainer = "messages" | "input";
@@ -107,9 +111,40 @@ export interface DescribedVideo {
modelUsed?: string;
}
function decodeVideoDataUri(ref: string): Buffer | null {
function normalizeBase64(base64: string): string {
const normalized = base64.replace(/\s/g, "");
if (
normalized.length === 0 ||
normalized.length % 4 !== 0 ||
!/^[A-Za-z0-9+/]*={0,2}$/.test(normalized)
) {
throw new Error("Video data URI contains invalid base64");
}
return normalized;
}
function estimateNormalizedBase64Bytes(normalized: string): number {
const padding = normalized.endsWith("==") ? 2 : normalized.endsWith("=") ? 1 : 0;
return (normalized.length / 4) * 3 - padding;
}
export function estimateDecodedBase64Bytes(base64: string): number {
return estimateNormalizedBase64Bytes(normalizeBase64(base64));
}
export function decodeVideoDataUri(
ref: string,
maxBytes = VIDEO_BRIDGE_INLINE_MAX_BYTES,
decode: (base64: string) => Buffer = (base64) => Buffer.from(base64, "base64")
): Buffer | null {
const match = /^data:video\/[A-Za-z0-9.+-]+;base64,([A-Za-z0-9+/=\s]+)$/i.exec(ref);
return match ? Buffer.from(match[1].replace(/\s/g, ""), "base64") : null;
if (!match) return null;
const normalized = normalizeBase64(match[1]);
const estimatedBytes = estimateNormalizedBase64Bytes(normalized);
if (estimatedBytes > maxBytes) {
throw new Error("Inline video exceeds the maximum size");
}
return decode(normalized);
}
async function loadVideoBytes(
@@ -120,7 +155,7 @@ async function loadVideoBytes(
deps: DescribeVideoDependencies
): Promise<Buffer> {
if (signal.aborted) throw new Error("Video Bridge processing timed out or was aborted");
const dataBytes = decodeVideoDataUri(part.ref);
const dataBytes = decodeVideoDataUri(part.ref, Math.min(maxBytes, VIDEO_BRIDGE_INLINE_MAX_BYTES));
let bytes: Buffer;
if (dataBytes) {
bytes = dataBytes;

View File

@@ -33,6 +33,7 @@ export interface VideoProbeMetadata {
durationSeconds: number;
formatName: string;
height: number;
streamIndex: number;
width: number;
}
@@ -79,26 +80,6 @@ function assertLocalPath(filePath: string): void {
}
}
function assertNoEmbeddedMediaReferences(bytes: Uint8Array): void {
const buffer = Buffer.from(bytes);
const blockedMarkers = [
"http:",
"https:",
"ftp:",
"file:",
"concat:",
"tcp:",
"udp:",
"../",
"..\\",
"#EXTM3U",
"ffconcat version",
];
if (blockedMarkers.some((marker) => buffer.includes(Buffer.from(marker)))) {
throw new Error("Video content contains an external or traversing media reference");
}
}
function parseVersion(output: string): string | null {
const version = /\bversion\s+([^\s]+)/i.exec(output)?.[1];
return version ? version.slice(0, 80).replace(/[^A-Za-z0-9._+-]/g, "_") : null;
@@ -204,7 +185,7 @@ export async function probeLocalVideo(
"-threads",
"1",
"-show_entries",
"format=duration,format_name:stream=codec_type,width,height",
"format=duration,format_name:stream=index,codec_type,width,height",
"-of",
"json",
inputPath,
@@ -215,16 +196,45 @@ export async function probeLocalVideo(
let formatName = "";
let width = Number.NaN;
let height = Number.NaN;
let streamIndex = Number.NaN;
let allVideoStreamsSafe = false;
try {
const parsed = JSON.parse(result.stdout) as {
format?: { duration?: unknown; format_name?: unknown };
streams?: Array<{ codec_type?: unknown; width?: unknown; height?: unknown }>;
streams?: Array<{
codec_type?: unknown;
height?: unknown;
index?: unknown;
width?: unknown;
}>;
};
durationSeconds = Number(parsed.format?.duration);
formatName = typeof parsed.format?.format_name === "string" ? parsed.format.format_name : "";
const videoStream = parsed.streams?.find((stream) => stream.codec_type === "video");
width = Number(videoStream?.width);
height = Number(videoStream?.height);
const videoStreams = parsed.streams?.filter((stream) => stream.codec_type === "video") ?? [];
allVideoStreamsSafe =
videoStreams.length > 0 &&
!videoStreams.some((stream) => {
const streamWidth = Number(stream.width);
const streamHeight = Number(stream.height);
const candidateIndex = Number(stream.index);
return (
!Number.isInteger(candidateIndex) ||
candidateIndex < 0 ||
!Number.isInteger(streamWidth) ||
!Number.isInteger(streamHeight) ||
streamWidth < 1 ||
streamHeight < 1 ||
streamWidth > VIDEO_MAX_DIMENSION ||
streamHeight > VIDEO_MAX_DIMENSION ||
streamWidth * streamHeight > VIDEO_MAX_PIXELS
);
});
const selectedStream = [...videoStreams].sort(
(left, right) => Number(left.index) - Number(right.index)
)[0];
streamIndex = Number(selectedStream.index);
width = Number(selectedStream.width);
height = Number(selectedStream.height);
} catch {
// The stable error below deliberately excludes raw ffprobe output.
}
@@ -242,18 +252,10 @@ export async function probeLocalVideo(
if (formats.length === 0 || formats.some((entry) => !SAFE_FORMATS.has(entry))) {
throw new Error("Video container format is not allowed");
}
if (
!Number.isInteger(width) ||
!Number.isInteger(height) ||
width < 1 ||
height < 1 ||
width > VIDEO_MAX_DIMENSION ||
height > VIDEO_MAX_DIMENSION ||
width * height > VIDEO_MAX_PIXELS
) {
if (!allVideoStreamsSafe) {
throw new Error("Video dimensions exceed the safe processing limit");
}
return { durationSeconds, formatName, height, width };
return { durationSeconds, formatName, height, streamIndex, width };
}
export async function extractFramesFromLocalVideo(
@@ -264,12 +266,16 @@ export async function extractFramesFromLocalVideo(
frameCount: number;
runner?: VideoCommandRunner;
signal?: AbortSignal;
streamIndex: number;
timeoutMs?: number;
}
): Promise<VideoFrameFile[]> {
assertLocalPath(inputPath);
assertLocalPath(outputDirectory);
const timestamps = calculateFrameTimestamps(options.durationSeconds, options.frameCount);
if (!Number.isInteger(options.streamIndex) || options.streamIndex < 0) {
throw new Error("Video stream index is invalid");
}
const runner = options.runner ?? defaultRunner;
const frames: VideoFrameFile[] = [];
@@ -295,6 +301,8 @@ export async function extractFramesFromLocalVideo(
timestampSeconds.toFixed(3),
"-i",
inputPath,
"-map",
`0:${options.streamIndex}`,
"-vf",
"scale=w='min(1024,iw)':h='min(1024,ih)':force_original_aspect_ratio=decrease",
"-frames:v",
@@ -352,7 +360,6 @@ export async function extractVideoFramesFromBytes(
timeoutMs: number;
}
): Promise<{ durationSeconds: number; frames: ExtractedVideoFrame[] }> {
assertNoEmbeddedMediaReferences(bytes);
const temporaryDirectory = await mkdtemp(join(tmpdir(), "omniroute-video-broker-"));
try {
if (options.signal?.aborted) throw new Error("Video extraction request aborted");
@@ -371,6 +378,7 @@ export async function extractVideoFramesFromBytes(
frameCount: options.frameCount,
runner: options.runner,
signal: options.signal,
streamIndex: metadata.streamIndex,
timeoutMs: options.timeoutMs,
});
const frameBytes = await readBoundedExtractedFrames(frameFiles);

View File

@@ -9,6 +9,9 @@ import { VISION_BRIDGE_DEFAULTS } from "./visionBridgeDefaults";
export type VisionBridgeMode = "auto" | "describe" | "reroute";
export const VIDEO_BRIDGE_TIMEOUT_MIN_MS = 1_000;
export const VIDEO_BRIDGE_TIMEOUT_MAX_MS = 120_000;
export const MODALITY_BRIDGE_DEFAULTS = {
visionMode: "auto" as VisionBridgeMode,
visionTaskAware: true,
@@ -145,7 +148,13 @@ export function resolveVideoBridgeRuntimeSettings(
pickNumber(s.modalityBridgeVideoFrameCount) ?? MODALITY_BRIDGE_DEFAULTS.videoFrameCount,
maxVideos:
pickNumber(s.modalityBridgeVideoMaxVideos) ?? MODALITY_BRIDGE_DEFAULTS.videoMaxVideos,
timeoutMs: pickNumber(s.modalityBridgeVideoTimeout) ?? MODALITY_BRIDGE_DEFAULTS.videoTimeoutMs,
timeoutMs: Math.min(
VIDEO_BRIDGE_TIMEOUT_MAX_MS,
Math.max(
VIDEO_BRIDGE_TIMEOUT_MIN_MS,
pickNumber(s.modalityBridgeVideoTimeout) ?? MODALITY_BRIDGE_DEFAULTS.videoTimeoutMs
)
),
cacheEnabled:
pickBoolean(s.modalityBridgeCacheEnabled) ?? MODALITY_BRIDGE_DEFAULTS.cacheEnabled,
cacheTtlMinutes:

View File

@@ -31,6 +31,9 @@ export const MAX_BODY_BYTES_FILE = 500 * 1024 * 1024;
/** Larger limit for LLM request payloads: 50 MB */
export const MAX_BODY_BYTES_LLM_API = 50 * 1024 * 1024;
/** Fixed limit for the loopback-only Video Bridge extraction broker: 50 MB. */
export const MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER = 50 * 1024 * 1024;
/**
* Media (image generate / edit / upscale / video) is not capped by OmniRoute.
* JSON + base64 inflates payloads by roughly 33%, and provider limits vary by model,
@@ -44,9 +47,20 @@ export const MAX_BODY_BYTES_IMAGE_EDIT = MAX_BODY_BYTES_MEDIA;
/** Configured limit — reads from env or falls back to 10 MB */
export const MAX_BODY_BYTES = parseRequestBodyLimitBytes(process.env.MAX_BODY_SIZE_BYTES);
type BodySizeRule = { prefix: string; limit: number };
type BodySizeRule = {
prefix: string;
limit: number;
exactPath?: boolean;
fixedLimit?: boolean;
};
const ROUTE_LIMITS: BodySizeRule[] = [
{
prefix: "/api/modality-bridge/video/extract",
limit: MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER,
exactPath: true,
fixedLimit: true,
},
{ prefix: "/api/db-backups/import", limit: MAX_BODY_BYTES_IMPORT },
{ prefix: "/api/v1/chat/completions", limit: MAX_BODY_BYTES_LLM_API },
{ prefix: "/api/v1/responses", limit: MAX_BODY_BYTES_LLM_API },
@@ -69,8 +83,11 @@ export function getConfiguredBodySizeLimitBytes(settings?: Record<string, unknow
export function getBodySizeLimit(pathname: string, settings?: Record<string, unknown>): number {
const configuredLimit = getConfiguredBodySizeLimitBytes(settings);
if (PROVIDER_IMAGE_GENERATION_ROUTE.test(pathname)) return MAX_BODY_BYTES_MEDIA;
const customRule = ROUTE_LIMITS.find((rule) => pathname.startsWith(rule.prefix));
return customRule ? Math.max(customRule.limit, configuredLimit) : configuredLimit;
const customRule = ROUTE_LIMITS.find((rule) =>
rule.exactPath ? pathname === rule.prefix : pathname.startsWith(rule.prefix)
);
if (!customRule) return configuredLimit;
return customRule.fixedLimit ? customRule.limit : Math.max(customRule.limit, configuredLimit);
}
/**

View File

@@ -12,6 +12,10 @@ import { HIDEABLE_SIDEBAR_GROUP_IDS } from "@/shared/constants/sidebarGroupVisib
import { HIDEABLE_SIDEBAR_ITEM_IDS, SIDEBAR_SECTIONS } from "@/shared/constants/sidebarVisibility";
import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
import { RESPONSES_PREVIOUS_RESPONSE_ID_MODES } from "@/shared/constants/responsesPreviousResponseId";
import {
VIDEO_BRIDGE_TIMEOUT_MAX_MS,
VIDEO_BRIDGE_TIMEOUT_MIN_MS,
} from "@/shared/constants/modalityBridgeDefaults";
// Import from the server-free constants leaf, NOT from `@/server/authz/routeGuard`:
// this schema is reachable from client components (dashboard onboarding wizard), and
// routeGuard drags in server runtime (→ ioredis) that breaks the client/CLI build.
@@ -360,7 +364,12 @@ export const updateSettingsSchema = z.object({
modalityBridgeVideoModel: z.string().max(200).optional(),
modalityBridgeVideoFrameCount: z.number().int().min(1).max(16).optional(),
modalityBridgeVideoMaxVideos: z.number().int().min(1).max(4).optional(),
modalityBridgeVideoTimeout: z.number().int().min(1000).max(300000).optional(),
modalityBridgeVideoTimeout: z
.number()
.int()
.min(VIDEO_BRIDGE_TIMEOUT_MIN_MS)
.max(VIDEO_BRIDGE_TIMEOUT_MAX_MS)
.optional(),
modalityBridgeCacheEnabled: z.boolean().optional(),
modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(),
modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(),

View File

@@ -7,6 +7,7 @@ import {
MAX_BODY_BYTES_IMAGE_EDIT,
MAX_BODY_BYTES_MEDIA,
MAX_BODY_BYTES_LLM_API,
MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER,
RequestBodyTooLargeError,
readRequestBodyWithLimit,
getBodySizeLimit,
@@ -50,6 +51,44 @@ test("body size guard keeps dedicated upload limits as lower bounds", () => {
);
});
test("Video Bridge broker admission is exactly 50 MiB before policy and route handling", async () => {
const pathname = "/api/modality-bridge/video/extract";
const admittedBytes = 20 * 1024 * 1024;
const rejectedBytes = MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER + 1;
assert.equal(
getBodySizeLimit(pathname, { maxBodySizeMb: 10 }),
MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER
);
assert.equal(
getBodySizeLimit(pathname, { maxBodySizeMb: 100 }),
MAX_BODY_BYTES_VIDEO_BRIDGE_BROKER,
"a broader global setting must not widen the local spawn broker"
);
assert.equal(
checkBodySize(
new Request(`http://localhost${pathname}`, {
method: "POST",
headers: { "content-length": String(admittedBytes) },
}),
getBodySizeLimit(pathname, { maxBodySizeMb: 10 })
),
null,
">10 MiB and <=50 MiB must continue to auth policy and the streamed route cap"
);
const rejection = checkBodySize(
new Request(`http://localhost${pathname}`, {
method: "POST",
headers: { "content-length": String(rejectedBytes) },
}),
getBodySizeLimit(pathname, { maxBodySizeMb: 100 })
);
assert.ok(rejection);
assert.equal(rejection.status, 413);
assert.equal((await rejection.json()).error.code, "PAYLOAD_TOO_LARGE");
});
test("/api/v1/images/edits admits a 20 MiB image in multipart or base64 JSON envelopes", () => {
const multipartBytes = 20 * 1024 * 1024 + 1024 * 1024;
const base64JsonBytes = Math.ceil((20 * 1024 * 1024 * 4) / 3) + 1024;

View File

@@ -2,11 +2,39 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
VIDEO_BRIDGE_INLINE_MAX_BYTES,
decodeVideoDataUri,
describeVideoPart,
estimateDecodedBase64Bytes,
extractVideoParts,
replaceVideoParts,
} from "../../../src/lib/guardrails/videoBridgeHelpers.ts";
test("inline base64 is size-estimated and rejected before allocation", () => {
assert.equal(VIDEO_BRIDGE_INLINE_MAX_BYTES, 36 * 1024 * 1024);
assert.equal(estimateDecodedBase64Bytes("QUJDRA=="), 4);
assert.equal(estimateDecodedBase64Bytes("QUJD\nRA=="), 4);
let decodeCalls = 0;
assert.throws(
() =>
decodeVideoDataUri("data:video/mp4;base64,QUJDRA==", 3, (base64) => {
decodeCalls += 1;
return Buffer.from(base64, "base64");
}),
/maximum size/
);
assert.equal(decodeCalls, 0, "oversized inline payload must fail before Buffer.from");
assert.deepEqual(
decodeVideoDataUri("data:video/mp4;base64,QUJDRA==", 4, (base64) => {
decodeCalls += 1;
return Buffer.from(base64, "base64");
}),
Buffer.from("ABCD")
);
assert.equal(decodeCalls, 1);
});
test("extracts and replaces video parts in Chat and Responses payloads without shifting siblings", () => {
const chatBody = {
messages: [
@@ -247,7 +275,7 @@ test("extracts Anthropic type:video base64 and URL sources and replaces them in
{ type: "text", text: "middle" },
{
type: "video",
source: { type: "url", media_type: "video/webm", url: "https://cdn.example/a.webm" },
source: { type: "url", url: "https://cdn.example/a.webm" },
},
],
},

View File

@@ -28,7 +28,7 @@ test("probes and extracts a local video using shell-free bounded commands", asyn
return {
stdout: JSON.stringify({
format: { duration: "8.0", format_name: "mov,mp4,m4a,3gp,3g2,mj2" },
streams: [{ codec_type: "video", width: 1920, height: 1080 }],
streams: [{ index: 0, codec_type: "video", width: 1920, height: 1080 }],
}),
stderr: "",
};
@@ -45,6 +45,7 @@ test("probes and extracts a local video using shell-free bounded commands", asyn
durationSeconds: metadata.durationSeconds,
frameCount: 4,
runner,
streamIndex: metadata.streamIndex,
timeoutMs: 10_000,
});
@@ -107,7 +108,7 @@ test("rejects remote process inputs and videos beyond the duration bound", async
const runner: VideoCommandRunner = async () => ({
stdout: JSON.stringify({
format: { duration: "601", format_name: "mp4" },
streams: [{ codec_type: "video", width: 1280, height: 720 }],
streams: [{ index: 0, codec_type: "video", width: 1280, height: 720 }],
}),
stderr: "private upstream details",
});
@@ -128,7 +129,7 @@ test("rejects reference-bearing formats before extraction and confines both tool
return {
stdout: JSON.stringify({
format: { duration: "10", format_name: "hls" },
streams: [{ codec_type: "video", width: 640, height: 360 }],
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
}),
stderr: "http://169.254.169.254/latest/meta-data",
};
@@ -149,41 +150,99 @@ test("rejects reference-bearing formats before extraction and confines both tool
);
});
test("rejects embedded network and traversal references before invoking either process", async () => {
let calls = 0;
const runner: VideoCommandRunner = async () => {
calls += 1;
test("safe containers may contain URL or traversal-like compressed bytes without false rejection", async () => {
const calls: string[] = [];
const runner: VideoCommandRunner = async (executable, args) => {
calls.push(executable);
if (executable === "ffprobe") {
return {
stdout: JSON.stringify({
format: { duration: "2", format_name: "mp4" },
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
}),
stderr: "",
};
}
await writeFile(args.at(-1) ?? "", Buffer.from([0xff, 0xd8, 0xff, 0xd9]));
return { stdout: "", stderr: "" };
};
for (const bytes of [
Buffer.from("#EXTM3U\nhttp://127.0.0.1/private.ts"),
Buffer.from("file:../../etc/passwd"),
]) {
await assert.rejects(
() =>
extractVideoFramesFromBytes(bytes, {
frameCount: 1,
maxDurationSeconds: 600,
runner,
timeoutMs: 5_000,
}),
/media reference/
);
}
assert.equal(calls, 0);
const validContainerBytes = Buffer.concat([
Buffer.from([0, 0, 0, 24, 0x66, 0x74, 0x79, 0x70]),
Buffer.from("compressed-chunk:http://127.0.0.1/../not-a-reference"),
]);
const result = await extractVideoFramesFromBytes(validContainerBytes, {
frameCount: 1,
maxDurationSeconds: 600,
runner,
timeoutMs: 5_000,
});
assert.deepEqual(calls, ["ffprobe", "ffmpeg"]);
assert.equal(result.frames.length, 1);
});
test("rejects oversized dimensions and pixel counts from sanitized probe metadata", async () => {
const runner: VideoCommandRunner = async () => ({
stdout: JSON.stringify({
format: { duration: "2", format_name: "mp4" },
streams: [{ codec_type: "video", width: 16384, height: 16384 }],
streams: [{ index: 0, codec_type: "video", width: 16384, height: 16384 }],
}),
stderr: "private path",
});
await assert.rejects(() => probeLocalVideo("/tmp/oversized.mp4", { runner }), /dimensions/);
});
test("rejects a container when any video stream exceeds dimension or pixel limits", async () => {
const runner: VideoCommandRunner = async () => ({
stdout: JSON.stringify({
format: { duration: "2", format_name: "mp4" },
streams: [
{ index: 0, codec_type: "video", width: 640, height: 360 },
{ index: 1, codec_type: "video", width: 16384, height: 16384 },
],
}),
stderr: "",
});
await assert.rejects(
() => probeLocalVideo("/tmp/multiple-streams.mp4", { runner }),
/dimensions/
);
});
test("selects the lowest validated video stream index and maps it explicitly in ffmpeg", async () => {
const calls: Array<{ executable: string; args: string[] }> = [];
const runner: VideoCommandRunner = async (executable, args) => {
calls.push({ executable, args: [...args] });
return executable === "ffprobe"
? {
stdout: JSON.stringify({
format: { duration: "4", format_name: "mp4" },
streams: [
{ index: 3, codec_type: "video", width: 1280, height: 720 },
{ index: 1, codec_type: "video", width: 640, height: 360 },
],
}),
stderr: "",
}
: { stdout: "", stderr: "" };
};
const metadata = await probeLocalVideo("/tmp/multiple-safe.mp4", { runner });
await extractFramesFromLocalVideo("/tmp/multiple-safe.mp4", "/tmp/frames", {
durationSeconds: metadata.durationSeconds,
frameCount: 1,
runner,
streamIndex: metadata.streamIndex,
});
assert.equal(metadata.streamIndex, 1);
const ffmpegArgs = calls.find((call) => call.executable === "ffmpeg")?.args ?? [];
const mapIndex = ffmpegArgs.indexOf("-map");
assert.deepEqual(ffmpegArgs.slice(mapIndex, mapIndex + 2), ["-map", "0:1"]);
});
test("runtime status exposes sanitized versions and a sanitized unavailable reason", async () => {
resetVideoRuntimeProbeCacheForTests();
const ready = await probeVideoRuntime({

View File

@@ -5,6 +5,7 @@ import path from "node:path";
import test from "node:test";
import { makeManagementSessionRequest } from "../helpers/managementSession.ts";
import { AUTHZ_HEADER_PEER_LOCALITY } from "../../src/server/authz/headers.ts";
const dataDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-video-runtime-route-"));
const originalDataDirectory = process.env.DATA_DIR;
@@ -16,6 +17,12 @@ const core = await import("../../src/lib/db/core.ts");
const settings = await import("../../src/lib/db/settings.ts");
const route = await import("../../src/app/api/modality-bridge/video/runtime/route.ts");
async function withLocality(request: Request, locality: "loopback" | "lan"): Promise<Request> {
const headers = new Headers(request.headers);
headers.set(AUTHZ_HEADER_PEER_LOCALITY, locality);
return new Request(request, { headers });
}
test.beforeEach(async () => {
core.resetDbInstance();
fs.rmSync(dataDirectory, { force: true, recursive: true });
@@ -37,10 +44,12 @@ test.after(() => {
test("Video Bridge runtime status requires management auth and returns only sanitized fields", async () => {
const url = "http://localhost/api/modality-bridge/video/runtime";
const unauthenticated = await route.GET(new Request(url));
const unauthenticated = await route.GET(await withLocality(new Request(url), "loopback"));
assert.equal(unauthenticated.status, 401);
const authenticated = await route.GET(await makeManagementSessionRequest(url));
const authenticated = await route.GET(
await withLocality(await makeManagementSessionRequest(url), "loopback")
);
assert.equal(authenticated.status, 200);
assert.equal(authenticated.headers.get("cache-control"), "no-store");
const body = (await authenticated.json()) as Record<string, unknown>;
@@ -54,3 +63,25 @@ test("Video Bridge runtime status requires management auth and returns only sani
assert.equal(JSON.stringify(body).includes("/private/"), false);
assert.equal(JSON.stringify(body).includes("stderr"), false);
});
test("Video Bridge runtime rejects private-LAN callers before auth or subprocess probing", async () => {
const url = "http://localhost/api/modality-bridge/video/runtime";
let probes = 0;
const probe = async () => {
probes += 1;
return { available: true, ffmpegVersion: "test", ffprobeVersion: "test" };
};
const unauthenticated = await route.handleVideoRuntimeStatus(
await withLocality(new Request(url), "lan"),
{ probe }
);
const authenticated = await route.handleVideoRuntimeStatus(
await withLocality(await makeManagementSessionRequest(url), "lan"),
{ probe }
);
assert.equal(unauthenticated.status, 403);
assert.equal(authenticated.status, 403);
assert.equal(probes, 0);
});

View File

@@ -119,7 +119,7 @@ describe("ModalityBridgeVideoTab", () => {
const options = Array.from(element.querySelectorAll("option")).map((option) => option.value);
expect(options).toContain("openai/gpt-4o-mini");
expect(options).not.toContain("example/text-only");
expect(element.textContent).toContain("4 requests");
expect(element.textContent).toContain("4 requestlogger.attempts");
expect(element.textContent).toContain("3 modalityBridgeStatsBridged");
expect(element.textContent).toContain("1 modalityBridgeStatsFailures");
expect(element.textContent).toContain("trafficInspector.timingTotalLatency: 400 ms");
@@ -162,6 +162,35 @@ describe("ModalityBridgeVideoTab", () => {
expect(patches).toContainEqual({ modalityBridgeVideoEnabled: true });
});
it("caps the configurable timeout at the broker's 120 second hard deadline", async () => {
const element = await render();
const timeout = element.querySelector(
'[data-testid="modality-bridge-video-timeout"]'
) as HTMLInputElement;
expect(timeout.max).toBe("120000");
act(() => {
const setter = Object.getOwnPropertyDescriptor(
window.HTMLInputElement.prototype,
"value"
)?.set;
setter?.call(timeout, "180000");
timeout.dispatchEvent(new Event("input", { bubbles: true }));
});
await act(async () => {
timeout.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
await new Promise((resolve) => setTimeout(resolve, 0));
});
await waitFor(
() =>
fetchMock.mock.calls.some(([, init]) => {
if (init?.method !== "PATCH") return false;
const body = JSON.parse(String(init.body)) as Record<string, unknown>;
return body.modalityBridgeVideoTimeout === 120_000;
}),
"clamped timeout PATCH"
);
});
it("shows a load error instead of silently applying defaults", async () => {
failSettingsLoad = true;
const element = await renderWithoutWaiting();

View File

@@ -6,20 +6,38 @@ import { LOCAL_ONLY_API_PREFIXES, isLocalOnlyPath } from "../../src/server/authz
import { SPAWN_CAPABLE_PREFIXES } from "../../src/shared/constants/spawnCapablePrefixes.ts";
import { managementPolicy } from "../../src/server/authz/policies/management.ts";
import {
BROKER_TIMEOUT_MS,
POST,
handleVideoExtractionBrokerRequest,
readBoundedVideoBrokerBody,
} from "../../src/app/api/modality-bridge/video/extract/route.ts";
import { buildVideoBridgeBrokerHeaders } from "../../src/lib/guardrails/videoBridgeBrokerAuth.ts";
import { createVideoExtractionQueue } from "../../src/lib/guardrails/videoBridgeBrokerQueue.ts";
import { AUTHZ_HEADER_PEER_LOCALITY } from "../../src/server/authz/headers.ts";
import { VIDEO_BRIDGE_TIMEOUT_MAX_MS } from "../../src/shared/constants/modalityBridgeDefaults.ts";
const PREFIX = "/api/modality-bridge/video/";
const EXTRACT_PATH = `${PREFIX}extract`;
function trustedBrokerRequest(signal?: AbortSignal): Request {
return new Request(`http://localhost${EXTRACT_PATH}?frames=1`, {
method: "POST",
headers: {
...buildVideoBridgeBrokerHeaders(),
[AUTHZ_HEADER_PEER_LOCALITY]: "loopback",
"Content-Type": "application/octet-stream",
},
body: Buffer.from("video"),
signal,
});
}
test("Video Bridge runtime and broker share an exact LOCAL_ONLY + SPAWN_CAPABLE prefix", () => {
assert.ok(LOCAL_ONLY_API_PREFIXES.includes(PREFIX));
assert.ok(SPAWN_CAPABLE_PREFIXES.includes(PREFIX));
assert.equal(isLocalOnlyPath(EXTRACT_PATH, "POST"), true);
assert.equal(isLocalOnlyPath(`${PREFIX}runtime`, "GET"), true);
assert.equal(BROKER_TIMEOUT_MS, VIDEO_BRIDGE_TIMEOUT_MAX_MS);
});
test("non-loopback broker access is rejected as LOCAL_ONLY before authentication", async () => {
@@ -84,6 +102,35 @@ test("bounded broker body reading accepts absent length and cancels a lying over
assert.equal(cancelled, true);
});
test("broker route maps queue capacity, client disconnect, and deadline to distinct HTTP statuses", async () => {
const neverExtract = async () => {
throw new Error("extractor must not run");
};
const capacity = await handleVideoExtractionBrokerRequest(trustedBrokerRequest(), {
queue: createVideoExtractionQueue({ concurrency: 1, maxPending: 0, maxQueuedBytes: 1 }),
extractFrames: neverExtract,
});
assert.equal(capacity.status, 503);
assert.equal(capacity.headers.get("Retry-After"), "1");
const clientController = new AbortController();
clientController.abort();
const clientAbort = await handleVideoExtractionBrokerRequest(
trustedBrokerRequest(clientController.signal),
{ extractFrames: neverExtract }
);
assert.equal(clientAbort.status, 499);
assert.equal(clientAbort.headers.get("Retry-After"), null);
const deadline = await handleVideoExtractionBrokerRequest(trustedBrokerRequest(), {
deadlineSignal: AbortSignal.abort(),
extractFrames: neverExtract,
});
assert.equal(deadline.status, 504);
assert.equal(deadline.headers.get("Retry-After"), null);
});
test("configured base path preserves the exact self-hop without widening broker authentication", async () => {
const previousBasePath = process.env.OMNIROUTE_BASE_PATH;
process.env.OMNIROUTE_BASE_PATH = "/omniroute";
@@ -141,3 +188,28 @@ test("OpenAPI marks both Video Bridge process routes loopback-only", () => {
assert.match(openapi.slice(start, start + 800), /x-loopback-only:\s*true/);
}
});
test("public docs describe the exact Video Bridge quota, deadline, and abort contracts", () => {
const openapi = readFileSync("docs/openapi.yaml", "utf8");
const statsStart = openapi.indexOf(" /api/modality-bridge/stats:");
const runtimeStart = openapi.indexOf(` ${PREFIX}runtime:`);
const extractStart = openapi.indexOf(` ${EXTRACT_PATH}:`);
const statsContract = openapi.slice(statsStart, runtimeStart);
const runtimeContract = openapi.slice(runtimeStart, extractStart);
const extractContract = openapi.slice(extractStart, openapi.indexOf(" /api/cache/stats:"));
assert.match(statsContract, /latencySamples/);
assert.match(runtimeContract, /trusted loopback[^\n]*before authentication/i);
assert.match(extractContract, /50 MiB/);
assert.match(extractContract, /32 MiB/);
assert.match(extractContract, /"499":[\s\S]*Client request aborted/);
assert.match(extractContract, /"503":[\s\S]*Retry-After/);
assert.match(extractContract, /"504":[\s\S]*deadline/i);
const guardrails = readFileSync("docs/security/GUARDRAILS.md", "utf8");
assert.match(guardrails, /inline[\s\S]{0,120}36 MiB/i);
assert.match(guardrails, /remote[\s\S]{0,120}50 MiB/i);
assert.match(guardrails, /`signal\?: AbortSignal`/);
assert.match(guardrails, /request abort[^\n]*fail-open exception/i);
assert.doesNotMatch(guardrails, /modalityBridgeVideoTimeout`[^\n]*300000/);
});

View File

@@ -45,7 +45,7 @@ test("Video Bridge settings schema rejects values outside extraction bounds", ()
for (const [field, value] of Object.entries({
modalityBridgeVideoFrameCount: 17,
modalityBridgeVideoMaxVideos: 0,
modalityBridgeVideoTimeout: 300_001,
modalityBridgeVideoTimeout: 120_001,
})) {
assert.equal(
updateSettingsSchema.safeParse({ [field]: value }).success,
@@ -54,3 +54,17 @@ test("Video Bridge settings schema rejects values outside extraction bounds", ()
);
}
});
test("persisted legacy Video Bridge timeouts clamp to the broker's 120 second deadline", () => {
for (const timeoutMs of [180_000, 300_000]) {
assert.equal(
resolveVideoBridgeRuntimeSettings({ modalityBridgeVideoTimeout: timeoutMs }).timeoutMs,
120_000
);
assert.equal(
updateSettingsSchema.safeParse({ modalityBridgeVideoTimeout: timeoutMs }).success,
false,
`new writes must reject ${timeoutMs}ms instead of exceeding the broker deadline`
);
}
});