mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-19 05:12:16 +03:00
FU-06 (Audio Bridge STT orchestration): one download, two extractions — takes already-downloaded video bytes and extracts bounded mono 16kHz PCM WAV via the loopback broker's new mode=audio operation, sharing the exact same queue/deadline/byte budgets as the frame path. Dual opt-in (operator setting default false + per-request), only reaches the STT call when both are on. Rebased onto the tip after sibling #12011 (subtitle mode) landed first, both touching the same broker route/client — combined additively so frames/audio/subtitles all share the one extractionQueue singleton. Re-validated: 59/59 focused tests pass.
This commit is contained in:
committed by
GitHub
parent
e8b2cd208d
commit
5fcd39bd6f
@@ -0,0 +1 @@
|
||||
- **feat(video):** orchestrate optional Video Bridge audio extraction and Audio Bridge STT behind a dual opt-in (operator setting AND per-request signal) — a new loopback-only broker `mode=audio` operation shares the frame path's exact process queue, deadline, AbortSignal, and byte budgets to extract a bounded mono 16 kHz PCM WAV from the same already-downloaded video, then reuses the existing Audio Bridge transcription boundary; provider segment timing is preserved when available and marked coarse otherwise, and every failure degrades to a visual-only-safe partial instead of throwing (#11654).
|
||||
@@ -9,6 +9,10 @@ import {
|
||||
type VideoExtractionQueue,
|
||||
VideoExtractionQueueError,
|
||||
} from "@/lib/guardrails/videoBridgeBrokerQueue";
|
||||
import {
|
||||
extractVideoAudioFromBytes,
|
||||
type ExtractedVideoAudio,
|
||||
} from "@/lib/guardrails/videoBridgeAudioExtraction";
|
||||
import {
|
||||
extractVideoFramesFromBytes,
|
||||
type VideoFocusBounds,
|
||||
@@ -42,15 +46,35 @@ function invalid(message: string, status = 400, headers?: Record<string, string>
|
||||
return response;
|
||||
}
|
||||
|
||||
/** The extract route serves two mutually exclusive operations behind one authenticated path. */
|
||||
/**
|
||||
* The extract route serves up to three mutually exclusive operations behind one authenticated
|
||||
* path: subtitle probe (`?subtitles=1`), audio extraction (`?mode=audio`), and frame sampling
|
||||
* (default / `?mode=frames`).
|
||||
*/
|
||||
function isSubtitleProbeRequest(url: URL): boolean {
|
||||
return url.searchParams.get("subtitles") === "1";
|
||||
}
|
||||
|
||||
export type VideoBrokerMode = "audio" | "frames";
|
||||
|
||||
const FRAME_ONLY_QUERY_KEYS = ["frames", "samplingPolicy", "start", "end"];
|
||||
|
||||
/** `mode` is opt-in and defaults to the pre-existing frame path (back-compat). */
|
||||
function parseBrokerMode(url: URL): VideoBrokerMode | null {
|
||||
const raw = url.searchParams.get("mode");
|
||||
if (raw === null) return "frames";
|
||||
return raw === "frames" || raw === "audio" ? raw : null;
|
||||
}
|
||||
|
||||
/** Audio mode shares this route/queue but takes no frame-shaping parameters (v1). */
|
||||
function hasFrameOnlyParams(url: URL): boolean {
|
||||
return FRAME_ONLY_QUERY_KEYS.some((key) => url.searchParams.has(key));
|
||||
}
|
||||
|
||||
function parseFrameCount(url: URL): number | null {
|
||||
if (
|
||||
[...url.searchParams.keys()].some(
|
||||
(key) => !["frames", "samplingPolicy", "start", "end"].includes(key)
|
||||
(key) => !["mode", ...FRAME_ONLY_QUERY_KEYS].includes(key)
|
||||
)
|
||||
) {
|
||||
return null;
|
||||
@@ -126,12 +150,13 @@ export async function readBoundedVideoBrokerBody(
|
||||
|
||||
interface VideoExtractionBrokerRouteDependencies {
|
||||
deadlineSignal?: AbortSignal;
|
||||
extractAudio?: typeof extractVideoAudioFromBytes;
|
||||
extractFrames?: typeof extractVideoFramesFromBytes;
|
||||
extractSubtitles?: typeof extractVideoSubtitlesFromBytes;
|
||||
queue?: VideoExtractionQueue;
|
||||
}
|
||||
|
||||
/** Shared by both operations: declared-length pre-check, bounded read, actual-size check. */
|
||||
/** Shared by all three operations: declared-length pre-check, bounded read, actual-size check. */
|
||||
async function readValidatedVideoBrokerBytes(request: Request): Promise<Buffer | Response> {
|
||||
const declaredHeader = request.headers.get("content-length");
|
||||
const declaredLength = declaredHeader === null ? null : Number(declaredHeader);
|
||||
@@ -161,6 +186,56 @@ async function readValidatedVideoBrokerBytes(request: Request): Promise<Buffer |
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Same status/telemetry mapping for both broker operations — one failure taxonomy. */
|
||||
function mapBrokerExtractionError(
|
||||
error: unknown,
|
||||
context: { deadline: AbortSignal; inputBytes: number; mode: VideoBrokerMode; request: Request }
|
||||
): Response {
|
||||
const unavailable =
|
||||
error && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
||||
const queueCapacity =
|
||||
error instanceof VideoExtractionQueueError && error.code === "QUEUE_CAPACITY";
|
||||
const clientAborted = context.request.signal.aborted;
|
||||
const deadlineExceeded = !clientAborted && context.deadline.aborted;
|
||||
log.warn(
|
||||
{
|
||||
aborted: clientAborted,
|
||||
code: clientAborted
|
||||
? "CLIENT_ABORTED"
|
||||
: queueCapacity
|
||||
? "QUEUE_CAPACITY"
|
||||
: deadlineExceeded
|
||||
? "DEADLINE_EXCEEDED"
|
||||
: unavailable
|
||||
? "RUNTIME_UNAVAILABLE"
|
||||
: "EXTRACTION_FAILED",
|
||||
inputBytes: context.inputBytes,
|
||||
mode: context.mode,
|
||||
},
|
||||
"Video Bridge broker extraction failed"
|
||||
);
|
||||
if (clientAborted) return invalid("Video extraction was aborted", 499);
|
||||
if (deadlineExceeded) return invalid("Video extraction deadline exceeded", 504);
|
||||
if (queueCapacity) {
|
||||
return invalid("Video extraction capacity is temporarily unavailable", 503, {
|
||||
"Retry-After": "1",
|
||||
});
|
||||
}
|
||||
if (unavailable) return invalid("Video extraction runtime is unavailable", 503);
|
||||
return invalid("Video extraction failed", 422);
|
||||
}
|
||||
|
||||
function serializeAudioResult(result: ExtractedVideoAudio & { durationSeconds: number }): unknown {
|
||||
return {
|
||||
audio: {
|
||||
channels: result.channels,
|
||||
dataUri: result.dataUri,
|
||||
sampleRateHz: result.sampleRateHz,
|
||||
},
|
||||
durationSeconds: result.durationSeconds,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded, allowlisted-codec subtitle probe (#11659). Stamps the response with the shared
|
||||
* broker fingerprint so the client-side adapter can refuse anything not actually produced by
|
||||
@@ -245,6 +320,45 @@ export async function handleVideoExtractionBrokerRequest(
|
||||
if (isSubtitleProbeRequest(url)) {
|
||||
return handleSubtitleProbeBrokerRequest(request, url, dependencies);
|
||||
}
|
||||
const mode = parseBrokerMode(url);
|
||||
if (!mode) return invalid("Video Bridge broker mode must be frames or audio");
|
||||
|
||||
if (mode === "audio") {
|
||||
if (hasFrameOnlyParams(url)) {
|
||||
return invalid("Video Bridge audio mode does not accept frame parameters");
|
||||
}
|
||||
const bytes = await readValidatedVideoBrokerBytes(request);
|
||||
if (bytes instanceof Response) return bytes;
|
||||
|
||||
// Same deadline/queue/byte budget the frame path uses — not a parallel set of limits.
|
||||
const deadline = dependencies.deadlineSignal ?? AbortSignal.timeout(BROKER_TIMEOUT_MS);
|
||||
const signal = AbortSignal.any([request.signal, deadline]);
|
||||
const queue = dependencies.queue ?? extractionQueue;
|
||||
const extractAudio = dependencies.extractAudio ?? extractVideoAudioFromBytes;
|
||||
try {
|
||||
const result = await queue.run(
|
||||
bytes.byteLength,
|
||||
() =>
|
||||
extractAudio(bytes, {
|
||||
maxDurationSeconds: MAX_DURATION_SECONDS,
|
||||
signal,
|
||||
timeoutMs: BROKER_TIMEOUT_MS,
|
||||
}),
|
||||
signal
|
||||
);
|
||||
return Response.json(serializeAudioResult(result), {
|
||||
headers: { "Cache-Control": "no-store" },
|
||||
});
|
||||
} catch (error) {
|
||||
return mapBrokerExtractionError(error, {
|
||||
deadline,
|
||||
inputBytes: bytes.byteLength,
|
||||
mode,
|
||||
request,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const frameCount = parseFrameCount(url);
|
||||
if (!frameCount) return invalid("Video Bridge frame count must be between 1 and 16");
|
||||
const samplingPolicy = parseSamplingPolicy(url);
|
||||
@@ -252,32 +366,8 @@ export async function handleVideoExtractionBrokerRequest(
|
||||
if (focusWindow === null && (url.searchParams.has("start") || url.searchParams.has("end"))) {
|
||||
return invalid("Video Bridge focus window bounds are invalid");
|
||||
}
|
||||
const declaredHeader = request.headers.get("content-length");
|
||||
const declaredLength = declaredHeader === null ? null : Number(declaredHeader);
|
||||
if (
|
||||
declaredLength !== null &&
|
||||
(!Number.isFinite(declaredLength) || declaredLength < 1 || declaredLength > MAX_INPUT_BYTES)
|
||||
) {
|
||||
await request.body?.cancel("Video Bridge input exceeds the byte limit");
|
||||
return invalid("Video Bridge input exceeds the byte limit", 413);
|
||||
}
|
||||
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
bytes = await readBoundedVideoBrokerBody(request);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === "VIDEO_INPUT_TOO_LARGE") {
|
||||
return invalid("Video Bridge input exceeds the byte limit", 413);
|
||||
}
|
||||
return invalid("Video Bridge input could not be read");
|
||||
}
|
||||
if (
|
||||
bytes.byteLength < 1 ||
|
||||
bytes.byteLength > MAX_INPUT_BYTES ||
|
||||
(declaredLength !== null && bytes.byteLength !== declaredLength)
|
||||
) {
|
||||
return invalid("Video Bridge input exceeds the byte limit", 413);
|
||||
}
|
||||
const bytes = await readValidatedVideoBrokerBytes(request);
|
||||
if (bytes instanceof Response) return bytes;
|
||||
|
||||
const deadline = dependencies.deadlineSignal ?? AbortSignal.timeout(BROKER_TIMEOUT_MS);
|
||||
const signal = AbortSignal.any([request.signal, deadline]);
|
||||
@@ -299,38 +389,7 @@ export async function handleVideoExtractionBrokerRequest(
|
||||
);
|
||||
return Response.json(result, { headers: { "Cache-Control": "no-store" } });
|
||||
} catch (error) {
|
||||
const unavailable =
|
||||
error && typeof error === "object" && "code" in error && error.code === "ENOENT";
|
||||
const queueCapacity =
|
||||
error instanceof VideoExtractionQueueError && error.code === "QUEUE_CAPACITY";
|
||||
const clientAborted = request.signal.aborted;
|
||||
const deadlineExceeded = !clientAborted && deadline.aborted;
|
||||
log.warn(
|
||||
{
|
||||
aborted: clientAborted,
|
||||
code: clientAborted
|
||||
? "CLIENT_ABORTED"
|
||||
: queueCapacity
|
||||
? "QUEUE_CAPACITY"
|
||||
: deadlineExceeded
|
||||
? "DEADLINE_EXCEEDED"
|
||||
: unavailable
|
||||
? "RUNTIME_UNAVAILABLE"
|
||||
: "EXTRACTION_FAILED",
|
||||
frameCount,
|
||||
inputBytes: bytes.byteLength,
|
||||
},
|
||||
"Video Bridge broker extraction failed"
|
||||
);
|
||||
if (clientAborted) return invalid("Video extraction was aborted", 499);
|
||||
if (deadlineExceeded) return invalid("Video extraction deadline exceeded", 504);
|
||||
if (queueCapacity) {
|
||||
return invalid("Video extraction capacity is temporarily unavailable", 503, {
|
||||
"Retry-After": "1",
|
||||
});
|
||||
}
|
||||
if (unavailable) return invalid("Video extraction runtime is unavailable", 503);
|
||||
return invalid("Video extraction failed", 422);
|
||||
return mapBrokerExtractionError(error, { deadline, inputBytes: bytes.byteLength, mode, request });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,19 @@ export interface AudioTranscriptionConfig {
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface AudioTranscriptionSegment {
|
||||
confidence?: number;
|
||||
endSeconds: number;
|
||||
startSeconds: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface AudioTranscriptionTimedResult {
|
||||
/** Present only when the provider actually returned per-segment timing. */
|
||||
segments?: AudioTranscriptionSegment[];
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface AudioTranscriptionDependencies {
|
||||
fetchImpl?: typeof fetch;
|
||||
fetchRemote?: (
|
||||
@@ -163,39 +176,94 @@ export async function selectAudioBridgeModel(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Send one audio part through OmniRoute's existing multipart transcription route. */
|
||||
export async function callAudioTranscription(
|
||||
/** Resolve one audio part's raw bytes + best-guess MIME, decoding/fetching its `ref`. */
|
||||
async function resolveAudioBytes(
|
||||
part: AudioPart,
|
||||
config: AudioTranscriptionConfig,
|
||||
deps: AudioTranscriptionDependencies = {}
|
||||
): Promise<string> {
|
||||
deps: AudioTranscriptionDependencies,
|
||||
signal: AbortSignal
|
||||
): Promise<{ bytes: Buffer; mime?: string }> {
|
||||
const dataUri = /^data:([^;,]+);base64,(.+)$/is.exec(part.ref);
|
||||
if (dataUri) {
|
||||
return { bytes: Buffer.from(dataUri[2], "base64"), mime: dataUri[1].toLowerCase() };
|
||||
}
|
||||
if (/^https?:\/\//i.test(part.ref)) {
|
||||
const fetchRemote =
|
||||
deps.fetchRemote ??
|
||||
((url: string, options: { signal: AbortSignal }) =>
|
||||
fetchRemoteImage(url, {
|
||||
guard: "public-only",
|
||||
maxBytes: 25 * 1024 * 1024,
|
||||
pinDns: true,
|
||||
signal: options.signal,
|
||||
timeoutMs: config.timeoutMs,
|
||||
}));
|
||||
const remote = await fetchRemote(part.ref, { signal });
|
||||
return { bytes: remote.buffer, mime: remote.contentType.split(";", 1)[0]?.trim().toLowerCase() };
|
||||
}
|
||||
return { bytes: Buffer.from(part.ref, "base64") };
|
||||
}
|
||||
|
||||
/** Build the exact multipart body OmniRoute's own transcription route expects. */
|
||||
function buildTranscriptionMultipartBody(
|
||||
bytes: Buffer,
|
||||
format: string,
|
||||
safeMime: string,
|
||||
model: string,
|
||||
extraFields: Readonly<Record<string, string>>
|
||||
// Buffer<ArrayBuffer> (what Buffer.concat actually returns), not the wider
|
||||
// Buffer<ArrayBufferLike>: only the former is assignable to fetch's BodyInit under the
|
||||
// open-sse tsconfig, and this body is passed straight to fetch() below (#11654).
|
||||
): { body: Buffer<ArrayBuffer>; boundary: string } {
|
||||
const fileName = `audio.${format.replace(/[^a-z0-9]/g, "") || "wav"}`;
|
||||
const boundary = `----OmniRouteAudioBridge${randomUUID().replace(/-/g, "")}`;
|
||||
const CRLF = "\r\n";
|
||||
const extraFieldParts = Object.entries(extraFields).map(([name, value]) =>
|
||||
Buffer.from(
|
||||
`--${boundary}${CRLF}` +
|
||||
`Content-Disposition: form-data; name="${name}"${CRLF}${CRLF}` +
|
||||
`${value}${CRLF}`
|
||||
)
|
||||
);
|
||||
const body = Buffer.concat([
|
||||
Buffer.from(
|
||||
`--${boundary}${CRLF}` +
|
||||
`Content-Disposition: form-data; name="file"; filename="${fileName}"${CRLF}` +
|
||||
`Content-Type: ${safeMime}${CRLF}${CRLF}`
|
||||
),
|
||||
bytes,
|
||||
Buffer.from(
|
||||
`${CRLF}--${boundary}${CRLF}` +
|
||||
`Content-Disposition: form-data; name="model"${CRLF}${CRLF}` +
|
||||
`${model}${CRLF}`
|
||||
),
|
||||
...extraFieldParts,
|
||||
Buffer.from(`--${boundary}--${CRLF}`),
|
||||
]);
|
||||
return { body, boundary };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send one audio part through OmniRoute's existing multipart transcription
|
||||
* route (the one Audio Bridge transcription boundary) and return the parsed
|
||||
* JSON response. `extraFields` lets callers request provider extras (e.g.
|
||||
* `response_format=verbose_json`) without duplicating this HTTP client.
|
||||
*/
|
||||
async function sendAudioTranscriptionRequest(
|
||||
part: AudioPart,
|
||||
config: AudioTranscriptionConfig,
|
||||
deps: AudioTranscriptionDependencies,
|
||||
extraFields: Readonly<Record<string, string>>
|
||||
): Promise<Record<string, unknown>> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), config.timeoutMs);
|
||||
try {
|
||||
let bytes: Buffer;
|
||||
let detectedMime: string | undefined;
|
||||
const dataUri = /^data:([^;,]+);base64,(.+)$/is.exec(part.ref);
|
||||
if (dataUri) {
|
||||
detectedMime = dataUri[1].toLowerCase();
|
||||
bytes = Buffer.from(dataUri[2], "base64");
|
||||
} else if (/^https?:\/\//i.test(part.ref)) {
|
||||
const fetchRemote =
|
||||
deps.fetchRemote ??
|
||||
((url: string, options: { signal: AbortSignal }) =>
|
||||
fetchRemoteImage(url, {
|
||||
guard: "public-only",
|
||||
maxBytes: 25 * 1024 * 1024,
|
||||
pinDns: true,
|
||||
signal: options.signal,
|
||||
timeoutMs: config.timeoutMs,
|
||||
}));
|
||||
const remote = await fetchRemote(part.ref, { signal: controller.signal });
|
||||
bytes = remote.buffer;
|
||||
detectedMime = remote.contentType.split(";", 1)[0]?.trim().toLowerCase();
|
||||
} else {
|
||||
bytes = Buffer.from(part.ref, "base64");
|
||||
}
|
||||
|
||||
const { bytes, mime: detectedMime } = await resolveAudioBytes(
|
||||
part,
|
||||
config,
|
||||
deps,
|
||||
controller.signal
|
||||
);
|
||||
const configuredFormat = part.format?.trim().toLowerCase();
|
||||
const format =
|
||||
configuredFormat || (detectedMime ? AUDIO_MIME_FORMAT[detectedMime] : undefined) || "wav";
|
||||
@@ -206,22 +274,13 @@ export async function callAudioTranscription(
|
||||
const safeMime = /^[a-z0-9][a-z0-9.+-]*\/[a-z0-9][a-z0-9.+-]*$/i.test(mime)
|
||||
? mime
|
||||
: "application/octet-stream";
|
||||
const fileName = `audio.${format.replace(/[^a-z0-9]/g, "") || "wav"}`;
|
||||
const boundary = `----OmniRouteAudioBridge${randomUUID().replace(/-/g, "")}`;
|
||||
const CRLF = "\r\n";
|
||||
const multipartBody = Buffer.concat([
|
||||
Buffer.from(
|
||||
`--${boundary}${CRLF}` +
|
||||
`Content-Disposition: form-data; name="file"; filename="${fileName}"${CRLF}` +
|
||||
`Content-Type: ${safeMime}${CRLF}${CRLF}`
|
||||
),
|
||||
const { body: multipartBody, boundary } = buildTranscriptionMultipartBody(
|
||||
bytes,
|
||||
Buffer.from(
|
||||
`${CRLF}--${boundary}${CRLF}` +
|
||||
`Content-Disposition: form-data; name="model"${CRLF}${CRLF}` +
|
||||
`${config.model}${CRLF}--${boundary}--${CRLF}`
|
||||
),
|
||||
]);
|
||||
format,
|
||||
safeMime,
|
||||
config.model,
|
||||
extraFields
|
||||
);
|
||||
|
||||
const port = (deps.getPort ?? (() => getRuntimePorts().port))();
|
||||
const bearer = (deps.getBearer ?? resolveSelfLoopBearer)();
|
||||
@@ -241,11 +300,11 @@ export async function callAudioTranscription(
|
||||
if (!response.ok) {
|
||||
throw new Error(`Audio transcription failed (${response.status})`);
|
||||
}
|
||||
const data = (await response.json()) as { text?: unknown };
|
||||
const data = (await response.json()) as Record<string, unknown>;
|
||||
if (typeof data.text !== "string") {
|
||||
throw new Error("Audio transcription returned an invalid response");
|
||||
}
|
||||
return data.text.trim();
|
||||
return data;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new Error("Audio transcription timed out");
|
||||
@@ -255,3 +314,72 @@ export async function callAudioTranscription(
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send one audio part through OmniRoute's existing multipart transcription route. */
|
||||
export async function callAudioTranscription(
|
||||
part: AudioPart,
|
||||
config: AudioTranscriptionConfig,
|
||||
deps: AudioTranscriptionDependencies = {}
|
||||
): Promise<string> {
|
||||
const data = await sendAudioTranscriptionRequest(part, config, deps, {});
|
||||
return (data.text as string).trim();
|
||||
}
|
||||
|
||||
function normalizeTranscriptionSegment(raw: unknown): AudioTranscriptionSegment | null {
|
||||
if (!raw || typeof raw !== "object") return null;
|
||||
const segment = raw as Record<string, unknown>;
|
||||
const startSeconds =
|
||||
typeof segment.start === "number"
|
||||
? segment.start
|
||||
: typeof segment.startSeconds === "number"
|
||||
? segment.startSeconds
|
||||
: Number.NaN;
|
||||
const endSeconds =
|
||||
typeof segment.end === "number"
|
||||
? segment.end
|
||||
: typeof segment.endSeconds === "number"
|
||||
? segment.endSeconds
|
||||
: Number.NaN;
|
||||
const text = typeof segment.text === "string" ? segment.text.trim() : "";
|
||||
if (
|
||||
!text ||
|
||||
!Number.isFinite(startSeconds) ||
|
||||
!Number.isFinite(endSeconds) ||
|
||||
startSeconds < 0 ||
|
||||
endSeconds <= startSeconds
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const confidence =
|
||||
typeof segment.confidence === "number" && Number.isFinite(segment.confidence)
|
||||
? Math.min(1, Math.max(0, segment.confidence))
|
||||
: undefined;
|
||||
return {
|
||||
endSeconds,
|
||||
startSeconds,
|
||||
text,
|
||||
...(confidence === undefined ? {} : { confidence }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Same Audio Bridge transcription boundary as {@link callAudioTranscription},
|
||||
* requesting `verbose_json` so a provider that supports it can return
|
||||
* per-segment timing. Callers must treat a result with no `segments` as
|
||||
* coarse (whole-clip) timing only — the provider did not supply detail.
|
||||
*/
|
||||
export async function callAudioTranscriptionTimed(
|
||||
part: AudioPart,
|
||||
config: AudioTranscriptionConfig,
|
||||
deps: AudioTranscriptionDependencies = {}
|
||||
): Promise<AudioTranscriptionTimedResult> {
|
||||
const data = await sendAudioTranscriptionRequest(part, config, deps, {
|
||||
response_format: "verbose_json",
|
||||
});
|
||||
const text = (data.text as string).trim();
|
||||
const rawSegments = Array.isArray(data.segments) ? data.segments : [];
|
||||
const segments = rawSegments
|
||||
.map(normalizeTranscriptionSegment)
|
||||
.filter((segment): segment is AudioTranscriptionSegment => segment !== null);
|
||||
return segments.length > 0 ? { segments, text } : { text };
|
||||
}
|
||||
|
||||
124
src/lib/guardrails/videoBridgeAudioExtraction.ts
Normal file
124
src/lib/guardrails/videoBridgeAudioExtraction.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
/**
|
||||
* Video Bridge — bounded mono 16 kHz PCM WAV audio extraction (FU-06, #11654).
|
||||
*
|
||||
* Runs strictly local, array-argument `ffmpeg` (never a shell string — Hard
|
||||
* Rule #13) against a video file already probed by
|
||||
* {@link probeLocalVideo}, so the safe-container/format/dimension validation
|
||||
* that path already enforces for frame extraction stays the single source of
|
||||
* truth. This module owns only the audio-specific extraction step and its
|
||||
* own output byte cap; it does not duplicate probing.
|
||||
*/
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { probeLocalVideo, type VideoCommandRunner } from "./videoBridgeRuntime";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Bounded output — a 10-minute mono 16 kHz 16-bit WAV tops out well under this. */
|
||||
export const VIDEO_AUDIO_MAX_BYTES = 25 * 1024 * 1024;
|
||||
export const VIDEO_AUDIO_SAMPLE_RATE_HZ = 16_000;
|
||||
export const VIDEO_AUDIO_CHANNELS = 1;
|
||||
|
||||
export interface ExtractedVideoAudio {
|
||||
channels: number;
|
||||
dataUri: string;
|
||||
durationSeconds: number;
|
||||
sampleRateHz: number;
|
||||
}
|
||||
|
||||
export interface VideoAudioExtractionOptions {
|
||||
maxDurationSeconds: number;
|
||||
maxOutputBytes?: number;
|
||||
runner?: VideoCommandRunner;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
const defaultRunner: VideoCommandRunner = async (executable, args, options) => {
|
||||
const result = await execFileAsync(executable, [...args], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 1024 * 1024,
|
||||
signal: options.signal,
|
||||
timeout: options.timeoutMs,
|
||||
windowsHide: true,
|
||||
});
|
||||
return { stdout: String(result.stdout), stderr: String(result.stderr) };
|
||||
};
|
||||
|
||||
/** Extract the first audio stream of `bytes` as a bounded mono 16 kHz PCM WAV. */
|
||||
export async function extractVideoAudioFromBytes(
|
||||
bytes: Uint8Array,
|
||||
options: VideoAudioExtractionOptions
|
||||
): Promise<ExtractedVideoAudio> {
|
||||
const temporaryDirectory = await mkdtemp(join(tmpdir(), "omniroute-video-audio-broker-"));
|
||||
try {
|
||||
if (options.signal?.aborted) throw new Error("Video audio extraction request aborted");
|
||||
const runner = options.runner ?? defaultRunner;
|
||||
const inputPath = join(temporaryDirectory, "input.video");
|
||||
const outputPath = join(temporaryDirectory, "audio.wav");
|
||||
await writeFile(inputPath, bytes, { mode: 0o600 });
|
||||
// Reuses the frame path's container/format/dimension safety checks — this
|
||||
// module adds no parallel validation of its own.
|
||||
const metadata = await probeLocalVideo(inputPath, {
|
||||
maxDurationSeconds: options.maxDurationSeconds,
|
||||
runner,
|
||||
signal: options.signal,
|
||||
timeoutMs: Math.min(options.timeoutMs, 30_000),
|
||||
});
|
||||
await runner(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-nostdin",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-protocol_whitelist",
|
||||
"file",
|
||||
"-threads",
|
||||
"1",
|
||||
"-i",
|
||||
inputPath,
|
||||
"-vn",
|
||||
"-map",
|
||||
"0:a:0",
|
||||
"-ac",
|
||||
String(VIDEO_AUDIO_CHANNELS),
|
||||
"-ar",
|
||||
String(VIDEO_AUDIO_SAMPLE_RATE_HZ),
|
||||
"-sample_fmt",
|
||||
"s16",
|
||||
"-t",
|
||||
metadata.durationSeconds.toFixed(3),
|
||||
"-f",
|
||||
"wav",
|
||||
"-y",
|
||||
outputPath,
|
||||
],
|
||||
{ signal: options.signal, timeoutMs: options.timeoutMs }
|
||||
);
|
||||
const outputStats = await stat(outputPath);
|
||||
const maxOutputBytes = options.maxOutputBytes ?? VIDEO_AUDIO_MAX_BYTES;
|
||||
if (!outputStats.isFile() || outputStats.size < 1) {
|
||||
throw new Error("Video audio extraction produced no output");
|
||||
}
|
||||
if (outputStats.size > maxOutputBytes) {
|
||||
throw new Error("Extracted video audio byte limit exceeded");
|
||||
}
|
||||
const audioBytes = await readFile(outputPath);
|
||||
if (audioBytes.byteLength !== outputStats.size) {
|
||||
throw new Error("Extracted video audio changed before it could be read");
|
||||
}
|
||||
return {
|
||||
channels: VIDEO_AUDIO_CHANNELS,
|
||||
dataUri: `data:audio/wav;base64,${audioBytes.toString("base64")}`,
|
||||
durationSeconds: metadata.durationSeconds,
|
||||
sampleRateHz: VIDEO_AUDIO_SAMPLE_RATE_HZ,
|
||||
};
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
246
src/lib/guardrails/videoBridgeAudioOrchestration.ts
Normal file
246
src/lib/guardrails/videoBridgeAudioOrchestration.ts
Normal file
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Video Bridge — Audio Bridge STT orchestration (FU-06, #11654).
|
||||
*
|
||||
* Composes three already-independent seams behind one dual opt-in gate:
|
||||
* - the SAME video bytes the frame path already downloaded — this module
|
||||
* has no download path of its own, so it structurally cannot fetch the
|
||||
* video a second time;
|
||||
* - the loopback-only broker's bounded mono 16 kHz WAV operation
|
||||
* ({@link extractVideoAudioViaBroker}), which shares the frame path's
|
||||
* process queue, deadline, AbortSignal, and byte budgets because it is
|
||||
* the exact same broker route (`mode=audio` on the frame route);
|
||||
* - the existing Audio Bridge transcription boundary
|
||||
* ({@link callAudioTranscriptionTimed}) — no new STT client.
|
||||
*
|
||||
* "No paid transcription without dual opt-in" is enforced by two early
|
||||
* returns, both before the broker or the transcription boundary is ever
|
||||
* touched. Every failure path returns `track: null` instead of throwing, so
|
||||
* a caller can always fall back to a visual-only description.
|
||||
*/
|
||||
import {
|
||||
callAudioTranscriptionTimed,
|
||||
selectAudioBridgeModel,
|
||||
type AudioCredentialCheck,
|
||||
type AudioPart,
|
||||
type AudioTranscriptionTimedResult,
|
||||
} from "./audioBridgeHelpers";
|
||||
import { bridgeCacheKey, type BridgeCacheStore } from "./modalityBridge/bridgeCache";
|
||||
import type { FusionObservation, FusionTrack } from "./videoAudioFusion";
|
||||
import {
|
||||
extractVideoAudioViaBroker,
|
||||
type BrokerAudioExtractionResult,
|
||||
} from "./videoBridgeBrokerClient";
|
||||
|
||||
export type VideoAudioOrchestrationReason =
|
||||
| "ABORTED"
|
||||
| "EXTRACTION_FAILED"
|
||||
| "OPERATOR_OPT_OUT"
|
||||
| "PROVIDER_UNAVAILABLE"
|
||||
| "REQUEST_OPT_OUT"
|
||||
| "TIMEOUT"
|
||||
| "TRANSCRIPTION_FAILED";
|
||||
|
||||
export interface VideoAudioOrchestrationOptions {
|
||||
/** Result cache; omit (or omit `cacheKeyRef`) to skip caching entirely. */
|
||||
cache?: BridgeCacheStore | null;
|
||||
/** Stable identity for the ORIGINAL video part `ref` — never the raw bytes. */
|
||||
cacheKeyRef?: string;
|
||||
extractAudio?: typeof extractVideoAudioViaBroker;
|
||||
hasUsableCredentials?: AudioCredentialCheck;
|
||||
/** Configured STT model — `"auto"` or a fixed `provider/model` string. */
|
||||
model: string;
|
||||
/** Operator half of the dual opt-in (settings). */
|
||||
operatorOptIn: boolean;
|
||||
/** Request half of the dual opt-in (a per-request signal from the caller). */
|
||||
requestOptIn: boolean;
|
||||
selectModel?: typeof selectAudioBridgeModel;
|
||||
signal?: AbortSignal;
|
||||
/** The SAME budget the sibling frame extraction for this video used. */
|
||||
timeoutMs: number;
|
||||
transcribe?: typeof callAudioTranscriptionTimed;
|
||||
/** The SAME already-downloaded video bytes used for frame extraction. */
|
||||
videoBytes: Uint8Array;
|
||||
}
|
||||
|
||||
export interface VideoAudioOrchestrationResult {
|
||||
/** False only for the two opt-out reasons — nothing was attempted at all. */
|
||||
attempted: boolean;
|
||||
reason?: VideoAudioOrchestrationReason;
|
||||
sttModel: string | null;
|
||||
/** "coarse" means the provider gave no segment timing (whole-clip span only). */
|
||||
timingPrecision?: "coarse" | "exact";
|
||||
track: FusionTrack | null;
|
||||
}
|
||||
|
||||
interface CachedOrchestration {
|
||||
timingPrecision: "coarse" | "exact";
|
||||
track: FusionTrack;
|
||||
}
|
||||
|
||||
type StepOutcome<T> = { ok: true; value: T } | { ok: false; result: VideoAudioOrchestrationResult };
|
||||
|
||||
function optedOut(reason: "OPERATOR_OPT_OUT" | "REQUEST_OPT_OUT"): VideoAudioOrchestrationResult {
|
||||
return { attempted: false, reason, sttModel: null, track: null };
|
||||
}
|
||||
|
||||
function failed(
|
||||
reason: Exclude<VideoAudioOrchestrationReason, "OPERATOR_OPT_OUT" | "REQUEST_OPT_OUT">,
|
||||
sttModel: string | null
|
||||
): VideoAudioOrchestrationResult {
|
||||
return { attempted: true, reason, sttModel, track: null };
|
||||
}
|
||||
|
||||
/** Caller-signal cancellation always wins; else a timeout-shaped message is a shared-budget timeout. */
|
||||
function classifyStepFailure(
|
||||
error: unknown,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutPattern: RegExp
|
||||
): "ABORTED" | "TIMEOUT" | "FAILED" {
|
||||
if (signal?.aborted) return "ABORTED";
|
||||
if (error instanceof Error && timeoutPattern.test(error.message)) return "TIMEOUT";
|
||||
return "FAILED";
|
||||
}
|
||||
|
||||
function cacheKeyFor(ref: string, sttModel: string): string {
|
||||
return bridgeCacheKey(ref, "video-audio-transcription", sttModel);
|
||||
}
|
||||
|
||||
function readCache(
|
||||
options: VideoAudioOrchestrationOptions,
|
||||
sttModel: string
|
||||
): CachedOrchestration | null {
|
||||
if (!options.cache || !options.cacheKeyRef) return null;
|
||||
const entry = options.cache.getEntry(cacheKeyFor(options.cacheKeyRef, sttModel));
|
||||
if (!entry) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(entry.value) as Partial<CachedOrchestration>;
|
||||
if (!parsed.track || !Array.isArray(parsed.track.observations)) return null;
|
||||
if (parsed.timingPrecision !== "coarse" && parsed.timingPrecision !== "exact") return null;
|
||||
return { timingPrecision: parsed.timingPrecision, track: parsed.track };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function writeCache(
|
||||
options: VideoAudioOrchestrationOptions,
|
||||
sttModel: string,
|
||||
value: CachedOrchestration
|
||||
): void {
|
||||
if (!options.cache || !options.cacheKeyRef) return;
|
||||
options.cache.setEntry(cacheKeyFor(options.cacheKeyRef, sttModel), {
|
||||
value: JSON.stringify(value),
|
||||
});
|
||||
}
|
||||
|
||||
async function runExtraction(
|
||||
options: VideoAudioOrchestrationOptions,
|
||||
sttModel: string
|
||||
): Promise<StepOutcome<BrokerAudioExtractionResult>> {
|
||||
const extractAudio = options.extractAudio ?? extractVideoAudioViaBroker;
|
||||
try {
|
||||
const value = await extractAudio(options.videoBytes, {
|
||||
signal: options.signal,
|
||||
timeoutMs: options.timeoutMs,
|
||||
});
|
||||
return { ok: true, value };
|
||||
} catch (error) {
|
||||
const outcome = classifyStepFailure(error, options.signal, /abort|timed out/i);
|
||||
return {
|
||||
ok: false,
|
||||
result: failed(outcome === "FAILED" ? "EXTRACTION_FAILED" : outcome, sttModel),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function runTranscription(
|
||||
options: VideoAudioOrchestrationOptions,
|
||||
sttModel: string,
|
||||
extraction: BrokerAudioExtractionResult
|
||||
): Promise<StepOutcome<AudioTranscriptionTimedResult>> {
|
||||
const transcribe = options.transcribe ?? callAudioTranscriptionTimed;
|
||||
const audioPart: AudioPart = {
|
||||
format: "wav",
|
||||
messageIndex: -1,
|
||||
partIndex: -1,
|
||||
ref: extraction.audio.dataUri,
|
||||
shape: "input_audio",
|
||||
};
|
||||
try {
|
||||
const value = await transcribe(audioPart, { model: sttModel, timeoutMs: options.timeoutMs });
|
||||
return { ok: true, value };
|
||||
} catch (error) {
|
||||
const outcome = classifyStepFailure(error, options.signal, /timed out/i);
|
||||
return {
|
||||
ok: false,
|
||||
result: failed(outcome === "FAILED" ? "TRANSCRIPTION_FAILED" : outcome, sttModel),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Server-minted observations only — never trusts a caller-declared source. */
|
||||
function buildObservations(
|
||||
extraction: BrokerAudioExtractionResult,
|
||||
transcription: AudioTranscriptionTimedResult
|
||||
): { observations: FusionObservation[]; timingPrecision: "coarse" | "exact" } {
|
||||
if (transcription.segments && transcription.segments.length > 0) {
|
||||
return {
|
||||
observations: transcription.segments.map((segment) => ({
|
||||
confidence: segment.confidence ?? 1,
|
||||
endSeconds: segment.endSeconds,
|
||||
source: "audio",
|
||||
startSeconds: segment.startSeconds,
|
||||
text: segment.text,
|
||||
})),
|
||||
timingPrecision: "exact",
|
||||
};
|
||||
}
|
||||
const text = transcription.text.trim();
|
||||
if (!text) return { observations: [], timingPrecision: "coarse" };
|
||||
return {
|
||||
observations: [
|
||||
{
|
||||
confidence: 1,
|
||||
endSeconds: Math.max(extraction.durationSeconds, 0.001),
|
||||
source: "audio",
|
||||
startSeconds: 0,
|
||||
text,
|
||||
},
|
||||
],
|
||||
timingPrecision: "coarse",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrate server-side Video Bridge audio extraction + Audio Bridge STT,
|
||||
* gated on BOTH the operator and the request opt-in. Never throws: every
|
||||
* outcome — opt-out, no provider, extraction/transcription failure, timeout,
|
||||
* abort — comes back as `track: null` so the caller's visual-only fallback
|
||||
* always has a well-defined result to check against.
|
||||
*/
|
||||
export async function orchestrateVideoAudioTranscription(
|
||||
options: VideoAudioOrchestrationOptions
|
||||
): Promise<VideoAudioOrchestrationResult> {
|
||||
if (!options.operatorOptIn) return optedOut("OPERATOR_OPT_OUT");
|
||||
if (!options.requestOptIn) return optedOut("REQUEST_OPT_OUT");
|
||||
|
||||
const selectModel = options.selectModel ?? selectAudioBridgeModel;
|
||||
const sttModel = await selectModel(options.model, options.hasUsableCredentials);
|
||||
if (!sttModel) return failed("PROVIDER_UNAVAILABLE", null);
|
||||
|
||||
const cached = readCache(options, sttModel);
|
||||
if (cached) {
|
||||
return { attempted: true, sttModel, timingPrecision: cached.timingPrecision, track: cached.track };
|
||||
}
|
||||
|
||||
const extraction = await runExtraction(options, sttModel);
|
||||
if (!extraction.ok) return extraction.result;
|
||||
|
||||
const transcription = await runTranscription(options, sttModel, extraction.value);
|
||||
if (!transcription.ok) return transcription.result;
|
||||
|
||||
const { observations, timingPrecision } = buildObservations(extraction.value, transcription.value);
|
||||
const track: FusionTrack = { observations };
|
||||
writeCache(options, sttModel, { timingPrecision, track });
|
||||
return { attempted: true, sttModel, timingPrecision, track };
|
||||
}
|
||||
@@ -218,6 +218,17 @@ export interface BrokerSubtitleExtractionOptions {
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
export interface BrokerAudioExtractionResult {
|
||||
audio: { channels: number; dataUri: string; sampleRateHz: number };
|
||||
durationSeconds: number;
|
||||
}
|
||||
|
||||
export interface BrokerAudioExtractionOptions {
|
||||
maxDurationSeconds?: number;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Requests a bounded, allowlisted-codec subtitle probe from the loopback broker. Only the
|
||||
* transport + envelope shape are validated here (safe container metadata, bounded stream
|
||||
@@ -267,3 +278,71 @@ export async function extractVideoSubtitlesViaBroker(
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function parseBrokerAudioResult(value: unknown): BrokerAudioExtractionResult {
|
||||
const record = value && typeof value === "object" ? (value as Record<string, unknown>) : null;
|
||||
const durationSeconds = Number(record?.durationSeconds);
|
||||
const audio =
|
||||
record?.audio && typeof record.audio === "object"
|
||||
? (record.audio as Record<string, unknown>)
|
||||
: null;
|
||||
const dataUri = typeof audio?.dataUri === "string" ? audio.dataUri : "";
|
||||
const sampleRateHz = Number(audio?.sampleRateHz);
|
||||
const channels = Number(audio?.channels);
|
||||
if (
|
||||
!Number.isFinite(durationSeconds) ||
|
||||
durationSeconds <= 0 ||
|
||||
!/^data:audio\/wav;base64,[A-Za-z0-9+/=]+$/.test(dataUri) ||
|
||||
!Number.isInteger(sampleRateHz) ||
|
||||
sampleRateHz <= 0 ||
|
||||
!Number.isInteger(channels) ||
|
||||
channels <= 0
|
||||
) {
|
||||
throw new Error("Video audio extraction broker returned invalid metadata");
|
||||
}
|
||||
return { audio: { channels, dataUri, sampleRateHz }, durationSeconds };
|
||||
}
|
||||
|
||||
/**
|
||||
* The loopback-only broker's mono 16 kHz PCM WAV extraction operation. Shares
|
||||
* the exact route, queue, deadline, and byte budgets as
|
||||
* {@link extractVideoFramesViaBroker} — only the `mode=audio` query flag and
|
||||
* response shape differ.
|
||||
*/
|
||||
export async function extractVideoAudioViaBroker(
|
||||
bytes: Uint8Array,
|
||||
options: BrokerAudioExtractionOptions,
|
||||
dependencies: { fetchImpl?: typeof fetch; maxResponseBytes?: number } = {}
|
||||
): Promise<BrokerAudioExtractionResult> {
|
||||
if (options.signal?.aborted) throw new Error("Video audio extraction request aborted");
|
||||
const baseUrl = resolveVideoBridgeBrokerBaseUrl();
|
||||
const url = new URL(`${baseUrl}${VIDEO_BRIDGE_BROKER_PATH}`);
|
||||
url.searchParams.set("mode", "audio");
|
||||
const fetchImpl = dependencies.fetchImpl ?? fetchModelSyncInternal;
|
||||
const timeoutSignal = AbortSignal.timeout(options.timeoutMs);
|
||||
const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetchImpl(url, {
|
||||
method: "POST",
|
||||
body: Buffer.from(bytes),
|
||||
headers: {
|
||||
"Content-Type": "application/octet-stream",
|
||||
...buildVideoBridgeBrokerHeaders(),
|
||||
},
|
||||
redirect: "error",
|
||||
signal,
|
||||
});
|
||||
} catch {
|
||||
if (signal.aborted) throw new Error("Video audio extraction request aborted");
|
||||
throw new Error("Video extraction broker is unavailable");
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(`Video extraction broker failed (${response.status})`);
|
||||
}
|
||||
const maxResponseBytes = Math.min(
|
||||
MAX_BROKER_RESPONSE_BYTES,
|
||||
dependencies.maxResponseBytes ?? MAX_BROKER_RESPONSE_BYTES
|
||||
);
|
||||
return parseBrokerAudioResult(await readBoundedResponse(response, maxResponseBytes));
|
||||
}
|
||||
|
||||
@@ -33,6 +33,12 @@ export const MODALITY_BRIDGE_DEFAULTS = {
|
||||
videoSamplingPolicy: "uniform" as VideoSamplingPolicy,
|
||||
videoMaxVideos: 1,
|
||||
videoTimeoutMs: 120000,
|
||||
// Server-orchestrated Audio Bridge STT over Video Bridge audio extraction
|
||||
// (FU-06, #11654) spends provider credit on the operator's behalf, so it
|
||||
// stays OFF by default — Hard Rule #20's "never spend by default" spirit.
|
||||
// Every transcription attempt additionally requires a per-request opt-in;
|
||||
// this flag alone never triggers a call.
|
||||
videoAudioTranscriptionEnabled: false,
|
||||
} as const;
|
||||
|
||||
export interface VisionBridgeRuntimeSettings {
|
||||
@@ -59,6 +65,11 @@ export interface AudioBridgeRuntimeSettings {
|
||||
cacheMaxEntries: number;
|
||||
}
|
||||
|
||||
export interface VideoAudioTranscriptionRuntimeSettings {
|
||||
/** Operator opt-in only — a request still needs its own opt-in (FU-06, #11654). */
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface VideoBridgeRuntimeSettings {
|
||||
enabled: boolean;
|
||||
model: string;
|
||||
@@ -142,6 +153,22 @@ export function resolveAudioBridgeRuntimeSettings(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the operator half of the FU-06 dual opt-in for Video Bridge audio
|
||||
* transcription. The request-side opt-in is a separate, per-request signal —
|
||||
* this settings flag alone never authorizes a transcription call.
|
||||
*/
|
||||
export function resolveVideoAudioTranscriptionRuntimeSettings(
|
||||
settings: Record<string, unknown> | null | undefined
|
||||
): VideoAudioTranscriptionRuntimeSettings {
|
||||
const s = settings ?? {};
|
||||
return {
|
||||
enabled:
|
||||
pickBoolean(s.modalityBridgeVideoAudioTranscriptionEnabled) ??
|
||||
MODALITY_BRIDGE_DEFAULTS.videoAudioTranscriptionEnabled,
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolve persisted Video Bridge settings with safe, bounded defaults. */
|
||||
export function resolveVideoBridgeRuntimeSettings(
|
||||
settings: Record<string, unknown> | null | undefined
|
||||
|
||||
@@ -464,6 +464,10 @@ export const updateSettingsSchema = z.object({
|
||||
.min(VIDEO_BRIDGE_TIMEOUT_MIN_MS)
|
||||
.max(VIDEO_BRIDGE_TIMEOUT_MAX_MS)
|
||||
.optional(),
|
||||
// Operator half of the FU-06 dual opt-in (#11654) for server-orchestrated
|
||||
// Audio Bridge STT over Video Bridge audio extraction — defaults false
|
||||
// (Hard Rule #20). A request-side opt-in is required in addition to this.
|
||||
modalityBridgeVideoAudioTranscriptionEnabled: z.boolean().optional(),
|
||||
modalityBridgeCacheEnabled: z.boolean().optional(),
|
||||
modalityBridgeCacheTtlMinutes: z.number().int().min(1).max(1440).optional(),
|
||||
modalityBridgeCacheMaxEntries: z.number().int().min(10).max(5000).optional(),
|
||||
|
||||
@@ -3,6 +3,7 @@ import test from "node:test";
|
||||
|
||||
import {
|
||||
callAudioTranscription,
|
||||
callAudioTranscriptionTimed,
|
||||
extractAudioParts,
|
||||
replaceAudioParts,
|
||||
selectAudioBridgeModel,
|
||||
@@ -217,3 +218,99 @@ test("remote audio_url uses the guarded remote fetch before self-loop upload", a
|
||||
assert.equal(fetchedUrl, "https://media.example.test/clip.ogg");
|
||||
assertMultipartFile(uploaded, "audio.ogg", "audio/ogg", Buffer.from("OggS remote audio"));
|
||||
});
|
||||
|
||||
test("timed transcription requests verbose_json and preserves provider-supplied segment timing", async () => {
|
||||
let uploaded: RequestInit | undefined;
|
||||
const result = await callAudioTranscriptionTimed(
|
||||
{
|
||||
messageIndex: 0,
|
||||
partIndex: 0,
|
||||
ref: Buffer.from("RIFF timed audio").toString("base64"),
|
||||
shape: "input_audio",
|
||||
format: "wav",
|
||||
},
|
||||
{ model: "deepgram/nova-3", timeoutMs: 1_000 },
|
||||
{
|
||||
fetchImpl: async (_input, init) => {
|
||||
uploaded = init;
|
||||
return Response.json({
|
||||
text: "hello world",
|
||||
segments: [
|
||||
{ start: 0, end: 1.2, text: "hello", confidence: 0.95 },
|
||||
{ start: 1.2, end: 2.4, text: "world" },
|
||||
],
|
||||
});
|
||||
},
|
||||
getPort: () => 3210,
|
||||
getBearer: () => "internal-test-key",
|
||||
}
|
||||
);
|
||||
|
||||
const contentType = new Headers(uploaded?.headers).get("content-type");
|
||||
const boundary = contentType?.split("boundary=", 2)[1];
|
||||
const body = uploaded?.body as Buffer;
|
||||
assert.ok(
|
||||
body.includes(
|
||||
Buffer.from(
|
||||
`--${boundary}\r\n` +
|
||||
'Content-Disposition: form-data; name="response_format"\r\n\r\n' +
|
||||
`verbose_json\r\n`
|
||||
)
|
||||
),
|
||||
"verbose_json must be requested from the same transcription boundary"
|
||||
);
|
||||
assert.equal(result.text, "hello world");
|
||||
assert.deepEqual(result.segments, [
|
||||
{ startSeconds: 0, endSeconds: 1.2, text: "hello", confidence: 0.95 },
|
||||
{ startSeconds: 1.2, endSeconds: 2.4, text: "world" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("timed transcription reports no segments when the provider ignores verbose_json", async () => {
|
||||
const result = await callAudioTranscriptionTimed(
|
||||
{
|
||||
messageIndex: 0,
|
||||
partIndex: 0,
|
||||
ref: Buffer.from("RIFF coarse audio").toString("base64"),
|
||||
shape: "input_audio",
|
||||
format: "wav",
|
||||
},
|
||||
{ model: "deepgram/nova-3", timeoutMs: 1_000 },
|
||||
{
|
||||
fetchImpl: async () => Response.json({ text: "coarse only" }),
|
||||
getPort: () => 3210,
|
||||
getBearer: () => "internal-test-key",
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(result.text, "coarse only");
|
||||
assert.equal(result.segments, undefined);
|
||||
});
|
||||
|
||||
test("timed transcription drops malformed provider segments instead of trusting them", async () => {
|
||||
const result = await callAudioTranscriptionTimed(
|
||||
{
|
||||
messageIndex: 0,
|
||||
partIndex: 0,
|
||||
ref: Buffer.from("RIFF bad segments").toString("base64"),
|
||||
shape: "input_audio",
|
||||
format: "wav",
|
||||
},
|
||||
{ model: "deepgram/nova-3", timeoutMs: 1_000 },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
text: "text",
|
||||
segments: [
|
||||
{ start: 5, end: 2, text: "backwards" },
|
||||
{ start: 0, end: 1, text: "" },
|
||||
{ start: "nope", end: 1, text: "not a number" },
|
||||
],
|
||||
}),
|
||||
getPort: () => 3210,
|
||||
getBearer: () => "internal-test-key",
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(result.segments, undefined);
|
||||
});
|
||||
|
||||
140
tests/unit/guardrails/videoBridgeAudioExtraction.test.ts
Normal file
140
tests/unit/guardrails/videoBridgeAudioExtraction.test.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
VIDEO_AUDIO_CHANNELS,
|
||||
VIDEO_AUDIO_SAMPLE_RATE_HZ,
|
||||
extractVideoAudioFromBytes,
|
||||
} from "../../../src/lib/guardrails/videoBridgeAudioExtraction.ts";
|
||||
import type { VideoCommandRunner } from "../../../src/lib/guardrails/videoBridgeRuntime.ts";
|
||||
|
||||
test("extracts bounded mono 16kHz WAV bytes and reuses the probed duration", async () => {
|
||||
const calls: Array<{ executable: string; args: string[] }> = [];
|
||||
const runner: VideoCommandRunner = async (executable, args) => {
|
||||
calls.push({ executable, args: [...args] });
|
||||
if (executable === "ffprobe") {
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
format: { duration: "4.5", format_name: "mp4" },
|
||||
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
|
||||
}),
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
await writeFile(args.at(-1) ?? "", Buffer.from("RIFF....WAVEfmt "));
|
||||
return { stdout: "", stderr: "" };
|
||||
};
|
||||
|
||||
const result = await extractVideoAudioFromBytes(Buffer.from("fake video bytes"), {
|
||||
maxDurationSeconds: 600,
|
||||
runner,
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
assert.equal(result.durationSeconds, 4.5);
|
||||
assert.equal(result.channels, VIDEO_AUDIO_CHANNELS);
|
||||
assert.equal(result.sampleRateHz, VIDEO_AUDIO_SAMPLE_RATE_HZ);
|
||||
assert.match(result.dataUri, /^data:audio\/wav;base64,/);
|
||||
assert.deepEqual(
|
||||
Buffer.from(result.dataUri.split(",", 2)[1], "base64"),
|
||||
Buffer.from("RIFF....WAVEfmt ")
|
||||
);
|
||||
|
||||
const ffmpegCall = calls.find((call) => call.executable === "ffmpeg");
|
||||
assert.ok(ffmpegCall, "ffmpeg must be invoked");
|
||||
assert.deepEqual(ffmpegCall!.args.slice(ffmpegCall!.args.indexOf("-map"), ffmpegCall!.args.indexOf("-map") + 2), [
|
||||
"-map",
|
||||
"0:a:0",
|
||||
]);
|
||||
assert.deepEqual(ffmpegCall!.args.slice(ffmpegCall!.args.indexOf("-ac"), ffmpegCall!.args.indexOf("-ac") + 2), [
|
||||
"-ac",
|
||||
"1",
|
||||
]);
|
||||
assert.deepEqual(ffmpegCall!.args.slice(ffmpegCall!.args.indexOf("-ar"), ffmpegCall!.args.indexOf("-ar") + 2), [
|
||||
"-ar",
|
||||
"16000",
|
||||
]);
|
||||
// No shell string ever built — every arg is a discrete array element (Hard Rule #13).
|
||||
assert.equal(
|
||||
ffmpegCall!.args.some((arg) => arg.includes(";") || arg.includes("&&")),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("propagates the shared probe's duration cap instead of re-validating it", async () => {
|
||||
const runner: VideoCommandRunner = async (executable) => {
|
||||
if (executable === "ffprobe") {
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
format: { duration: "700", format_name: "mp4" },
|
||||
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
|
||||
}),
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
throw new Error("ffmpeg must not run once the shared duration cap already rejected the file");
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
extractVideoAudioFromBytes(Buffer.from("fake video bytes"), {
|
||||
maxDurationSeconds: 600,
|
||||
runner,
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
/maximum duration/
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects a WAV output that exceeds the configured byte cap", async () => {
|
||||
const runner: VideoCommandRunner = async (executable, args) => {
|
||||
if (executable === "ffprobe") {
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
format: { duration: "4", format_name: "mp4" },
|
||||
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
|
||||
}),
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
await writeFile(args.at(-1) ?? "", Buffer.alloc(16));
|
||||
return { stdout: "", stderr: "" };
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
extractVideoAudioFromBytes(Buffer.from("fake video bytes"), {
|
||||
maxDurationSeconds: 600,
|
||||
maxOutputBytes: 8,
|
||||
runner,
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
/byte limit exceeded/
|
||||
);
|
||||
});
|
||||
|
||||
test("propagates ffmpeg's own failure when the container has no audio stream", async () => {
|
||||
const runner: VideoCommandRunner = async (executable) => {
|
||||
if (executable === "ffprobe") {
|
||||
return {
|
||||
stdout: JSON.stringify({
|
||||
format: { duration: "4", format_name: "mp4" },
|
||||
streams: [{ index: 0, codec_type: "video", width: 640, height: 360 }],
|
||||
}),
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
throw new Error("Stream map '0:a:0' matches no streams");
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
extractVideoAudioFromBytes(Buffer.from("fake video bytes"), {
|
||||
maxDurationSeconds: 600,
|
||||
runner,
|
||||
timeoutMs: 5_000,
|
||||
}),
|
||||
/matches no streams/
|
||||
);
|
||||
});
|
||||
381
tests/unit/guardrails/videoBridgeAudioOrchestration.test.ts
Normal file
381
tests/unit/guardrails/videoBridgeAudioOrchestration.test.ts
Normal file
@@ -0,0 +1,381 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { BridgeCache } from "../../../src/lib/guardrails/modalityBridge/bridgeCache.ts";
|
||||
import { orchestrateVideoAudioTranscription } from "../../../src/lib/guardrails/videoBridgeAudioOrchestration.ts";
|
||||
|
||||
const VIDEO_BYTES = Buffer.from("fake video bytes");
|
||||
|
||||
function baseOptions(overrides: Partial<Parameters<typeof orchestrateVideoAudioTranscription>[0]> = {}) {
|
||||
return {
|
||||
extractAudio: async () => {
|
||||
throw new Error("extractAudio must not be called in this test");
|
||||
},
|
||||
hasUsableCredentials: async () => true,
|
||||
model: "deepgram/nova-3",
|
||||
operatorOptIn: true,
|
||||
requestOptIn: true,
|
||||
timeoutMs: 5_000,
|
||||
transcribe: async () => {
|
||||
throw new Error("transcribe must not be called in this test");
|
||||
},
|
||||
videoBytes: VIDEO_BYTES,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function extraction(overrides: Partial<{ durationSeconds: number; dataUri: string }> = {}) {
|
||||
return {
|
||||
audio: {
|
||||
channels: 1,
|
||||
dataUri: overrides.dataUri ?? "data:audio/wav;base64,UklGRg==",
|
||||
sampleRateHz: 16_000,
|
||||
},
|
||||
durationSeconds: overrides.durationSeconds ?? 4,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Dual opt-in permutations -------------------------------------------------
|
||||
|
||||
test("neither opt-in: no extraction and no transcription call happens", async () => {
|
||||
let extractCalled = false;
|
||||
let transcribeCalled = false;
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
operatorOptIn: false,
|
||||
requestOptIn: false,
|
||||
extractAudio: async () => {
|
||||
extractCalled = true;
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => {
|
||||
transcribeCalled = true;
|
||||
return { text: "should never happen" };
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, false);
|
||||
assert.equal(result.reason, "OPERATOR_OPT_OUT");
|
||||
assert.equal(result.track, null);
|
||||
assert.equal(extractCalled, false);
|
||||
assert.equal(transcribeCalled, false);
|
||||
});
|
||||
|
||||
test("operator opt-in only: request opt-out still blocks every call", async () => {
|
||||
let extractCalled = false;
|
||||
let transcribeCalled = false;
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
operatorOptIn: true,
|
||||
requestOptIn: false,
|
||||
extractAudio: async () => {
|
||||
extractCalled = true;
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => {
|
||||
transcribeCalled = true;
|
||||
return { text: "should never happen" };
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, false);
|
||||
assert.equal(result.reason, "REQUEST_OPT_OUT");
|
||||
assert.equal(extractCalled, false);
|
||||
assert.equal(transcribeCalled, false);
|
||||
});
|
||||
|
||||
test("request opt-in only: operator opt-out still blocks every call", async () => {
|
||||
let extractCalled = false;
|
||||
let transcribeCalled = false;
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
operatorOptIn: false,
|
||||
requestOptIn: true,
|
||||
extractAudio: async () => {
|
||||
extractCalled = true;
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => {
|
||||
transcribeCalled = true;
|
||||
return { text: "should never happen" };
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, false);
|
||||
assert.equal(result.reason, "OPERATOR_OPT_OUT");
|
||||
assert.equal(extractCalled, false);
|
||||
assert.equal(transcribeCalled, false);
|
||||
});
|
||||
|
||||
test("both opt-ins present: this is the only permutation that transcribes", async () => {
|
||||
let extractCalled = false;
|
||||
let transcribeCalled = false;
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
operatorOptIn: true,
|
||||
requestOptIn: true,
|
||||
extractAudio: async (bytes) => {
|
||||
extractCalled = true;
|
||||
assert.equal(bytes, VIDEO_BYTES, "must reuse the exact same already-downloaded bytes");
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => {
|
||||
transcribeCalled = true;
|
||||
return { text: "hello world" };
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, true);
|
||||
assert.equal(extractCalled, true);
|
||||
assert.equal(transcribeCalled, true);
|
||||
assert.ok(result.track);
|
||||
assert.equal(result.track?.observations[0]?.text, "hello world");
|
||||
});
|
||||
|
||||
// --- Core scenarios ------------------------------------------------------------
|
||||
|
||||
test("audio extraction success feeds STT and returns a source-owned observation", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => extraction({ durationSeconds: 6 }),
|
||||
transcribe: async () => ({ text: "spoken words" }),
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, true);
|
||||
assert.equal(result.sttModel, "deepgram/nova-3");
|
||||
assert.equal(result.timingPrecision, "coarse");
|
||||
assert.deepEqual(result.track, {
|
||||
observations: [
|
||||
{ confidence: 1, endSeconds: 6, source: "audio", startSeconds: 0, text: "spoken words" },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("STT success with provider segments preserves exact per-segment timing", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => extraction({ durationSeconds: 10 }),
|
||||
transcribe: async () => ({
|
||||
segments: [
|
||||
{ confidence: 0.9, endSeconds: 2, startSeconds: 0, text: "hello" },
|
||||
{ endSeconds: 4, startSeconds: 2, text: "world" },
|
||||
],
|
||||
text: "hello world",
|
||||
}),
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.timingPrecision, "exact");
|
||||
assert.deepEqual(result.track?.observations, [
|
||||
{ confidence: 0.9, endSeconds: 2, source: "audio", startSeconds: 0, text: "hello" },
|
||||
{ confidence: 1, endSeconds: 4, source: "audio", startSeconds: 2, text: "world" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("unavailable STT provider: extraction never runs and the result is a visual-only-safe partial", async () => {
|
||||
let extractCalled = false;
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
hasUsableCredentials: async () => false,
|
||||
extractAudio: async () => {
|
||||
extractCalled = true;
|
||||
return extraction();
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.attempted, true);
|
||||
assert.equal(result.reason, "PROVIDER_UNAVAILABLE");
|
||||
assert.equal(result.sttModel, null);
|
||||
assert.equal(result.track, null);
|
||||
assert.equal(extractCalled, false, "no broker call once no provider is usable");
|
||||
});
|
||||
|
||||
test("extraction timeout is reported distinctly from a generic extraction failure", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => {
|
||||
throw new Error("Video audio extraction request aborted");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "TIMEOUT");
|
||||
assert.equal(result.track, null);
|
||||
});
|
||||
|
||||
test("caller abort during extraction is reported as ABORTED, not TIMEOUT", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
signal: controller.signal,
|
||||
extractAudio: async () => {
|
||||
throw new Error("Video audio extraction request aborted");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "ABORTED");
|
||||
});
|
||||
|
||||
test("caller abort during transcription is reported as ABORTED", async () => {
|
||||
const controller = new AbortController();
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
signal: controller.signal,
|
||||
extractAudio: async () => extraction(),
|
||||
transcribe: async () => {
|
||||
controller.abort();
|
||||
throw new Error("Audio transcription timed out");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "ABORTED");
|
||||
});
|
||||
|
||||
test("transcription-boundary timeout is reported distinctly from a generic transcription failure", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => extraction(),
|
||||
transcribe: async () => {
|
||||
throw new Error("Audio transcription timed out");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "TIMEOUT");
|
||||
});
|
||||
|
||||
test("a generic transcription failure is reported as TRANSCRIPTION_FAILED, not confused with a timeout", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => extraction(),
|
||||
transcribe: async () => {
|
||||
throw new Error("Audio transcription failed (500)");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "TRANSCRIPTION_FAILED");
|
||||
});
|
||||
|
||||
test("a generic extraction failure (no audio track) is reported as EXTRACTION_FAILED", async () => {
|
||||
const result = await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => {
|
||||
throw new Error("Stream map '0:a:0' matches no streams");
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(result.reason, "EXTRACTION_FAILED");
|
||||
});
|
||||
|
||||
test("every failure path stays a well-defined partial — never throws, always track: null", async () => {
|
||||
for (const scenario of [
|
||||
() =>
|
||||
orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
})
|
||||
),
|
||||
() =>
|
||||
orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
extractAudio: async () => extraction(),
|
||||
transcribe: async () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
})
|
||||
),
|
||||
() => orchestrateVideoAudioTranscription(baseOptions({ hasUsableCredentials: async () => false })),
|
||||
]) {
|
||||
const result = await scenario();
|
||||
assert.equal(result.track, null, "visual-only fallback must remain available");
|
||||
}
|
||||
});
|
||||
|
||||
// --- Cache -----------------------------------------------------------------
|
||||
|
||||
test("a cache hit skips both the broker call and the transcription boundary", async () => {
|
||||
const cache = new BridgeCache({ maxEntries: 10, ttlMs: 60_000 });
|
||||
let extractCalls = 0;
|
||||
let transcribeCalls = 0;
|
||||
const options = baseOptions({
|
||||
cache,
|
||||
cacheKeyRef: "video-ref-1",
|
||||
extractAudio: async () => {
|
||||
extractCalls += 1;
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => {
|
||||
transcribeCalls += 1;
|
||||
return { text: "cached transcript" };
|
||||
},
|
||||
});
|
||||
|
||||
const first = await orchestrateVideoAudioTranscription(options);
|
||||
assert.equal(extractCalls, 1);
|
||||
assert.equal(transcribeCalls, 1);
|
||||
|
||||
const second = await orchestrateVideoAudioTranscription(options);
|
||||
assert.equal(extractCalls, 1, "second call must not re-extract");
|
||||
assert.equal(transcribeCalls, 1, "second call must not re-transcribe — no repeat paid STT call");
|
||||
assert.deepEqual(second.track, first.track);
|
||||
assert.equal(second.attempted, true);
|
||||
});
|
||||
|
||||
test("a different cache key (different video) does not share another video's cache entry", async () => {
|
||||
const cache = new BridgeCache({ maxEntries: 10, ttlMs: 60_000 });
|
||||
let extractCalls = 0;
|
||||
const run = (cacheKeyRef: string) =>
|
||||
orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
cache,
|
||||
cacheKeyRef,
|
||||
extractAudio: async () => {
|
||||
extractCalls += 1;
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async () => ({ text: "distinct transcript" }),
|
||||
})
|
||||
);
|
||||
|
||||
await run("video-a");
|
||||
await run("video-b");
|
||||
assert.equal(extractCalls, 2, "each distinct video ref must be extracted independently");
|
||||
});
|
||||
|
||||
// --- Shared budgets ----------------------------------------------------------
|
||||
|
||||
test("the same timeoutMs and signal thread through both the extraction and transcription steps", async () => {
|
||||
const controller = new AbortController();
|
||||
const seen: Array<{ signal?: AbortSignal; timeoutMs: number }> = [];
|
||||
await orchestrateVideoAudioTranscription(
|
||||
baseOptions({
|
||||
signal: controller.signal,
|
||||
timeoutMs: 42_000,
|
||||
extractAudio: async (_bytes, options) => {
|
||||
seen.push({ signal: options.signal, timeoutMs: options.timeoutMs });
|
||||
return extraction();
|
||||
},
|
||||
transcribe: async (_part, config) => {
|
||||
seen.push({ timeoutMs: config.timeoutMs });
|
||||
return { text: "ok" };
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(seen[0].timeoutMs, 42_000);
|
||||
assert.equal(seen[0].signal, controller.signal);
|
||||
assert.equal(seen[1].timeoutMs, 42_000, "the transcription boundary must reuse the same shared budget");
|
||||
});
|
||||
@@ -5,12 +5,19 @@ import test from "node:test";
|
||||
import {
|
||||
VIDEO_BRIDGE_BROKER_PATH,
|
||||
buildVideoBridgeBrokerHeaders,
|
||||
extractVideoAudioViaBroker,
|
||||
extractVideoFramesViaBroker,
|
||||
isVideoBridgeBrokerInternalRequest,
|
||||
resolveVideoBridgeBrokerBaseUrl,
|
||||
} from "../../src/lib/guardrails/videoBridgeBrokerClient.ts";
|
||||
import { createVideoExtractionQueue } from "../../src/lib/guardrails/videoBridgeBrokerQueue.ts";
|
||||
import { AUTHZ_HEADER_PEER_LOCALITY } from "../../src/server/authz/headers.ts";
|
||||
import {
|
||||
BROKER_TIMEOUT_MS,
|
||||
handleVideoExtractionBrokerRequest,
|
||||
} from "../../src/app/api/modality-bridge/video/extract/route.ts";
|
||||
|
||||
const EXTRACT_PATH = "/api/modality-bridge/video/extract";
|
||||
|
||||
test("broker origin is pinned to the active loopback listener and ignores client-controlled origins", () => {
|
||||
const previousPort = process.env.PORT;
|
||||
@@ -211,6 +218,152 @@ test("broker queue bounds pending jobs and queued bytes", async () => {
|
||||
await Promise.all([active, pending]);
|
||||
});
|
||||
|
||||
test("audio broker client requests mode=audio on the exact same pinned route", async () => {
|
||||
let requestedUrl = "";
|
||||
let requestedInit: RequestInit | undefined;
|
||||
const response = await extractVideoAudioViaBroker(
|
||||
Buffer.from("safe-video"),
|
||||
{ timeoutMs: 5_000 },
|
||||
{
|
||||
fetchImpl: async (input, init) => {
|
||||
requestedUrl = String(input);
|
||||
requestedInit = init;
|
||||
return Response.json({
|
||||
audio: { channels: 1, dataUri: "data:audio/wav;base64,UklGRg==", sampleRateHz: 16000 },
|
||||
durationSeconds: 4,
|
||||
});
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
assert.match(requestedUrl, /\/api\/modality-bridge\/video\/extract\?mode=audio$/);
|
||||
assert.equal(new URL(requestedUrl).hostname, "127.0.0.1");
|
||||
assert.equal(requestedInit?.method, "POST");
|
||||
assert.deepEqual(Buffer.from(requestedInit?.body as Uint8Array), Buffer.from("safe-video"));
|
||||
assert.equal(response.durationSeconds, 4);
|
||||
assert.deepEqual(response.audio, {
|
||||
channels: 1,
|
||||
dataUri: "data:audio/wav;base64,UklGRg==",
|
||||
sampleRateHz: 16000,
|
||||
});
|
||||
});
|
||||
|
||||
test("audio broker client rejects a malformed or non-WAV response instead of trusting it", async () => {
|
||||
await assert.rejects(
|
||||
() =>
|
||||
extractVideoAudioViaBroker(
|
||||
Buffer.from("safe-video"),
|
||||
{ timeoutMs: 5_000 },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
audio: { channels: 1, dataUri: "data:image/jpeg;base64,QQ==", sampleRateHz: 16000 },
|
||||
durationSeconds: 4,
|
||||
}),
|
||||
}
|
||||
),
|
||||
/invalid metadata/
|
||||
);
|
||||
});
|
||||
|
||||
test("broker route enforces the exact same queue-capacity contract for the audio operation as for frames", async () => {
|
||||
// maxPending: 0 rejects every run() as capacity-exceeded from the first call —
|
||||
// this is the identical VideoExtractionQueueError path frames already relies on
|
||||
// (see "broker route maps queue capacity..." above), now exercised for mode=audio.
|
||||
const queue = createVideoExtractionQueue({ concurrency: 1, maxPending: 0, maxQueuedBytes: 100 });
|
||||
|
||||
const audioResponse = await handleVideoExtractionBrokerRequest(
|
||||
new Request(`http://localhost${EXTRACT_PATH}?mode=audio`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
...buildVideoBridgeBrokerHeaders(),
|
||||
[AUTHZ_HEADER_PEER_LOCALITY]: "loopback",
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
body: Buffer.from("video"),
|
||||
}),
|
||||
{
|
||||
queue,
|
||||
extractAudio: async () => {
|
||||
throw new Error("audio extractor must not run once the shared queue reports capacity");
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(audioResponse.status, 503);
|
||||
assert.equal(audioResponse.headers.get("Retry-After"), "1");
|
||||
});
|
||||
|
||||
test("the module-level broker singleton queue is the same object for every dispatch branch", async () => {
|
||||
// Neither branch of the route is given a `queue` dependency here, so all
|
||||
// three must fall back to the identical `extractionQueue` singleton —
|
||||
// proving production traffic for frames, audio, and subtitles (#11659,
|
||||
// merged alongside #11654 in the same /merge-batch) shares one process
|
||||
// queue instead of each operation racing its own.
|
||||
const routeSource = await import("node:fs/promises").then((fs) =>
|
||||
fs.readFile(
|
||||
new URL("../../src/app/api/modality-bridge/video/extract/route.ts", import.meta.url),
|
||||
"utf8"
|
||||
)
|
||||
);
|
||||
const queueReferences = routeSource.match(/dependencies\.queue \?\? extractionQueue/g) ?? [];
|
||||
assert.equal(
|
||||
queueReferences.length,
|
||||
3,
|
||||
"the frame, audio, and subtitle branches must all default to the one extractionQueue singleton"
|
||||
);
|
||||
});
|
||||
|
||||
test("broker route rejects frame-only parameters when mode=audio", async () => {
|
||||
const response = await handleVideoExtractionBrokerRequest(
|
||||
new Request(`http://localhost${EXTRACT_PATH}?mode=audio&frames=2`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
...buildVideoBridgeBrokerHeaders(),
|
||||
[AUTHZ_HEADER_PEER_LOCALITY]: "loopback",
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
body: Buffer.from("video"),
|
||||
})
|
||||
);
|
||||
assert.equal(response.status, 400);
|
||||
});
|
||||
|
||||
test("broker route runs the injected audio extractor with the shared deadline and byte budget", async () => {
|
||||
let receivedTimeoutMs = 0;
|
||||
let receivedByteLength = 0;
|
||||
const response = await handleVideoExtractionBrokerRequest(
|
||||
new Request(`http://localhost${EXTRACT_PATH}?mode=audio`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
...buildVideoBridgeBrokerHeaders(),
|
||||
[AUTHZ_HEADER_PEER_LOCALITY]: "loopback",
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
body: Buffer.from("video-bytes"),
|
||||
}),
|
||||
{
|
||||
extractAudio: async (bytes, options) => {
|
||||
receivedByteLength = bytes.byteLength;
|
||||
receivedTimeoutMs = options.timeoutMs;
|
||||
return {
|
||||
channels: 1,
|
||||
dataUri: "data:audio/wav;base64,UklGRg==",
|
||||
durationSeconds: 3,
|
||||
sampleRateHz: 16000,
|
||||
};
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(receivedByteLength, Buffer.from("video-bytes").byteLength);
|
||||
assert.equal(receivedTimeoutMs, BROKER_TIMEOUT_MS);
|
||||
const body = (await response.json()) as { audio: { dataUri: string }; durationSeconds: number };
|
||||
assert.equal(body.durationSeconds, 3);
|
||||
assert.equal(body.audio.dataUri, "data:audio/wav;base64,UklGRg==");
|
||||
});
|
||||
|
||||
test("broker queue removes an aborted pending item and never executes it", async () => {
|
||||
const queue = createVideoExtractionQueue({ concurrency: 1, maxPending: 2, maxQueuedBytes: 16 });
|
||||
let release!: () => void;
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from "../../src/lib/modelCapabilityModalities.ts";
|
||||
import {
|
||||
MODALITY_BRIDGE_DEFAULTS,
|
||||
resolveVideoAudioTranscriptionRuntimeSettings,
|
||||
resolveVideoBridgeRuntimeSettings,
|
||||
} from "../../src/shared/constants/modalityBridgeDefaults.ts";
|
||||
import { updateSettingsSchema } from "../../src/shared/validation/settingsSchemas.ts";
|
||||
@@ -95,3 +96,22 @@ test("persisted segment-aware policy remains an explicit opt-in", () => {
|
||||
"segment_aware"
|
||||
);
|
||||
});
|
||||
|
||||
test("the operator half of the FU-06 audio-transcription dual opt-in defaults OFF (#11654)", () => {
|
||||
assert.deepEqual(resolveVideoAudioTranscriptionRuntimeSettings({}), { enabled: false });
|
||||
assert.deepEqual(resolveVideoAudioTranscriptionRuntimeSettings(undefined), { enabled: false });
|
||||
assert.deepEqual(
|
||||
resolveVideoAudioTranscriptionRuntimeSettings({
|
||||
modalityBridgeVideoAudioTranscriptionEnabled: true,
|
||||
}),
|
||||
{ enabled: true }
|
||||
);
|
||||
assert.equal(
|
||||
updateSettingsSchema.safeParse({ modalityBridgeVideoAudioTranscriptionEnabled: true }).success,
|
||||
true
|
||||
);
|
||||
assert.equal(
|
||||
updateSettingsSchema.safeParse({ modalityBridgeVideoAudioTranscriptionEnabled: "yes" }).success,
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user