diff --git a/src/lib/guardrails/audioBridgeHelpers.ts b/src/lib/guardrails/audioBridgeHelpers.ts new file mode 100644 index 0000000000..1124a33930 --- /dev/null +++ b/src/lib/guardrails/audioBridgeHelpers.ts @@ -0,0 +1,239 @@ +import { AUDIO_TRANSCRIPTION_PROVIDERS } from "@omniroute/open-sse/config/audioRegistry.ts"; +import { detectMediaParts } from "@omniroute/open-sse/utils/mediaParts"; + +import { getRuntimePorts } from "@/lib/runtime/ports"; +import { fetchRemoteImage } from "@/shared/network/remoteImageFetch"; +import { resolveSelfLoopBearer } from "@/shared/middleware/chatBodyAdmission"; + +import { hasUsableCredentialsForModel } from "./visionBridgeCredentials"; + +export type AudioCredentialCheck = (model: string) => Promise; + +export interface AudioPart { + messageIndex: number; + partIndex: number; + ref: string; + shape: "input_audio" | "audio_url" | "audio_source"; + format?: string; +} + +export interface AudioTranscriptionConfig { + model: string; + timeoutMs: number; +} + +export interface AudioTranscriptionDependencies { + fetchImpl?: typeof fetch; + fetchRemote?: ( + url: string, + options: { signal: AbortSignal } + ) => Promise<{ buffer: Buffer; contentType: string; url: string }>; + getPort?: () => number; + getBearer?: () => string; +} + +const AUDIO_FORMAT_MIME: Record = { + aac: "audio/aac", + flac: "audio/flac", + m4a: "audio/mp4", + mp3: "audio/mpeg", + mp4: "audio/mp4", + ogg: "audio/ogg", + opus: "audio/opus", + wav: "audio/wav", + webm: "audio/webm", +}; + +const AUDIO_MIME_FORMAT: Record = { + "audio/aac": "aac", + "audio/flac": "flac", + "audio/mp4": "mp4", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/ogg": "ogg", + "audio/opus": "opus", + "audio/wav": "wav", + "audio/wave": "wav", + "audio/webm": "webm", + "audio/x-wav": "wav", +}; + +type AudioMessage = { role?: string; content?: unknown }; + +function formatFromRef(ref: string): string | undefined { + const dataMime = /^data:([^;,]+)/i.exec(ref)?.[1]?.toLowerCase(); + if (dataMime) return AUDIO_MIME_FORMAT[dataMime]; + try { + const pathname = new URL(ref).pathname; + const extension = pathname.includes(".") ? pathname.split(".").pop()?.toLowerCase() : undefined; + return extension && AUDIO_FORMAT_MIME[extension] ? extension : undefined; + } catch { + return undefined; + } +} + +function formatForDetectedPart( + messages: ReadonlyArray, + messageIndex: number, + partIndex: number, + ref: string +): string | undefined { + const content = messages[messageIndex]?.content; + const raw = Array.isArray(content) ? content[partIndex] : null; + if (!raw || typeof raw !== "object") return formatFromRef(ref); + const part = raw as Record; + const inputAudio = part.input_audio as Record | undefined; + if (typeof inputAudio?.format === "string" && inputAudio.format.trim()) { + return inputAudio.format.trim().toLowerCase(); + } + const source = part.source as Record | undefined; + if (typeof source?.media_type === "string") { + return AUDIO_MIME_FORMAT[source.media_type.toLowerCase()] ?? formatFromRef(ref); + } + return formatFromRef(ref); +} + +/** Extract only spliceable top-level audio parts from every message. */ +export function extractAudioParts( + messages: ReadonlyArray | undefined | null +): AudioPart[] { + if (!Array.isArray(messages)) return []; + return detectMediaParts(messages) + .filter( + (part) => + part.kind === "audio" && + !part.nested && + part.ref.length > 0 && + (part.shape === "input_audio" || + part.shape === "audio_url" || + part.shape === "audio_source") + ) + .map((part) => ({ + messageIndex: part.messageIndex, + partIndex: part.partIndex, + ref: part.ref, + shape: part.shape as AudioPart["shape"], + format: formatForDetectedPart(messages, part.messageIndex, part.partIndex, part.ref), + })); +} + +/** Replace successful transcripts in place; null results preserve the original audio. */ +export function replaceAudioParts( + body: TBody, + parts: readonly AudioPart[], + transcripts: readonly (string | null)[] +): TBody { + const result = structuredClone(body); + for (let index = 0; index < parts.length && index < transcripts.length; index++) { + const transcript = transcripts[index]; + if (transcript === null) continue; + const part = parts[index]; + const content = result.messages?.[part.messageIndex]?.content; + if (!Array.isArray(content) || part.partIndex >= content.length) continue; + content[part.partIndex] = { type: "text", text: transcript }; + } + return result; +} + +function listTranscriptionModels(): string[] { + const models: string[] = []; + for (const [providerId, provider] of Object.entries(AUDIO_TRANSCRIPTION_PROVIDERS)) { + for (const model of provider.models) { + models.push(model.id.startsWith(`${providerId}/`) ? model.id : `${providerId}/${model.id}`); + } + } + return models; +} + +/** Select a configured STT model, or the first catalog model with usable credentials. */ +export async function selectAudioBridgeModel( + configuredModel: string, + hasUsableCredentials: AudioCredentialCheck = hasUsableCredentialsForModel +): Promise { + const fixed = configuredModel.trim(); + if (fixed && fixed !== "auto") { + return (await hasUsableCredentials(fixed)) === false ? null : fixed; + } + + for (const candidate of listTranscriptionModels()) { + if ((await hasUsableCredentials(candidate)) === true) return candidate; + } + return null; +} + +/** Send one audio part through OmniRoute's existing multipart transcription route. */ +export async function callAudioTranscription( + part: AudioPart, + config: AudioTranscriptionConfig, + deps: AudioTranscriptionDependencies = {} +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), config.timeoutMs); + try { + let bytes: Buffer; + let detectedMime: string | undefined; + const dataUri = /^data:([^;,]+);base64,(.+)$/is.exec(part.ref); + if (dataUri) { + detectedMime = dataUri[1].toLowerCase(); + bytes = Buffer.from(dataUri[2], "base64"); + } else if (/^https?:\/\//i.test(part.ref)) { + const fetchRemote = + deps.fetchRemote ?? + ((url: string, options: { signal: AbortSignal }) => + fetchRemoteImage(url, { + guard: "public-only", + maxBytes: 25 * 1024 * 1024, + pinDns: true, + signal: options.signal, + timeoutMs: config.timeoutMs, + })); + const remote = await fetchRemote(part.ref, { signal: controller.signal }); + bytes = remote.buffer; + detectedMime = remote.contentType.split(";", 1)[0]?.trim().toLowerCase(); + } else { + bytes = Buffer.from(part.ref, "base64"); + } + + const configuredFormat = part.format?.trim().toLowerCase(); + const format = + configuredFormat || (detectedMime ? AUDIO_MIME_FORMAT[detectedMime] : undefined) || "wav"; + const mime = + (detectedMime?.startsWith("audio/") ? detectedMime : undefined) ?? + AUDIO_FORMAT_MIME[format] ?? + "application/octet-stream"; + const file = new Blob([Uint8Array.from(bytes)], { type: mime }); + const form = new FormData(); + form.set("file", file, `audio.${format.replace(/[^a-z0-9]/g, "") || "wav"}`); + form.set("model", config.model); + + const port = (deps.getPort ?? (() => getRuntimePorts().port))(); + const bearer = (deps.getBearer ?? resolveSelfLoopBearer)(); + const response = await (deps.fetchImpl ?? fetch)( + `http://localhost:${port}/v1/audio/transcriptions`, + { + method: "POST", + signal: controller.signal, + headers: { + Accept: "application/json", + Authorization: `Bearer ${bearer}`, + }, + body: form, + } + ); + if (!response.ok) { + throw new Error(`Audio transcription failed (${response.status})`); + } + const data = (await response.json()) as { text?: unknown }; + if (typeof data.text !== "string") { + throw new Error("Audio transcription returned an invalid response"); + } + return data.text.trim(); + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + throw new Error("Audio transcription timed out"); + } + throw error; + } finally { + clearTimeout(timeout); + } +} diff --git a/tests/unit/guardrails/audioBridgeHelpers.test.ts b/tests/unit/guardrails/audioBridgeHelpers.test.ts new file mode 100644 index 0000000000..89ee5444de --- /dev/null +++ b/tests/unit/guardrails/audioBridgeHelpers.test.ts @@ -0,0 +1,188 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + callAudioTranscription, + extractAudioParts, + replaceAudioParts, + selectAudioBridgeModel, + type AudioPart, +} from "../../../src/lib/guardrails/audioBridgeHelpers.ts"; + +test("fixed STT model is honored when its credential is usable", async () => { + const checked: string[] = []; + const selected = await selectAudioBridgeModel("deepgram/nova-2", async (model) => { + checked.push(model); + return true; + }); + + assert.equal(selected, "deepgram/nova-2"); + assert.deepEqual(checked, ["deepgram/nova-2"]); +}); + +test("fixed STT model is rejected when its credential is unavailable", async () => { + assert.equal(await selectAudioBridgeModel("deepgram/nova-2", async () => false), null); +}); + +test("auto selects the first catalog STT model with a usable credential", async () => { + const selected = await selectAudioBridgeModel( + "auto", + async (model) => model === "deepgram/nova-3" + ); + + assert.equal(selected, "deepgram/nova-3"); +}); + +test("input_audio is posted as multipart to the authenticated transcription self-loop", async () => { + let capturedUrl = ""; + let capturedInit: RequestInit | undefined; + const part: AudioPart = { + messageIndex: 0, + partIndex: 0, + ref: Buffer.from("RIFF test audio").toString("base64"), + shape: "input_audio", + format: "wav", + }; + + const transcript = await callAudioTranscription( + part, + { model: "deepgram/nova-3", timeoutMs: 1_000 }, + { + fetchImpl: async (input, init) => { + capturedUrl = String(input); + capturedInit = init; + return Response.json({ text: "hello from audio" }); + }, + getPort: () => 3210, + getBearer: () => "internal-test-key", + } + ); + + assert.equal(transcript, "hello from audio"); + assert.equal(capturedUrl, "http://localhost:3210/v1/audio/transcriptions"); + assert.equal(capturedInit?.method, "POST"); + assert.equal(new Headers(capturedInit?.headers).get("authorization"), "Bearer internal-test-key"); + + const form = capturedInit?.body as FormData; + assert.equal(form.get("model"), "deepgram/nova-3"); + const file = form.get("file") as File; + assert.equal(file.name, "audio.wav"); + assert.equal(file.type, "audio/wav"); + assert.equal(Buffer.from(await file.arrayBuffer()).toString(), "RIFF test audio"); +}); + +test("audio extraction and replacement cover the full history without dropping failed clips", () => { + const body = { + model: "text-only/model", + messages: [ + { + role: "user", + content: [ + { type: "input_audio", input_audio: { data: "UklGRg==", format: "wav" } }, + { type: "text", text: "first" }, + ], + }, + { + role: "user", + content: [ + { type: "audio_url", audio_url: { url: "data:audio/mpeg;base64,SUQz" } }, + { source: { media_type: "audio/ogg", data: "T2dnUw==" } }, + { + type: "text", + nested: { type: "input_audio", input_audio: { data: "bmVzdGVk", format: "wav" } }, + }, + ], + }, + ], + }; + + const parts = extractAudioParts(body.messages); + assert.deepEqual( + parts.map(({ messageIndex, partIndex, shape, format }) => ({ + messageIndex, + partIndex, + shape, + format, + })), + [ + { messageIndex: 0, partIndex: 0, shape: "input_audio", format: "wav" }, + { messageIndex: 1, partIndex: 0, shape: "audio_url", format: "mp3" }, + { messageIndex: 1, partIndex: 1, shape: "audio_source", format: "ogg" }, + ] + ); + + const replaced = replaceAudioParts(body, parts, ["[Audio 1]: hello", null, "[Audio 3]: bye"]); + assert.deepEqual(replaced.messages[0].content[0], { type: "text", text: "[Audio 1]: hello" }); + assert.deepEqual( + replaced.messages[1].content[0], + body.messages[1].content[0], + "a failed transcription must preserve the original audio part" + ); + assert.deepEqual(replaced.messages[1].content[1], { type: "text", text: "[Audio 3]: bye" }); + assert.deepEqual( + replaced.messages[1].content[2], + body.messages[1].content[2], + "nested audio is not a spliceable top-level part" + ); +}); + +test("audio_url data URIs are decoded before multipart upload", async () => { + let uploaded: File | null = null; + await callAudioTranscription( + { + messageIndex: 0, + partIndex: 0, + ref: "data:audio/mpeg;base64,SUQz", + shape: "audio_url", + format: "mp3", + }, + { model: "deepgram/nova-3", timeoutMs: 1_000 }, + { + fetchImpl: async (_input, init) => { + uploaded = (init?.body as FormData).get("file") as File; + return Response.json({ text: "ok" }); + }, + getPort: () => 3210, + getBearer: () => "internal-test-key", + } + ); + + assert.ok(uploaded); + assert.equal(uploaded.type, "audio/mpeg"); + assert.equal(Buffer.from(await uploaded.arrayBuffer()).toString(), "ID3"); +}); + +test("remote audio_url uses the guarded remote fetch before self-loop upload", async () => { + let fetchedUrl = ""; + let uploaded: File | null = null; + await callAudioTranscription( + { + messageIndex: 0, + partIndex: 0, + ref: "https://media.example.test/clip.ogg", + shape: "audio_url", + format: "ogg", + }, + { model: "deepgram/nova-3", timeoutMs: 1_000 }, + { + fetchRemote: async (url) => { + fetchedUrl = url; + return { + buffer: Buffer.from("OggS remote audio"), + contentType: "audio/ogg", + url, + }; + }, + fetchImpl: async (_input, init) => { + uploaded = (init?.body as FormData).get("file") as File; + return Response.json({ text: "ok" }); + }, + getPort: () => 3210, + getBearer: () => "internal-test-key", + } + ); + + assert.equal(fetchedUrl, "https://media.example.test/clip.ogg"); + assert.ok(uploaded); + assert.equal(Buffer.from(await uploaded.arrayBuffer()).toString(), "OggS remote audio"); +});