diff --git a/src/lib/guardrails/audioBridge.ts b/src/lib/guardrails/audioBridge.ts new file mode 100644 index 0000000000..d2e856c027 --- /dev/null +++ b/src/lib/guardrails/audioBridge.ts @@ -0,0 +1,137 @@ +import { getSettings as defaultGetSettings } from "@/lib/db/settings"; +import { getResolvedModelCapabilities } from "@/lib/modelCapabilities"; +import { resolveAudioBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults"; + +import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base"; +import { + callAudioTranscription as defaultCallAudioTranscription, + extractAudioParts, + replaceAudioParts, + selectAudioBridgeModel, + type AudioCredentialCheck, + type AudioPart, + type AudioTranscriptionConfig, +} from "./audioBridgeHelpers"; +import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache"; +import { recordBridgeUse } from "./modalityBridge/bridgeStats"; + +export interface AudioBridgeDependencies { + getSettings?: () => Promise>; + getCapabilities?: (model: string) => { supportsAudio: boolean | null }; + hasUsableCredentials?: AudioCredentialCheck; + selectModel?: (configuredModel: string) => Promise; + callTranscription?: (part: AudioPart, config: AudioTranscriptionConfig) => Promise; +} + +type AudioBridgeBody = { + model?: string; + messages?: Array<{ role?: string; content?: unknown }>; + [key: string]: unknown; +}; + +export class AudioBridgeGuardrail extends BaseGuardrail { + name = "audio-bridge"; + priority = 6; + + private readonly deps: AudioBridgeDependencies; + + constructor(options?: { enabled?: boolean; deps?: AudioBridgeDependencies }) { + super("audio-bridge", { priority: 6, enabled: options?.enabled }); + this.deps = options?.deps ?? {}; + } + + async preCall(payload: unknown, context: GuardrailContext): Promise> { + if (!this.enabled || context.disabledGuardrails?.includes("audio-bridge")) { + return { block: false }; + } + + const body = payload as AudioBridgeBody; + const model = context.model || body?.model; + if (!model || !Array.isArray(body?.messages) || body.messages.length === 0) { + return { block: false }; + } + + const getSettings = this.deps.getSettings ?? defaultGetSettings; + let persisted: Record = {}; + try { + persisted = await getSettings(); + } catch { + // Database settings are optional during early boot; defaults remain safe. + } + const runtime = resolveAudioBridgeRuntimeSettings(persisted); + if (!runtime.enabled) return { block: false }; + + const audioParts = extractAudioParts(body.messages); + if (audioParts.length === 0) return { block: false }; + + const capabilities = (this.deps.getCapabilities ?? getResolvedModelCapabilities)(model); + if (capabilities.supportsAudio === true) return { block: false }; + + const limitedParts = audioParts.slice(0, runtime.maxClips); + const startedAt = Date.now(); + const configuredModel = runtime.model || "auto"; + const sttModel = this.deps.selectModel + ? await this.deps.selectModel(configuredModel) + : await selectAudioBridgeModel(configuredModel, this.deps.hasUsableCredentials); + if (!sttModel) { + if (capabilities.supportsAudio !== false) return { block: false }; + const stubs = limitedParts.map( + (_part, index) => `[Audio ${index + 1}]: (unavailable — no STT provider connected)` + ); + for (const _part of limitedParts) recordBridgeUse("audio", { failure: true }); + return { + block: false, + modifiedPayload: replaceAudioParts(body, limitedParts, stubs), + meta: { + clipsProcessed: stubs.length, + processingTimeMs: Date.now() - startedAt, + sttModel: "unavailable", + }, + }; + } + + const callTranscription = this.deps.callTranscription ?? defaultCallAudioTranscription; + const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null; + const settled = await Promise.allSettled( + limitedParts.map(async (part, index) => { + const key = cache ? bridgeCacheKey(part.ref, "audio-transcription", sttModel) : null; + const cached = key && cache ? cache.get(key) : undefined; + const transcript = + cached ?? + (await callTranscription(part, { model: sttModel, timeoutMs: runtime.timeoutMs })); + if (cached === undefined && key && cache) cache.set(key, transcript); + recordBridgeUse("audio", { cacheHit: cached !== undefined }); + return `[Audio ${index + 1}]: ${transcript}`; + }) + ); + + const transcripts = settled.map((result, index): string | null => { + if (result.status === "fulfilled") return result.value; + const message = + result.reason instanceof Error ? result.reason.message : String(result.reason); + context.log?.warn?.("AUDIO_BRIDGE", `Failed to transcribe audio ${index + 1}: ${message}`); + recordBridgeUse("audio", { failure: true }); + return null; + }); + if ( + capabilities.supportsAudio === false && + transcripts.every((transcript) => transcript === null) + ) { + for (let index = 0; index < transcripts.length; index++) { + transcripts[index] = `[Audio ${index + 1}]: (unavailable — no STT provider connected)`; + } + } + const clipsProcessed = transcripts.filter((value) => value !== null).length; + if (clipsProcessed === 0) return { block: false }; + + return { + block: false, + modifiedPayload: replaceAudioParts(body, limitedParts, transcripts), + meta: { + clipsProcessed, + processingTimeMs: Date.now() - startedAt, + sttModel, + }, + }; + } +} diff --git a/src/lib/guardrails/modalityBridge/bridgeCache.ts b/src/lib/guardrails/modalityBridge/bridgeCache.ts index e0a10429e6..f4efb797fa 100644 --- a/src/lib/guardrails/modalityBridge/bridgeCache.ts +++ b/src/lib/guardrails/modalityBridge/bridgeCache.ts @@ -77,6 +77,8 @@ export function getSharedBridgeCache(ttlMs: number, maxEntries: number): BridgeC * bridge (vision, audio) goes through here so the minutes→ms conversion can * never diverge between callers and thrash the singleton on each request. */ -export function getSharedBridgeCacheFor(settings: VisionBridgeRuntimeSettings): BridgeCache { +export function getSharedBridgeCacheFor( + settings: Pick +): BridgeCache { return getSharedBridgeCache(settings.cacheTtlMinutes * 60_000, settings.cacheMaxEntries); } diff --git a/src/lib/guardrails/modalityBridge/bridgeStats.ts b/src/lib/guardrails/modalityBridge/bridgeStats.ts index 052c73fabe..df6c416175 100644 --- a/src/lib/guardrails/modalityBridge/bridgeStats.ts +++ b/src/lib/guardrails/modalityBridge/bridgeStats.ts @@ -62,7 +62,11 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string `image->text;model=${String(meta.visionModel ?? "unknown")};parts=${meta.imagesProcessed}` ); } - if (r.guardrail === "audio-bridge" && typeof meta.clipsProcessed === "number") { + if ( + r.guardrail === "audio-bridge" && + typeof meta.clipsProcessed === "number" && + !meta.rerouted + ) { segments.push( `audio->text;model=${String(meta.sttModel ?? "unknown")};parts=${meta.clipsProcessed}` ); diff --git a/src/lib/guardrails/registry.ts b/src/lib/guardrails/registry.ts index 531539ab17..774e47014a 100644 --- a/src/lib/guardrails/registry.ts +++ b/src/lib/guardrails/registry.ts @@ -7,6 +7,7 @@ import { import { PIIMaskerGuardrail } from "./piiMasker"; import { PromptInjectionGuardrail } from "./promptInjection"; import { VisionBridgeGuardrail } from "./visionBridge"; +import { AudioBridgeGuardrail } from "./audioBridge"; import { CredentialMaskerGuardrail } from "./credentialMasker"; /** @@ -286,6 +287,7 @@ export function registerDefaultGuardrails() { if (defaultGuardrailsRegistered) return guardrailRegistry; guardrailRegistry.register(new VisionBridgeGuardrail()); + guardrailRegistry.register(new AudioBridgeGuardrail()); guardrailRegistry.register(new PIIMaskerGuardrail()); guardrailRegistry.register(new CredentialMaskerGuardrail()); guardrailRegistry.register(new PromptInjectionGuardrail()); diff --git a/tests/unit/guardrails/audioBridge.test.ts b/tests/unit/guardrails/audioBridge.test.ts new file mode 100644 index 0000000000..f6dc52e011 --- /dev/null +++ b/tests/unit/guardrails/audioBridge.test.ts @@ -0,0 +1,267 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + AudioBridgeGuardrail, + type AudioBridgeDependencies, +} from "../../../src/lib/guardrails/audioBridge.ts"; +import { + registerDefaultGuardrails, + resetGuardrailsForTests, +} from "../../../src/lib/guardrails/registry.ts"; +import { buildModalityBridgeHeader } from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts"; + +const audioPayload = () => ({ + model: "example/text-only", + messages: [ + { + role: "user", + content: [ + { type: "input_audio", input_audio: { data: "UklGRg==", format: "wav" } }, + { type: "text", text: "What was said?" }, + ], + }, + ], +}); + +const twoAudioPayload = () => ({ + model: "example/text-only", + messages: [ + { + role: "user", + content: [ + { type: "input_audio", input_audio: { data: "UklGRjE=", format: "wav" } }, + { type: "audio_url", audio_url: { url: "data:audio/wav;base64,UklGRjI=" } }, + ], + }, + ], +}); + +function createGuardrail(overrides: Partial = {}) { + return new AudioBridgeGuardrail({ + deps: { + getSettings: async () => ({ + modalityBridgeAudioEnabled: true, + modalityBridgeAudioModel: "deepgram/nova-3", + }), + getCapabilities: () => ({ supportsAudio: false }), + selectModel: async () => "deepgram/nova-3", + callTranscription: async () => "hello from the clip", + ...overrides, + }, + }); +} + +test("AudioBridgeGuardrail has the approved name and priority", () => { + const guardrail = createGuardrail(); + assert.equal(guardrail.name, "audio-bridge"); + assert.equal(guardrail.priority, 6); +}); + +test("native audio-capable targets bypass transcription", async () => { + let calls = 0; + const guardrail = createGuardrail({ + getCapabilities: () => ({ supportsAudio: true }), + callTranscription: async () => { + calls += 1; + return "should not run"; + }, + }); + + const result = await guardrail.preCall(audioPayload(), {}); + assert.equal(calls, 0); + assert.equal(result.modifiedPayload, undefined); +}); + +test("disabled settings and per-request disable both bypass transcription", async () => { + let calls = 0; + const disabledBySetting = createGuardrail({ + getSettings: async () => ({ modalityBridgeAudioEnabled: false }), + callTranscription: async () => { + calls += 1; + return "should not run"; + }, + }); + const enabled = createGuardrail({ + callTranscription: async () => { + calls += 1; + return "should not run"; + }, + }); + + assert.equal((await disabledBySetting.preCall(audioPayload(), {})).modifiedPayload, undefined); + assert.equal( + ( + await enabled.preCall(audioPayload(), { + disabledGuardrails: ["audio-bridge"], + }) + ).modifiedPayload, + undefined + ); + assert.equal(calls, 0); +}); + +test("text-only targets receive the STT transcript in place of audio", async () => { + const guardrail = createGuardrail(); + const result = await guardrail.preCall(audioPayload(), {}); + const modified = result.modifiedPayload as ReturnType; + + assert.deepEqual(modified.messages[0].content[0], { + type: "text", + text: "[Audio 1]: hello from the clip", + }); + assert.equal(modified.messages[0].content[1].text, "What was said?"); + assert.equal(result.meta?.clipsProcessed, 1); + assert.equal(result.meta?.sttModel, "deepgram/nova-3"); + assert.equal(typeof result.meta?.processingTimeMs, "number"); +}); + +test("all STT failures become explicit stubs for a proven text-only target", async () => { + const guardrail = createGuardrail({ + callTranscription: async () => { + throw new Error("no STT connection"); + }, + }); + + const result = await guardrail.preCall(twoAudioPayload(), {}); + const modified = result.modifiedPayload as ReturnType; + assert.deepEqual( + modified.messages[0].content.map((part) => ("text" in part ? part.text : null)), + [ + "[Audio 1]: (unavailable — no STT provider connected)", + "[Audio 2]: (unavailable — no STT provider connected)", + ] + ); + assert.equal(result.meta?.clipsProcessed, 2); +}); + +test("missing STT credentials become stubs for a proven text-only target", async () => { + let calls = 0; + const guardrail = createGuardrail({ + selectModel: async () => null, + callTranscription: async () => { + calls += 1; + return "should not run"; + }, + }); + + const result = await guardrail.preCall(audioPayload(), {}); + const modified = result.modifiedPayload as ReturnType; + assert.equal(calls, 0); + assert.deepEqual(modified.messages[0].content[0], { + type: "text", + text: "[Audio 1]: (unavailable — no STT provider connected)", + }); + assert.equal(result.meta?.sttModel, "unavailable"); +}); + +test("partial STT failure preserves only the failed audio part", async () => { + const original = twoAudioPayload(); + const guardrail = createGuardrail({ + getSettings: async () => ({ + modalityBridgeAudioEnabled: true, + modalityBridgeAudioModel: "deepgram/nova-3", + modalityBridgeCacheEnabled: false, + }), + callTranscription: async (part) => { + if (part.partIndex === 0) throw new Error("first failed"); + return "second succeeded"; + }, + }); + + const result = await guardrail.preCall(original, {}); + const modified = result.modifiedPayload as ReturnType; + assert.deepEqual(modified.messages[0].content[0], original.messages[0].content[0]); + assert.deepEqual(modified.messages[0].content[1], { + type: "text", + text: "[Audio 2]: second succeeded", + }); + assert.equal(result.meta?.clipsProcessed, 1); +}); + +test("unknown target capability preserves audio when every STT call fails", async () => { + const original = audioPayload(); + original.messages[0].content[0].input_audio.data = "dW5rbm93bi1hdWRpbw=="; + const snapshot = structuredClone(original); + const guardrail = createGuardrail({ + getCapabilities: () => ({ supportsAudio: null }), + getSettings: async () => ({ + modalityBridgeAudioEnabled: true, + modalityBridgeAudioModel: "deepgram/nova-3", + modalityBridgeCacheEnabled: false, + }), + callTranscription: async () => { + throw new Error("temporary STT failure"); + }, + }); + + const result = await guardrail.preCall(original, {}); + assert.equal(result.modifiedPayload, undefined); + assert.deepEqual(original, snapshot, "the input object must not be mutated"); +}); + +test("successful transcripts are reused from the shared cache", async () => { + let calls = 0; + const payload = audioPayload(); + payload.messages[0].content[0].input_audio.data = "Y2FjaGUtdW5pcXVl"; + const guardrail = createGuardrail({ + callTranscription: async () => { + calls += 1; + return "cached transcript"; + }, + }); + + await guardrail.preCall(payload, {}); + await guardrail.preCall(payload, {}); + assert.equal(calls, 1); +}); + +test("maxClips limits work without dropping later audio parts", async () => { + const original = twoAudioPayload(); + const guardrail = createGuardrail({ + getSettings: async () => ({ + modalityBridgeAudioEnabled: true, + modalityBridgeAudioModel: "deepgram/nova-3", + modalityBridgeAudioMaxClips: 1, + modalityBridgeCacheEnabled: false, + }), + }); + + const result = await guardrail.preCall(original, {}); + const modified = result.modifiedPayload as ReturnType; + assert.deepEqual(modified.messages[0].content[0], { + type: "text", + text: "[Audio 1]: hello from the clip", + }); + assert.deepEqual(modified.messages[0].content[1], original.messages[0].content[1]); +}); + +test("default registry places Audio Bridge after Vision Bridge", () => { + resetGuardrailsForTests({ registerDefaults: false }); + const names = registerDefaultGuardrails() + .list() + .map((guardrail) => guardrail.name); + assert.deepEqual(names.slice(0, 2), ["vision-bridge", "audio-bridge"]); + resetGuardrailsForTests(); +}); + +test("audio transparency header is emitted only for transformed clips", () => { + assert.equal( + buildModalityBridgeHeader([ + { + guardrail: "audio-bridge", + meta: { clipsProcessed: 2, sttModel: "deepgram/nova-3" }, + }, + ]), + "audio->text;model=deepgram/nova-3;parts=2" + ); + assert.equal( + buildModalityBridgeHeader([ + { + guardrail: "audio-bridge", + meta: { clipsProcessed: 2, sttModel: "deepgram/nova-3", rerouted: true }, + }, + ]), + null + ); +});