feat(modality-bridge): add Audio Bridge guardrail

src/lib/guardrails/audioBridge.ts

@@ -0,0 +1,137 @@
+import { getSettings as defaultGetSettings } from "@/lib/db/settings";
+import { getResolvedModelCapabilities } from "@/lib/modelCapabilities";
+import { resolveAudioBridgeRuntimeSettings } from "@/shared/constants/modalityBridgeDefaults";
+
+import { BaseGuardrail, type GuardrailContext, type GuardrailResult } from "./base";
+import {
+  callAudioTranscription as defaultCallAudioTranscription,
+  extractAudioParts,
+  replaceAudioParts,
+  selectAudioBridgeModel,
+  type AudioCredentialCheck,
+  type AudioPart,
+  type AudioTranscriptionConfig,
+} from "./audioBridgeHelpers";
+import { bridgeCacheKey, getSharedBridgeCacheFor } from "./modalityBridge/bridgeCache";
+import { recordBridgeUse } from "./modalityBridge/bridgeStats";
+
+export interface AudioBridgeDependencies {
+  getSettings?: () => Promise<Record<string, unknown>>;
+  getCapabilities?: (model: string) => { supportsAudio: boolean | null };
+  hasUsableCredentials?: AudioCredentialCheck;
+  selectModel?: (configuredModel: string) => Promise<string | null>;
+  callTranscription?: (part: AudioPart, config: AudioTranscriptionConfig) => Promise<string>;
+}
+
+type AudioBridgeBody = {
+  model?: string;
+  messages?: Array<{ role?: string; content?: unknown }>;
+  [key: string]: unknown;
+};
+
+export class AudioBridgeGuardrail extends BaseGuardrail {
+  name = "audio-bridge";
+  priority = 6;
+
+  private readonly deps: AudioBridgeDependencies;
+
+  constructor(options?: { enabled?: boolean; deps?: AudioBridgeDependencies }) {
+    super("audio-bridge", { priority: 6, enabled: options?.enabled });
+    this.deps = options?.deps ?? {};
+  }
+
+  async preCall(payload: unknown, context: GuardrailContext): Promise<GuardrailResult<unknown>> {
+    if (!this.enabled || context.disabledGuardrails?.includes("audio-bridge")) {
+      return { block: false };
+    }
+
+    const body = payload as AudioBridgeBody;
+    const model = context.model || body?.model;
+    if (!model || !Array.isArray(body?.messages) || body.messages.length === 0) {
+      return { block: false };
+    }
+
+    const getSettings = this.deps.getSettings ?? defaultGetSettings;
+    let persisted: Record<string, unknown> = {};
+    try {
+      persisted = await getSettings();
+    } catch {
+      // Database settings are optional during early boot; defaults remain safe.
+    }
+    const runtime = resolveAudioBridgeRuntimeSettings(persisted);
+    if (!runtime.enabled) return { block: false };
+
+    const audioParts = extractAudioParts(body.messages);
+    if (audioParts.length === 0) return { block: false };
+
+    const capabilities = (this.deps.getCapabilities ?? getResolvedModelCapabilities)(model);
+    if (capabilities.supportsAudio === true) return { block: false };
+
+    const limitedParts = audioParts.slice(0, runtime.maxClips);
+    const startedAt = Date.now();
+    const configuredModel = runtime.model || "auto";
+    const sttModel = this.deps.selectModel
+      ? await this.deps.selectModel(configuredModel)
+      : await selectAudioBridgeModel(configuredModel, this.deps.hasUsableCredentials);
+    if (!sttModel) {
+      if (capabilities.supportsAudio !== false) return { block: false };
+      const stubs = limitedParts.map(
+        (_part, index) => `[Audio ${index + 1}]: (unavailable — no STT provider connected)`
+      );
+      for (const _part of limitedParts) recordBridgeUse("audio", { failure: true });
+      return {
+        block: false,
+        modifiedPayload: replaceAudioParts(body, limitedParts, stubs),
+        meta: {
+          clipsProcessed: stubs.length,
+          processingTimeMs: Date.now() - startedAt,
+          sttModel: "unavailable",
+        },
+      };
+    }
+
+    const callTranscription = this.deps.callTranscription ?? defaultCallAudioTranscription;
+    const cache = runtime.cacheEnabled ? getSharedBridgeCacheFor(runtime) : null;
+    const settled = await Promise.allSettled(
+      limitedParts.map(async (part, index) => {
+        const key = cache ? bridgeCacheKey(part.ref, "audio-transcription", sttModel) : null;
+        const cached = key && cache ? cache.get(key) : undefined;
+        const transcript =
+          cached ??
+          (await callTranscription(part, { model: sttModel, timeoutMs: runtime.timeoutMs }));
+        if (cached === undefined && key && cache) cache.set(key, transcript);
+        recordBridgeUse("audio", { cacheHit: cached !== undefined });
+        return `[Audio ${index + 1}]: ${transcript}`;
+      })
+    );
+
+    const transcripts = settled.map((result, index): string | null => {
+      if (result.status === "fulfilled") return result.value;
+      const message =
+        result.reason instanceof Error ? result.reason.message : String(result.reason);
+      context.log?.warn?.("AUDIO_BRIDGE", `Failed to transcribe audio ${index + 1}: ${message}`);
+      recordBridgeUse("audio", { failure: true });
+      return null;
+    });
+    if (
+      capabilities.supportsAudio === false &&
+      transcripts.every((transcript) => transcript === null)
+    ) {
+      for (let index = 0; index < transcripts.length; index++) {
+        transcripts[index] = `[Audio ${index + 1}]: (unavailable — no STT provider connected)`;
+      }
+    }
+    const clipsProcessed = transcripts.filter((value) => value !== null).length;
+    if (clipsProcessed === 0) return { block: false };
+
+    return {
+      block: false,
+      modifiedPayload: replaceAudioParts(body, limitedParts, transcripts),
+      meta: {
+        clipsProcessed,
+        processingTimeMs: Date.now() - startedAt,
+        sttModel,
+      },
+    };
+  }
+}

src/lib/guardrails/modalityBridge/bridgeCache.ts

@@ -77,6 +77,8 @@ export function getSharedBridgeCache(ttlMs: number, maxEntries: number): BridgeC
  * bridge (vision, audio) goes through here so the minutes→ms conversion can
  * never diverge between callers and thrash the singleton on each request.
  */
-export function getSharedBridgeCacheFor(settings: VisionBridgeRuntimeSettings): BridgeCache {
+export function getSharedBridgeCacheFor(
+  settings: Pick<VisionBridgeRuntimeSettings, "cacheTtlMinutes" | "cacheMaxEntries">
+): BridgeCache {
   return getSharedBridgeCache(settings.cacheTtlMinutes * 60_000, settings.cacheMaxEntries);
 }

src/lib/guardrails/modalityBridge/bridgeStats.ts

@@ -62,7 +62,11 @@ export function buildModalityBridgeHeader(results: GuardrailMetaEntry[]): string
         `image->text;model=${String(meta.visionModel ?? "unknown")};parts=${meta.imagesProcessed}`
       );
     }
-    if (r.guardrail === "audio-bridge" && typeof meta.clipsProcessed === "number") {
+    if (
+      r.guardrail === "audio-bridge" &&
+      typeof meta.clipsProcessed === "number" &&
+      !meta.rerouted
+    ) {
       segments.push(
         `audio->text;model=${String(meta.sttModel ?? "unknown")};parts=${meta.clipsProcessed}`
       );

src/lib/guardrails/registry.ts

@@ -7,6 +7,7 @@ import {
 import { PIIMaskerGuardrail } from "./piiMasker";
 import { PromptInjectionGuardrail } from "./promptInjection";
 import { VisionBridgeGuardrail } from "./visionBridge";
+import { AudioBridgeGuardrail } from "./audioBridge";
 import { CredentialMaskerGuardrail } from "./credentialMasker";
 
 /**
@@ -286,6 +287,7 @@ export function registerDefaultGuardrails() {
   if (defaultGuardrailsRegistered) return guardrailRegistry;
 
   guardrailRegistry.register(new VisionBridgeGuardrail());
+  guardrailRegistry.register(new AudioBridgeGuardrail());
   guardrailRegistry.register(new PIIMaskerGuardrail());
   guardrailRegistry.register(new CredentialMaskerGuardrail());
   guardrailRegistry.register(new PromptInjectionGuardrail());

tests/unit/guardrails/audioBridge.test.ts

@@ -0,0 +1,267 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  AudioBridgeGuardrail,
+  type AudioBridgeDependencies,
+} from "../../../src/lib/guardrails/audioBridge.ts";
+import {
+  registerDefaultGuardrails,
+  resetGuardrailsForTests,
+} from "../../../src/lib/guardrails/registry.ts";
+import { buildModalityBridgeHeader } from "../../../src/lib/guardrails/modalityBridge/bridgeStats.ts";
+
+const audioPayload = () => ({
+  model: "example/text-only",
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "input_audio", input_audio: { data: "UklGRg==", format: "wav" } },
+        { type: "text", text: "What was said?" },
+      ],
+    },
+  ],
+});
+
+const twoAudioPayload = () => ({
+  model: "example/text-only",
+  messages: [
+    {
+      role: "user",
+      content: [
+        { type: "input_audio", input_audio: { data: "UklGRjE=", format: "wav" } },
+        { type: "audio_url", audio_url: { url: "data:audio/wav;base64,UklGRjI=" } },
+      ],
+    },
+  ],
+});
+
+function createGuardrail(overrides: Partial<AudioBridgeDependencies> = {}) {
+  return new AudioBridgeGuardrail({
+    deps: {
+      getSettings: async () => ({
+        modalityBridgeAudioEnabled: true,
+        modalityBridgeAudioModel: "deepgram/nova-3",
+      }),
+      getCapabilities: () => ({ supportsAudio: false }),
+      selectModel: async () => "deepgram/nova-3",
+      callTranscription: async () => "hello from the clip",
+      ...overrides,
+    },
+  });
+}
+
+test("AudioBridgeGuardrail has the approved name and priority", () => {
+  const guardrail = createGuardrail();
+  assert.equal(guardrail.name, "audio-bridge");
+  assert.equal(guardrail.priority, 6);
+});
+
+test("native audio-capable targets bypass transcription", async () => {
+  let calls = 0;
+  const guardrail = createGuardrail({
+    getCapabilities: () => ({ supportsAudio: true }),
+    callTranscription: async () => {
+      calls += 1;
+      return "should not run";
+    },
+  });
+
+  const result = await guardrail.preCall(audioPayload(), {});
+  assert.equal(calls, 0);
+  assert.equal(result.modifiedPayload, undefined);
+});
+
+test("disabled settings and per-request disable both bypass transcription", async () => {
+  let calls = 0;
+  const disabledBySetting = createGuardrail({
+    getSettings: async () => ({ modalityBridgeAudioEnabled: false }),
+    callTranscription: async () => {
+      calls += 1;
+      return "should not run";
+    },
+  });
+  const enabled = createGuardrail({
+    callTranscription: async () => {
+      calls += 1;
+      return "should not run";
+    },
+  });
+
+  assert.equal((await disabledBySetting.preCall(audioPayload(), {})).modifiedPayload, undefined);
+  assert.equal(
+    (
+      await enabled.preCall(audioPayload(), {
+        disabledGuardrails: ["audio-bridge"],
+      })
+    ).modifiedPayload,
+    undefined
+  );
+  assert.equal(calls, 0);
+});
+
+test("text-only targets receive the STT transcript in place of audio", async () => {
+  const guardrail = createGuardrail();
+  const result = await guardrail.preCall(audioPayload(), {});
+  const modified = result.modifiedPayload as ReturnType<typeof audioPayload>;
+
+  assert.deepEqual(modified.messages[0].content[0], {
+    type: "text",
+    text: "[Audio 1]: hello from the clip",
+  });
+  assert.equal(modified.messages[0].content[1].text, "What was said?");
+  assert.equal(result.meta?.clipsProcessed, 1);
+  assert.equal(result.meta?.sttModel, "deepgram/nova-3");
+  assert.equal(typeof result.meta?.processingTimeMs, "number");
+});
+
+test("all STT failures become explicit stubs for a proven text-only target", async () => {
+  const guardrail = createGuardrail({
+    callTranscription: async () => {
+      throw new Error("no STT connection");
+    },
+  });
+
+  const result = await guardrail.preCall(twoAudioPayload(), {});
+  const modified = result.modifiedPayload as ReturnType<typeof twoAudioPayload>;
+  assert.deepEqual(
+    modified.messages[0].content.map((part) => ("text" in part ? part.text : null)),
+    [
+      "[Audio 1]: (unavailable — no STT provider connected)",
+      "[Audio 2]: (unavailable — no STT provider connected)",
+    ]
+  );
+  assert.equal(result.meta?.clipsProcessed, 2);
+});
+
+test("missing STT credentials become stubs for a proven text-only target", async () => {
+  let calls = 0;
+  const guardrail = createGuardrail({
+    selectModel: async () => null,
+    callTranscription: async () => {
+      calls += 1;
+      return "should not run";
+    },
+  });
+
+  const result = await guardrail.preCall(audioPayload(), {});
+  const modified = result.modifiedPayload as ReturnType<typeof audioPayload>;
+  assert.equal(calls, 0);
+  assert.deepEqual(modified.messages[0].content[0], {
+    type: "text",
+    text: "[Audio 1]: (unavailable — no STT provider connected)",
+  });
+  assert.equal(result.meta?.sttModel, "unavailable");
+});
+
+test("partial STT failure preserves only the failed audio part", async () => {
+  const original = twoAudioPayload();
+  const guardrail = createGuardrail({
+    getSettings: async () => ({
+      modalityBridgeAudioEnabled: true,
+      modalityBridgeAudioModel: "deepgram/nova-3",
+      modalityBridgeCacheEnabled: false,
+    }),
+    callTranscription: async (part) => {
+      if (part.partIndex === 0) throw new Error("first failed");
+      return "second succeeded";
+    },
+  });
+
+  const result = await guardrail.preCall(original, {});
+  const modified = result.modifiedPayload as ReturnType<typeof twoAudioPayload>;
+  assert.deepEqual(modified.messages[0].content[0], original.messages[0].content[0]);
+  assert.deepEqual(modified.messages[0].content[1], {
+    type: "text",
+    text: "[Audio 2]: second succeeded",
+  });
+  assert.equal(result.meta?.clipsProcessed, 1);
+});
+
+test("unknown target capability preserves audio when every STT call fails", async () => {
+  const original = audioPayload();
+  original.messages[0].content[0].input_audio.data = "dW5rbm93bi1hdWRpbw==";
+  const snapshot = structuredClone(original);
+  const guardrail = createGuardrail({
+    getCapabilities: () => ({ supportsAudio: null }),
+    getSettings: async () => ({
+      modalityBridgeAudioEnabled: true,
+      modalityBridgeAudioModel: "deepgram/nova-3",
+      modalityBridgeCacheEnabled: false,
+    }),
+    callTranscription: async () => {
+      throw new Error("temporary STT failure");
+    },
+  });
+
+  const result = await guardrail.preCall(original, {});
+  assert.equal(result.modifiedPayload, undefined);
+  assert.deepEqual(original, snapshot, "the input object must not be mutated");
+});
+
+test("successful transcripts are reused from the shared cache", async () => {
+  let calls = 0;
+  const payload = audioPayload();
+  payload.messages[0].content[0].input_audio.data = "Y2FjaGUtdW5pcXVl";
+  const guardrail = createGuardrail({
+    callTranscription: async () => {
+      calls += 1;
+      return "cached transcript";
+    },
+  });
+
+  await guardrail.preCall(payload, {});
+  await guardrail.preCall(payload, {});
+  assert.equal(calls, 1);
+});
+
+test("maxClips limits work without dropping later audio parts", async () => {
+  const original = twoAudioPayload();
+  const guardrail = createGuardrail({
+    getSettings: async () => ({
+      modalityBridgeAudioEnabled: true,
+      modalityBridgeAudioModel: "deepgram/nova-3",
+      modalityBridgeAudioMaxClips: 1,
+      modalityBridgeCacheEnabled: false,
+    }),
+  });
+
+  const result = await guardrail.preCall(original, {});
+  const modified = result.modifiedPayload as ReturnType<typeof twoAudioPayload>;
+  assert.deepEqual(modified.messages[0].content[0], {
+    type: "text",
+    text: "[Audio 1]: hello from the clip",
+  });
+  assert.deepEqual(modified.messages[0].content[1], original.messages[0].content[1]);
+});
+
+test("default registry places Audio Bridge after Vision Bridge", () => {
+  resetGuardrailsForTests({ registerDefaults: false });
+  const names = registerDefaultGuardrails()
+    .list()
+    .map((guardrail) => guardrail.name);
+  assert.deepEqual(names.slice(0, 2), ["vision-bridge", "audio-bridge"]);
+  resetGuardrailsForTests();
+});
+
+test("audio transparency header is emitted only for transformed clips", () => {
+  assert.equal(
+    buildModalityBridgeHeader([
+      {
+        guardrail: "audio-bridge",
+        meta: { clipsProcessed: 2, sttModel: "deepgram/nova-3" },
+      },
+    ]),
+    "audio->text;model=deepgram/nova-3;parts=2"
+  );
+  assert.equal(
+    buildModalityBridgeHeader([
+      {
+        guardrail: "audio-bridge",
+        meta: { clipsProcessed: 2, sttModel: "deepgram/nova-3", rerouted: true },
+      },
+    ]),
+    null
+  );
+});