feat(translator): accept OpenAI audio input parts in Gemini translation (#4434)

Rebuilt onto release/v3.8.33 (squash-base-stale). Integrated into release/v3.8.33.

CHANGELOG.md

@@ -97,6 +97,7 @@ _In development — bullets added per PR; finalized at release._
 
 ### ✨ New Features
 
+- **feat(translator):** Gemini accepts OpenAI `input_audio` and `audio_url` content parts. (thanks @mugnimaestra)
 - **perf(dashboard): combos UI leaf-split, Next.js config tuning, 1-click Redis & Bifrost sidecar** — delivers four of the five performance/UX tracks from the #3932 thread: the combos dashboard page is split into focused leaf components (smaller bundles, faster reloads), `next.config` is tuned for the standalone build, Redis can be provisioned in one click, and a Bifrost sidecar option is wired in. (The fifth track — chatLogHelpers extraction — was already covered upstream and dropped.) ([#4381](https://github.com/diegosouzapw/OmniRoute/pull/4381) — thanks @KooshaPari)
 
 ### 🐛 Fixed

open-sse/translator/helpers/geminiHelper.ts

@@ -101,6 +101,30 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
       const rec = toRecord(item);
       if (rec.type === "text") {
         parts.push({ text: rec.text });
+      } else if (rec.type === "input_audio") {
+        // OpenAI Chat Completions audio input shape (ported from upstream
+        // decolua/9router#913): { type:"input_audio", input_audio:{data,format} }
+        // -> Gemini `inlineData: { mimeType: "audio/<format>", data }`.
+        const audio = toRecord(rec.input_audio);
+        if (typeof audio.data === "string" && audio.data) {
+          const format = typeof audio.format === "string" && audio.format ? audio.format : "wav";
+          const mimeType = format === "mp3" ? "audio/mpeg" : `audio/${format}`;
+          parts.push({ inlineData: { mimeType, data: audio.data } });
+        }
+      } else if (rec.type === "audio_url") {
+        // OpenAI-style audio_url (data: URI). Mirrors the image_url data-URL
+        // parser below but produces an audio inlineData part (#913).
+        const audioUrl = toRecord(rec.audio_url);
+        const url = typeof audioUrl.url === "string" ? audioUrl.url : "";
+        if (url.startsWith("data:")) {
+          const commaIndex = url.indexOf(",");
+          if (commaIndex !== -1) {
+            const mimePart = url.substring(5, commaIndex); // skip "data:"
+            const data = url.substring(commaIndex + 1);
+            const mimeType = mimePart.split(";")[0] || "audio/wav";
+            parts.push({ inlineData: { mimeType, data } });
+          }
+        }
       } else {
         // 0. Handle OpenAI audio input parts → Gemini inlineData (#912).
         //    Chat Completions shape: {type:"input_audio", input_audio:{data, format}}.

open-sse/translator/helpers/openaiHelper.ts

@@ -1,6 +1,8 @@
 // OpenAI helper functions for translator
 
 // Valid OpenAI content block types
+// `input_audio` / `audio_url` ported from upstream decolua/9router#913 so audio
+// parts survive `filterToOpenAIFormat()` on OpenAI-target passthrough routes.
 export const VALID_OPENAI_CONTENT_TYPES = [
   "text",
   "image_url",
@@ -8,6 +10,8 @@ export const VALID_OPENAI_CONTENT_TYPES = [
   "file_url",
   "file",
   "document",
+  "input_audio",
+  "audio_url",
 ];
 export const VALID_OPENAI_MESSAGE_TYPES = [
   "text",
@@ -17,6 +21,8 @@ export const VALID_OPENAI_MESSAGE_TYPES = [
   "file",
   "document",
   "image",
+  "input_audio",
+  "audio_url",
   "tool_calls",
   "tool_result",
 ];

tests/unit/translator-gemini-audio-input.test.ts

@@ -0,0 +1,84 @@
+// Regression for upstream PR decolua/9router#913 — OpenAI `input_audio` and
+// `audio_url` content parts must be forwarded as Gemini `inlineData` audio parts
+// (instead of being silently dropped) so that callers can send audio (WAV/MP3/etc.)
+// to Gemini models via the Antigravity / Gemini / Gemini-CLI translation paths.
+
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { convertOpenAIContentToParts } = await import(
+  "../../open-sse/translator/helpers/geminiHelper.ts"
+);
+const { VALID_OPENAI_CONTENT_TYPES, filterToOpenAIFormat } = await import(
+  "../../open-sse/translator/helpers/openaiHelper.ts"
+);
+
+type Part = { inlineData?: { mimeType: string; data: string } };
+
+test("convertOpenAIContentToParts handles input_audio with explicit wav format (#913)", () => {
+  const parts = convertOpenAIContentToParts([
+    { type: "text", text: "Transcribe this" },
+    { type: "input_audio", input_audio: { data: "UklGRiQ", format: "wav" } },
+  ]) as Part[];
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "input_audio must produce an inlineData part");
+  assert.equal(inline!.inlineData!.mimeType, "audio/wav");
+  assert.equal(inline!.inlineData!.data, "UklGRiQ");
+});
+
+test("convertOpenAIContentToParts maps input_audio mp3 -> audio/mpeg mime (#913)", () => {
+  const parts = convertOpenAIContentToParts([
+    { type: "input_audio", input_audio: { data: "SUQzBAA", format: "mp3" } },
+  ]) as Part[];
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "input_audio mp3 must produce an inlineData part");
+  assert.equal(
+    inline!.inlineData!.mimeType,
+    "audio/mpeg",
+    "mp3 must canonicalize to audio/mpeg per RFC 3003"
+  );
+});
+
+test("convertOpenAIContentToParts defaults input_audio without format to audio/wav (#913)", () => {
+  const parts = convertOpenAIContentToParts([
+    { type: "input_audio", input_audio: { data: "AAAA" } },
+  ]) as Part[];
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "input_audio without format must still produce an inlineData part");
+  assert.equal(inline!.inlineData!.mimeType, "audio/wav");
+});
+
+test("convertOpenAIContentToParts handles audio_url data URI (#913)", () => {
+  const parts = convertOpenAIContentToParts([
+    { type: "audio_url", audio_url: { url: "data:audio/wav;base64,UklGRiQ" } },
+  ]) as Part[];
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "audio_url data URI must produce an inlineData part");
+  assert.equal(inline!.inlineData!.mimeType, "audio/wav");
+  assert.equal(inline!.inlineData!.data, "UklGRiQ");
+});
+
+test("input_audio and audio_url are preserved by filterToOpenAIFormat (#913)", () => {
+  // For OpenAI-target routes (passthrough), audio parts must not be stripped
+  // out by the content-type allowlist.
+  assert.ok(VALID_OPENAI_CONTENT_TYPES.includes("input_audio"));
+  assert.ok(VALID_OPENAI_CONTENT_TYPES.includes("audio_url"));
+
+  const body = {
+    messages: [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "What is this?" },
+          { type: "input_audio", input_audio: { data: "AAAA", format: "wav" } },
+          { type: "audio_url", audio_url: { url: "data:audio/wav;base64,AAAA" } },
+        ],
+      },
+    ],
+  };
+  const filtered = filterToOpenAIFormat(body);
+  const content = filtered.messages[0].content as Array<Record<string, unknown>>;
+  const types = content.map((c) => c.type);
+  assert.ok(types.includes("input_audio"), "input_audio survives passthrough filter");
+  assert.ok(types.includes("audio_url"), "audio_url survives passthrough filter");
+});