diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e6802b6f..40f36fc96a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,7 @@ _In development — bullets added per PR; finalized at release._ ### ✨ New Features +- **feat(translator):** Gemini accepts OpenAI `input_audio` and `audio_url` content parts. (thanks @mugnimaestra) - **perf(dashboard): combos UI leaf-split, Next.js config tuning, 1-click Redis & Bifrost sidecar** — delivers four of the five performance/UX tracks from the #3932 thread: the combos dashboard page is split into focused leaf components (smaller bundles, faster reloads), `next.config` is tuned for the standalone build, Redis can be provisioned in one click, and a Bifrost sidecar option is wired in. (The fifth track — chatLogHelpers extraction — was already covered upstream and dropped.) ([#4381](https://github.com/diegosouzapw/OmniRoute/pull/4381) — thanks @KooshaPari) ### 🐛 Fixed diff --git a/open-sse/translator/helpers/geminiHelper.ts b/open-sse/translator/helpers/geminiHelper.ts index 1efb572181..7480a6f9cd 100644 --- a/open-sse/translator/helpers/geminiHelper.ts +++ b/open-sse/translator/helpers/geminiHelper.ts @@ -101,6 +101,30 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] { const rec = toRecord(item); if (rec.type === "text") { parts.push({ text: rec.text }); + } else if (rec.type === "input_audio") { + // OpenAI Chat Completions audio input shape (ported from upstream + // decolua/9router#913): { type:"input_audio", input_audio:{data,format} } + // -> Gemini `inlineData: { mimeType: "audio/", data }`. + const audio = toRecord(rec.input_audio); + if (typeof audio.data === "string" && audio.data) { + const format = typeof audio.format === "string" && audio.format ? audio.format : "wav"; + const mimeType = format === "mp3" ? "audio/mpeg" : `audio/${format}`; + parts.push({ inlineData: { mimeType, data: audio.data } }); + } + } else if (rec.type === "audio_url") { + // OpenAI-style audio_url (data: URI). Mirrors the image_url data-URL + // parser below but produces an audio inlineData part (#913). + const audioUrl = toRecord(rec.audio_url); + const url = typeof audioUrl.url === "string" ? audioUrl.url : ""; + if (url.startsWith("data:")) { + const commaIndex = url.indexOf(","); + if (commaIndex !== -1) { + const mimePart = url.substring(5, commaIndex); // skip "data:" + const data = url.substring(commaIndex + 1); + const mimeType = mimePart.split(";")[0] || "audio/wav"; + parts.push({ inlineData: { mimeType, data } }); + } + } } else { // 0. Handle OpenAI audio input parts → Gemini inlineData (#912). // Chat Completions shape: {type:"input_audio", input_audio:{data, format}}. diff --git a/open-sse/translator/helpers/openaiHelper.ts b/open-sse/translator/helpers/openaiHelper.ts index f05ca0a857..b53bbb64e7 100644 --- a/open-sse/translator/helpers/openaiHelper.ts +++ b/open-sse/translator/helpers/openaiHelper.ts @@ -1,6 +1,8 @@ // OpenAI helper functions for translator // Valid OpenAI content block types +// `input_audio` / `audio_url` ported from upstream decolua/9router#913 so audio +// parts survive `filterToOpenAIFormat()` on OpenAI-target passthrough routes. export const VALID_OPENAI_CONTENT_TYPES = [ "text", "image_url", @@ -8,6 +10,8 @@ export const VALID_OPENAI_CONTENT_TYPES = [ "file_url", "file", "document", + "input_audio", + "audio_url", ]; export const VALID_OPENAI_MESSAGE_TYPES = [ "text", @@ -17,6 +21,8 @@ export const VALID_OPENAI_MESSAGE_TYPES = [ "file", "document", "image", + "input_audio", + "audio_url", "tool_calls", "tool_result", ]; diff --git a/tests/unit/translator-gemini-audio-input.test.ts b/tests/unit/translator-gemini-audio-input.test.ts new file mode 100644 index 0000000000..12ca3042fd --- /dev/null +++ b/tests/unit/translator-gemini-audio-input.test.ts @@ -0,0 +1,84 @@ +// Regression for upstream PR decolua/9router#913 — OpenAI `input_audio` and +// `audio_url` content parts must be forwarded as Gemini `inlineData` audio parts +// (instead of being silently dropped) so that callers can send audio (WAV/MP3/etc.) +// to Gemini models via the Antigravity / Gemini / Gemini-CLI translation paths. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { convertOpenAIContentToParts } = await import( + "../../open-sse/translator/helpers/geminiHelper.ts" +); +const { VALID_OPENAI_CONTENT_TYPES, filterToOpenAIFormat } = await import( + "../../open-sse/translator/helpers/openaiHelper.ts" +); + +type Part = { inlineData?: { mimeType: string; data: string } }; + +test("convertOpenAIContentToParts handles input_audio with explicit wav format (#913)", () => { + const parts = convertOpenAIContentToParts([ + { type: "text", text: "Transcribe this" }, + { type: "input_audio", input_audio: { data: "UklGRiQ", format: "wav" } }, + ]) as Part[]; + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "input_audio must produce an inlineData part"); + assert.equal(inline!.inlineData!.mimeType, "audio/wav"); + assert.equal(inline!.inlineData!.data, "UklGRiQ"); +}); + +test("convertOpenAIContentToParts maps input_audio mp3 -> audio/mpeg mime (#913)", () => { + const parts = convertOpenAIContentToParts([ + { type: "input_audio", input_audio: { data: "SUQzBAA", format: "mp3" } }, + ]) as Part[]; + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "input_audio mp3 must produce an inlineData part"); + assert.equal( + inline!.inlineData!.mimeType, + "audio/mpeg", + "mp3 must canonicalize to audio/mpeg per RFC 3003" + ); +}); + +test("convertOpenAIContentToParts defaults input_audio without format to audio/wav (#913)", () => { + const parts = convertOpenAIContentToParts([ + { type: "input_audio", input_audio: { data: "AAAA" } }, + ]) as Part[]; + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "input_audio without format must still produce an inlineData part"); + assert.equal(inline!.inlineData!.mimeType, "audio/wav"); +}); + +test("convertOpenAIContentToParts handles audio_url data URI (#913)", () => { + const parts = convertOpenAIContentToParts([ + { type: "audio_url", audio_url: { url: "data:audio/wav;base64,UklGRiQ" } }, + ]) as Part[]; + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "audio_url data URI must produce an inlineData part"); + assert.equal(inline!.inlineData!.mimeType, "audio/wav"); + assert.equal(inline!.inlineData!.data, "UklGRiQ"); +}); + +test("input_audio and audio_url are preserved by filterToOpenAIFormat (#913)", () => { + // For OpenAI-target routes (passthrough), audio parts must not be stripped + // out by the content-type allowlist. + assert.ok(VALID_OPENAI_CONTENT_TYPES.includes("input_audio")); + assert.ok(VALID_OPENAI_CONTENT_TYPES.includes("audio_url")); + + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What is this?" }, + { type: "input_audio", input_audio: { data: "AAAA", format: "wav" } }, + { type: "audio_url", audio_url: { url: "data:audio/wav;base64,AAAA" } }, + ], + }, + ], + }; + const filtered = filterToOpenAIFormat(body); + const content = filtered.messages[0].content as Array>; + const types = content.map((c) => c.type); + assert.ok(types.includes("input_audio"), "input_audio survives passthrough filter"); + assert.ok(types.includes("audio_url"), "audio_url survives passthrough filter"); +});