From 58e5ce39007fbecd9c9408042930851fffe11562 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sun, 10 May 2026 10:26:22 -0300 Subject: [PATCH] chore: enhance Inworld TTS support --- open-sse/config/audioRegistry.ts | 9 +- open-sse/config/runway.ts | 4 +- open-sse/handlers/audioSpeech.ts | 21 ++- .../dashboard/cache/media/MediaPageClient.tsx | 148 +++++++++++++++++- tests/unit/audio-speech-handler.test.ts | 61 +++++++- 5 files changed, 227 insertions(+), 16 deletions(-) diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index b0c18cafd0..359214aabb 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -17,6 +17,7 @@ export interface AudioProvider { authType: string; authHeader: string; format?: string; + supportedFormats?: string[]; async?: boolean; models: AudioModel[]; } @@ -124,8 +125,8 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "apikey", authHeader: "bearer", models: [ - { id: "tts-1", name: "TTS 1" }, { id: "tts-1-hd", name: "TTS 1 HD" }, + { id: "tts-1", name: "TTS 1" }, { id: "gpt-4o-mini-tts", name: "GPT-4o Mini TTS" }, ], }, @@ -226,8 +227,9 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "apikey", authHeader: "basic", format: "inworld", + supportedFormats: ["mp3", "wav", "opus", "pcm"], models: [ - { id: "inworld-tts-1.5-max", name: "Inworld TTS 1.5 Max" }, + { id: "inworld-tts-2", name: "Inworld TTS 2" }, { id: "inworld-tts-1.5-mini", name: "Inworld TTS 1.5 Mini" }, ], }, @@ -242,8 +244,8 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authHeader: "x-api-key", format: "cartesia", models: [ - { id: "sonic-2", name: "Sonic 2" }, { id: "sonic-3", name: "Sonic 3" }, + { id: "sonic-2", name: "Sonic 2" }, ], }, @@ -297,6 +299,7 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "apikey", authHeader: "bearer", format: "xiaomi-mimo-tts", + supportedFormats: ["mp3", "wav"], models: [ { id: "mimo-v2.5-tts", name: "MiMo V2.5 TTS" }, { id: "mimo-v2.5-tts-voicedesign", name: "MiMo V2.5 Voice Design" }, diff --git a/open-sse/config/runway.ts b/open-sse/config/runway.ts index a76719d757..165a9af4c2 100644 --- a/open-sse/config/runway.ts +++ b/open-sse/config/runway.ts @@ -6,11 +6,9 @@ export const RUNWAYML_SUPPORTED_VIDEO_MODELS = [ { id: "gen4_turbo", name: "Gen-4 Turbo" }, { id: "veo3.1", name: "Veo 3.1" }, { id: "veo3.1_fast", name: "Veo 3.1 Fast" }, - { id: "veo3", name: "Veo 3" }, - { id: "gen3a_turbo", name: "Gen-3 Alpha Turbo" }, ]; -export const RUNWAYML_IMAGE_REQUIRED_MODELS = new Set(["gen4_turbo", "gen3a_turbo"]); +export const RUNWAYML_IMAGE_REQUIRED_MODELS = new Set(["gen4_turbo"]); export function normalizeRunwayBaseUrl(baseUrl?: string | null) { const normalized = String(baseUrl || "") diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index 2342a6d9ea..ffa2542aa8 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -428,7 +428,21 @@ async function handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token) * POST { text, voiceId, modelId, audioConfig } → JSON { audioContent: "" } * Docs: https://docs.inworld.ai/api-reference/ttsAPI/texttospeech/synthesize-speech */ +const INWORLD_AUDIO_FORMATS = { + mp3: { audioEncoding: "MP3", mimeType: "audio/mpeg" }, + wav: { audioEncoding: "WAV", mimeType: "audio/wav" }, + opus: { audioEncoding: "OPUS", mimeType: "audio/opus" }, + pcm: { audioEncoding: "PCM", mimeType: "audio/pcm" }, +}; + async function handleInworldSpeech(providerConfig, body, modelId, token) { + const requestedFormat = + typeof body.response_format === "string" ? body.response_format.toLowerCase() : "mp3"; + const audioFormat = INWORLD_AUDIO_FORMATS[requestedFormat]; + if (!audioFormat) { + return errorResponse(400, "Inworld TTS supports response_format mp3, wav, opus, or pcm only"); + } + const res = await fetch(providerConfig.baseUrl, { method: "POST", headers: { @@ -440,7 +454,7 @@ async function handleInworldSpeech(providerConfig, body, modelId, token) { voiceId: body.voice || undefined, modelId, audioConfig: { - audioEncoding: body.response_format === "wav" ? "LINEAR16" : "MP3", + audioEncoding: audioFormat.audioEncoding, }, }), }); @@ -452,7 +466,10 @@ async function handleInworldSpeech(providerConfig, body, modelId, token) { const data = await res.json(); // Decode base64 audioContent to binary const audioBuffer = Uint8Array.from(atob(data.audioContent ?? ""), (c) => c.charCodeAt(0)); - const mimeType = body.response_format === "wav" ? "audio/wav" : "audio/mpeg"; + const mimeType = + typeof data.contentType === "string" && data.contentType + ? data.contentType + : audioFormat.mimeType; return new Response(audioBuffer, { status: 200, diff --git a/src/app/(dashboard)/dashboard/cache/media/MediaPageClient.tsx b/src/app/(dashboard)/dashboard/cache/media/MediaPageClient.tsx index 5b2ba40bea..ae5b69f32e 100644 --- a/src/app/(dashboard)/dashboard/cache/media/MediaPageClient.tsx +++ b/src/app/(dashboard)/dashboard/cache/media/MediaPageClient.tsx @@ -23,6 +23,7 @@ type MediaModelConfig = { id: string; name: string }; type MediaProviderConfig = { id: string; authType: string; + supportedFormats?: string[]; models: MediaModelConfig[]; }; type ProviderModelGroup = { @@ -172,8 +173,147 @@ const VOICE_PRESETS: Record = { { id: "aura-orion-en", label: "Orion (EN)" }, ], inworld: [ - { id: "Eva", label: "Eva (EN)" }, + { id: "Abby", label: "Abby (EN)" }, + { id: "Alex", label: "Alex (EN)" }, + { id: "Amina", label: "Amina (EN)" }, + { id: "Anjali", label: "Anjali (EN)" }, + { id: "Arjun", label: "Arjun (EN)" }, + { id: "Ashley", label: "Ashley (EN)" }, + { id: "Avery", label: "Avery (EN)" }, + { id: "Bianca", label: "Bianca (EN)" }, + { id: "Blake", label: "Blake (EN)" }, + { id: "Brandon", label: "Brandon (EN)" }, + { id: "Brian", label: "Brian (EN)" }, + { id: "Callum", label: "Callum (EN)" }, + { id: "Carter", label: "Carter (EN)" }, + { id: "Cedric", label: "Cedric (EN)" }, + { id: "Celeste", label: "Celeste (EN)" }, + { id: "Chloe", label: "Chloe (EN)" }, + { id: "Claire", label: "Claire (EN)" }, + { id: "Clive", label: "Clive (EN)" }, + { id: "Conrad", label: "Conrad (EN)" }, + { id: "Craig", label: "Craig (EN)" }, + { id: "Damon", label: "Damon (EN)" }, + { id: "Darlene", label: "Darlene (EN)" }, + { id: "Deborah", label: "Deborah (EN)" }, + { id: "Dennis", label: "Dennis (EN)" }, + { id: "Derek", label: "Derek (EN)" }, + { id: "Dominus", label: "Dominus (EN)" }, + { id: "Duncan", label: "Duncan (EN)" }, + { id: "Edward", label: "Edward (EN)" }, + { id: "Eleanor", label: "Eleanor (EN)" }, + { id: "Elliot", label: "Elliot (EN)" }, + { id: "Ethan", label: "Ethan (EN)" }, + { id: "Evan", label: "Evan (EN)" }, + { id: "Evelyn", label: "Evelyn (EN)" }, + { id: "Felix", label: "Felix (EN)" }, + { id: "Gareth", label: "Gareth (EN)" }, + { id: "Graham", label: "Graham (EN)" }, + { id: "Hades", label: "Hades (EN)" }, + { id: "Hamish", label: "Hamish (EN)" }, + { id: "Hana", label: "Hana (EN)" }, + { id: "Hank", label: "Hank (EN)" }, + { id: "James", label: "James (EN)" }, + { id: "Jason", label: "Jason (EN)" }, + { id: "Jessica", label: "Jessica (EN)" }, + { id: "Jonah", label: "Jonah (EN)" }, + { id: "Kelsey", label: "Kelsey (EN)" }, + { id: "Lauren", label: "Lauren (EN)" }, + { id: "Levi", label: "Levi (EN)" }, + { id: "Liam", label: "Liam (EN)" }, + { id: "Loretta", label: "Loretta (EN)" }, + { id: "Lucian", label: "Lucian (EN)" }, + { id: "Luna", label: "Luna (EN)" }, + { id: "Malcolm", label: "Malcolm (EN)" }, { id: "Marcus", label: "Marcus (EN)" }, + { id: "Mark", label: "Mark (EN)" }, + { id: "Marlene", label: "Marlene (EN)" }, + { id: "Mia", label: "Mia (EN)" }, + { id: "Miranda", label: "Miranda (EN)" }, + { id: "Mortimer", label: "Mortimer (EN)" }, + { id: "Nadia", label: "Nadia (EN)" }, + { id: "Naomi", label: "Naomi (EN)" }, + { id: "Nate", label: "Nate (EN)" }, + { id: "Oliver", label: "Oliver (EN)" }, + { id: "Olivia", label: "Olivia (EN)" }, + { id: "Pippa", label: "Pippa (EN)" }, + { id: "Pixie", label: "Pixie (EN)" }, + { id: "Reed", label: "Reed (EN)" }, + { id: "Riley", label: "Riley (EN)" }, + { id: "Ronald", label: "Ronald (EN)" }, + { id: "Rupert", label: "Rupert (EN)" }, + { id: "Saanvi", label: "Saanvi (EN)" }, + { id: "Sarah", label: "Sarah (EN)" }, + { id: "Sebastian", label: "Sebastian (EN)" }, + { id: "Selene", label: "Selene (EN)" }, + { id: "Serena", label: "Serena (EN)" }, + { id: "Simon", label: "Simon (EN)" }, + { id: "Snik", label: "Snik (EN)" }, + { id: "Sophie", label: "Sophie (EN)" }, + { id: "Tessa", label: "Tessa (EN)" }, + { id: "Theodore", label: "Theodore (EN)" }, + { id: "Timothy", label: "Timothy (EN)" }, + { id: "Trevor", label: "Trevor (EN)" }, + { id: "Tristan", label: "Tristan (EN)" }, + { id: "Tyler", label: "Tyler (EN)" }, + { id: "Veronica", label: "Veronica (EN)" }, + { id: "Victor", label: "Victor (EN)" }, + { id: "Victoria", label: "Victoria (EN)" }, + { id: "Vinny", label: "Vinny (EN)" }, + { id: "Wendy", label: "Wendy (EN)" }, + { id: "Aanya", label: "Aanya (HI)" }, + { id: "Aarav", label: "Aarav (HI)" }, + { id: "Manoj", label: "Manoj (HI)" }, + { id: "Riya", label: "Riya (HI)" }, + { id: "Alain", label: "Alain (FR)" }, + { id: "Étienne", label: "Étienne (FR)" }, + { id: "Hélène", label: "Hélène (FR)" }, + { id: "Mathieu", label: "Mathieu (FR)" }, + { id: "Asuka", label: "Asuka (JP)" }, + { id: "Haruto", label: "Haruto (JP)" }, + { id: "Hina", label: "Hina (JP)" }, + { id: "Satoshi", label: "Satoshi (JP)" }, + { id: "Beatriz", label: "Beatriz (PT)" }, + { id: "Heitor", label: "Heitor (PT)" }, + { id: "Maitê", label: "Maitê (PT)" }, + { id: "Mariana", label: "Mariana (PT)" }, + { id: "Murilo", label: "Murilo (PT)" }, + { id: "Camila", label: "Camila (ES)" }, + { id: "Diego", label: "Diego (ES)" }, + { id: "Lupita", label: "Lupita (ES)" }, + { id: "Mateo", label: "Mateo (ES)" }, + { id: "Mauricio", label: "Mauricio (ES)" }, + { id: "Miguel", label: "Miguel (ES)" }, + { id: "Rafael", label: "Rafael (ES)" }, + { id: "Sofia", label: "Sofia (ES)" }, + { id: "Dmitry", label: "Dmitry (RU)" }, + { id: "Elena", label: "Elena (RU)" }, + { id: "Nikolai", label: "Nikolai (RU)" }, + { id: "Svetlana", label: "Svetlana (RU)" }, + { id: "Erik", label: "Erik (NL)" }, + { id: "Katrien", label: "Katrien (NL)" }, + { id: "Lennart", label: "Lennart (NL)" }, + { id: "Lore", label: "Lore (NL)" }, + { id: "Gianni", label: "Gianni (IT)" }, + { id: "Orietta", label: "Orietta (IT)" }, + { id: "Hyunwoo", label: "Hyunwoo (KO)" }, + { id: "Minji", label: "Minji (KO)" }, + { id: "Seojun", label: "Seojun (KO)" }, + { id: "Yoona", label: "Yoona (KO)" }, + { id: "Jing", label: "Jing (ZH)" }, + { id: "Mei", label: "Mei (ZH)" }, + { id: "Ming", label: "Ming (ZH)" }, + { id: "Xiaoyin", label: "Xiaoyin (ZH)" }, + { id: "Xinyi", label: "Xinyi (ZH)" }, + { id: "Yichen", label: "Yichen (ZH)" }, + { id: "Johanna", label: "Johanna (DE)" }, + { id: "Josef", label: "Josef (DE)" }, + { id: "Nour", label: "Nour (AR)" }, + { id: "Omar", label: "Omar (AR)" }, + { id: "Oren", label: "Oren (HE)" }, + { id: "Yael", label: "Yael (HE)" }, + { id: "Szymon", label: "Szymon (PL)" }, + { id: "Wojciech", label: "Wojciech (PL)" }, ], "xiaomi-mimo": [ { id: "冰糖", label: "冰糖 (Chinese Female)" }, @@ -188,12 +328,10 @@ const VOICE_PRESETS: Record = { }; const SPEECH_FORMATS = ["mp3", "wav", "opus", "flac", "pcm"]; -const SPEECH_FORMATS_BY_PROVIDER: Record = { - "xiaomi-mimo": ["mp3", "wav"], -}; function getSpeechFormats(providerId: string): string[] { - return SPEECH_FORMATS_BY_PROVIDER[providerId] || SPEECH_FORMATS; + const providerFormats = AUDIO_SPEECH_PROVIDERS[providerId]?.supportedFormats; + return providerFormats?.length ? providerFormats : SPEECH_FORMATS; } function getVoiceList(providerId: string) { diff --git a/tests/unit/audio-speech-handler.test.ts b/tests/unit/audio-speech-handler.test.ts index 0cd3f39ead..49d24998c9 100644 --- a/tests/unit/audio-speech-handler.test.ts +++ b/tests/unit/audio-speech-handler.test.ts @@ -2,6 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; const { handleAudioSpeech } = await import("../../open-sse/handlers/audioSpeech.ts"); +const { AUDIO_SPEECH_PROVIDERS } = await import("../../open-sse/config/audioRegistry.ts"); test("handleAudioSpeech requires model", async () => { const response = await handleAudioSpeech({ @@ -486,7 +487,7 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy try { const response = await handleAudioSpeech({ body: { - model: "inworld/inworld-tts-1.5-max", + model: "inworld/inworld-tts-2", input: "inworld text", voice: "voice-9", response_format: "wav", @@ -498,8 +499,8 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy assert.deepEqual(captured.body, { text: "inworld text", voiceId: "voice-9", - modelId: "inworld-tts-1.5-max", - audioConfig: { audioEncoding: "LINEAR16" }, + modelId: "inworld-tts-2", + audioConfig: { audioEncoding: "WAV" }, }); assert.equal(response.headers.get("content-type"), "audio/wav"); assert.deepEqual(Array.from(new Uint8Array(await response.arrayBuffer())), [1, 2, 3, 4]); @@ -508,6 +509,60 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy } }); +test("Inworld speech registry exposes supported formats without flac", () => { + assert.deepEqual(AUDIO_SPEECH_PROVIDERS.inworld.supportedFormats, ["mp3", "wav", "opus", "pcm"]); +}); + +test("handleAudioSpeech maps Inworld opus output and rejects flac", async () => { + const originalFetch = globalThis.fetch; + let captured; + let callCount = 0; + + globalThis.fetch = async (_url, options = {}) => { + callCount += 1; + captured = JSON.parse(String(options.body || "{}")); + + return new Response(JSON.stringify({ audioContent: "BQY=", contentType: "audio/opus" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const opusResponse = await handleAudioSpeech({ + body: { + model: "inworld/inworld-tts-2", + input: "inworld text", + response_format: "opus", + }, + credentials: { apiKey: "encoded-basic-token" }, + }); + + assert.deepEqual(captured.audioConfig, { audioEncoding: "OPUS" }); + assert.equal(opusResponse.headers.get("content-type"), "audio/opus"); + assert.deepEqual(Array.from(new Uint8Array(await opusResponse.arrayBuffer())), [5, 6]); + + const flacResponse = await handleAudioSpeech({ + body: { + model: "inworld/inworld-tts-2", + input: "inworld text", + response_format: "flac", + }, + credentials: { apiKey: "encoded-basic-token" }, + }); + const payload = (await flacResponse.json()) as any; + + assert.equal(flacResponse.status, 400); + assert.equal( + payload.error.message, + "Inworld TTS supports response_format mp3, wav, opus, or pcm only" + ); + assert.equal(callCount, 1); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("handleAudioSpeech supports local Coqui providers without credentials", async () => { const originalFetch = globalThis.fetch; let captured;