diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index a61fd64701..4a2c8157bc 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -77,9 +77,7 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { authType: "apikey", authHeader: "bearer", format: "nvidia-asr", - models: [ - { id: "nvidia/parakeet-ctc-1.1b-asr", name: "Parakeet CTC 1.1B" }, - ], + models: [{ id: "nvidia/parakeet-ctc-1.1b-asr", name: "Parakeet CTC 1.1B" }], }, huggingface: { @@ -100,9 +98,7 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { authType: "none", authHeader: "none", format: "openai", - models: [ - { id: "qwen3-asr", name: "Qwen3 ASR" }, - ], + models: [{ id: "qwen3-asr", name: "Qwen3 ASR" }], }, }; @@ -183,9 +179,7 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "none", authHeader: "none", format: "coqui", - models: [ - { id: "tts_models/en/ljspeech/tacotron2-DDC", name: "Tacotron2 DDC (LJSpeech)" }, - ], + models: [{ id: "tts_models/en/ljspeech/tacotron2-DDC", name: "Tacotron2 DDC (LJSpeech)" }], }, tortoise: { @@ -194,9 +188,7 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "none", authHeader: "none", format: "tortoise", - models: [ - { id: "tortoise-v2", name: "Tortoise v2" }, - ], + models: [{ id: "tortoise-v2", name: "Tortoise v2" }], }, qwen: { @@ -205,8 +197,53 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { authType: "none", authHeader: "none", format: "openai", + models: [{ id: "qwen3-tts", name: "Qwen3 TTS" }], + }, + + // ── Cloud TTS Providers (#248) ──────────────────────────────────────────── + + inworld: { + id: "inworld", + // POST https://api.inworld.ai/tts/v1/voice + // Auth: Authorization: Basic + // Response: JSON { audioContent: "", contentType, sampleRateHertz } + baseUrl: "https://api.inworld.ai/tts/v1/voice", + authType: "apikey", + authHeader: "basic", + format: "inworld", models: [ - { id: "qwen3-tts", name: "Qwen3 TTS" }, + { id: "inworld-tts-1.5-max", name: "Inworld TTS 1.5 Max" }, + { id: "inworld-tts-1.5-mini", name: "Inworld TTS 1.5 Mini" }, + ], + }, + + cartesia: { + id: "cartesia", + // POST https://api.cartesia.ai/tts/bytes + // Auth: X-API-Key header, Cartesia-Version: 2024-06-10 + // Response: binary audio bytes + baseUrl: "https://api.cartesia.ai/tts/bytes", + authType: "apikey", + authHeader: "x-api-key", + format: "cartesia", + models: [ + { id: "sonic-2", name: "Sonic 2" }, + { id: "sonic-3", name: "Sonic 3" }, + ], + }, + + playht: { + id: "playht", + // POST https://api.play.ht/api/v2/tts/stream + // Auth: X-USER-ID + Authorization: Bearer + // Response: audio stream (mp3/wav) + baseUrl: "https://api.play.ht/api/v2/tts/stream", + authType: "apikey", + authHeader: "playht", + format: "playht", + models: [ + { id: "PlayDialog", name: "PlayDialog" }, + { id: "Play3.0-mini", name: "Play3.0 Mini" }, ], }, }; @@ -228,7 +265,10 @@ export function getSpeechProvider(providerId: string): AudioProvider | null { /** * Parse audio model string (format: "provider/model" or just "model") */ -function parseAudioModel(modelStr: string | null, registry: Record): { provider: string | null; model: string | null } { +function parseAudioModel( + modelStr: string | null, + registry: Record +): { provider: string | null; model: string | null } { if (!modelStr) return { provider: null, model: null }; for (const [providerId, config] of Object.entries(registry)) { diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index 3dd37fbb62..229e4eb14f 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -192,6 +192,115 @@ async function handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token) return audioStreamResponse(res, "audio/wav"); } +/** + * Handle Inworld TTS + * POST { text, voiceId, modelId, audioConfig } → JSON { audioContent: "" } + * Docs: https://docs.inworld.ai/api-reference/ttsAPI/texttospeech/synthesize-speech + */ +async function handleInworldSpeech(providerConfig, body, modelId, token) { + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Basic ${token}`, + }, + body: JSON.stringify({ + text: body.input, + voiceId: body.voice || undefined, + modelId, + audioConfig: { + audioEncoding: body.response_format === "wav" ? "LINEAR16" : "MP3", + }, + }), + }); + + if (!res.ok) { + return upstreamErrorResponse(res, await res.text()); + } + + const data = await res.json(); + // Decode base64 audioContent to binary + const audioBuffer = Uint8Array.from(atob(data.audioContent ?? ""), (c) => c.charCodeAt(0)); + const mimeType = body.response_format === "wav" ? "audio/wav" : "audio/mpeg"; + + return new Response(audioBuffer, { + status: 200, + headers: { + "Content-Type": mimeType, + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); +} + +/** + * Handle Cartesia TTS + * POST { model_id, transcript, voice, output_format } → binary audio bytes + * Docs: https://docs.cartesia.ai/api-reference/tts/bytes + */ +async function handleCartesiaSpeech(providerConfig, body, modelId, token) { + const outputFormat = + body.response_format === "wav" + ? { container: "wav", sample_rate: 44100 } + : { container: "mp3", bit_rate: 128000, sample_rate: 44100 }; + + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-API-Key": token, + "Cartesia-Version": "2024-06-10", + }, + body: JSON.stringify({ + model_id: modelId, + transcript: body.input, + ...(body.voice ? { voice: { mode: "id", id: body.voice } } : {}), + output_format: outputFormat, + }), + }); + + if (!res.ok) { + return upstreamErrorResponse(res, await res.text()); + } + + return audioStreamResponse(res); +} + +/** + * Handle PlayHT TTS + * POST { text, voice, voice_engine, output_format } → audio stream + * Auth: X-USER-ID header (from token string "userId:apiKey") + * Docs: https://docs.play.ht/reference/api-generate-tts-audio-stream + */ +async function handlePlayHtSpeech(providerConfig, body, modelId, token) { + // PlayHT tokens are stored as "userId:apiKey" + const [userId, apiKey] = (token || ":").split(":"); + + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "audio/mpeg", + "X-USER-ID": userId || "", + Authorization: `Bearer ${apiKey || token}`, + }, + body: JSON.stringify({ + text: body.input, + voice: + body.voice || + "s3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json", + voice_engine: modelId || "PlayDialog", + output_format: body.response_format || "mp3", + speed: body.speed || 1, + }), + }); + + if (!res.ok) { + return upstreamErrorResponse(res, await res.text()); + } + + return audioStreamResponse(res); +} + /** * Handle Coqui TTS (local, no auth) * POST {baseUrl} with { text, speaker_id } → WAV audio @@ -271,7 +380,7 @@ export async function handleAudioSpeech({ body, credentials }) { if (!providerConfig) { return errorResponse( 400, - `No speech provider found for model "${body.model}". Available: openai, hyperbolic, deepgram, nvidia, elevenlabs, huggingface, coqui, tortoise, qwen` + `No speech provider found for model "${body.model}". Use format provider/model. Available: openai, hyperbolic, deepgram, nvidia, elevenlabs, huggingface, inworld, cartesia, playht, coqui, tortoise, qwen` ); } @@ -304,6 +413,18 @@ export async function handleAudioSpeech({ body, credentials }) { return handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token); } + if (providerConfig.format === "inworld") { + return handleInworldSpeech(providerConfig, body, modelId, token); + } + + if (providerConfig.format === "cartesia") { + return handleCartesiaSpeech(providerConfig, body, modelId, token); + } + + if (providerConfig.format === "playht") { + return handlePlayHtSpeech(providerConfig, body, modelId, token); + } + if (providerConfig.format === "coqui") { return handleCoquiSpeech(providerConfig, body); } diff --git a/package-lock.json b/package-lock.json index cf78e613b7..53511701ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "2.0.18", + "version": "2.0.19", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "2.0.18", + "version": "2.0.19", "hasInstallScript": true, "license": "MIT", "workspaces": [ diff --git a/src/app/(dashboard)/dashboard/media/MediaPageClient.tsx b/src/app/(dashboard)/dashboard/media/MediaPageClient.tsx index 813293f5b3..431b61329c 100644 --- a/src/app/(dashboard)/dashboard/media/MediaPageClient.tsx +++ b/src/app/(dashboard)/dashboard/media/MediaPageClient.tsx @@ -3,11 +3,12 @@ import { useState } from "react"; import { useTranslations } from "next-intl"; -type Modality = "image" | "video" | "music"; +type Modality = "image" | "video" | "music" | "speech"; type GenerationResult = { type: Modality; data: any; timestamp: number; + audioUrl?: string; }; const MODALITY_CONFIG: Record< @@ -35,8 +36,70 @@ const MODALITY_CONFIG: Record< placeholder: "Upbeat electronic music with synth pads...", color: "from-orange-500 to-yellow-500", }, + speech: { + icon: "record_voice_over", + endpoint: "/api/v1/audio/speech", + label: "Text to Speech", + placeholder: "Hello! Welcome to OmniRoute, your intelligent AI gateway...", + color: "from-green-500 to-teal-500", + }, }; +// Provider-aware voice options for the TTS speech tab +const PROVIDER_VOICE_PRESETS: Record = { + // OpenAI standard voices (also used as defaults) + default: [ + { id: "alloy", label: "Alloy" }, + { id: "echo", label: "Echo" }, + { id: "fable", label: "Fable" }, + { id: "onyx", label: "Onyx" }, + { id: "nova", label: "Nova" }, + { id: "shimmer", label: "Shimmer" }, + ], + // ElevenLabs — popular premade voice IDs + elevenlabs: [ + { id: "21m00Tcm4TlvDq8ikWAM", label: "Rachel (EN, calm)" }, + { id: "AZnzlk1XvdvUeBnXmlld", label: "Domi (EN, strong)" }, + { id: "EXAVITQu4vr4xnSDxMaL", label: "Bella (EN, soft)" }, + { id: "ErXwobaYiN019PkySvjV", label: "Antoni (EN, well-rounded)" }, + { id: "MF3mGyEYCl7XYWbV9V6O", label: "Elli (EN, young)" }, + { id: "TxGEqnHWrfWFTfGW9XjX", label: "Josh (EN, deep)" }, + { id: "VR6AewLTigWG4xSOukaG", label: "Arnold (EN, crisp)" }, + { id: "pNInz6obpgDQGcFmaJgB", label: "Adam (EN, deep)" }, + { id: "yoZ06aMxZJJ28mfd3POQ", label: "Sam (EN, raspy)" }, + ], + // Cartesia — Sonic model default voice + cartesia: [ + { id: "a0e99841-438c-4a64-b679-ae501e7d6091", label: "Barbershop Man" }, + { id: "694f9389-aac1-45b6-b726-9d9369183238", label: "Friendly Reading Man" }, + { id: "b7d50908-b17c-442d-ad8d-810c63997ed9", label: "California Girl" }, + ], + // Deepgram Aura voices + deepgram: [ + { id: "aura-asteria-en", label: "Asteria (EN)" }, + { id: "aura-luna-en", label: "Luna (EN)" }, + { id: "aura-stella-en", label: "Stella (EN)" }, + { id: "aura-zeus-en", label: "Zeus (EN)" }, + { id: "aura-orion-en", label: "Orion (EN)" }, + ], + // Inworld TTS voices + inworld: [ + { id: "Eva", label: "Eva (EN)" }, + { id: "Marcus", label: "Marcus (EN)" }, + ], +}; + +function getVoicePresets(modelId: string) { + for (const prefix of Object.keys(PROVIDER_VOICE_PRESETS)) { + if (prefix !== "default" && modelId.startsWith(prefix + "/")) { + return PROVIDER_VOICE_PRESETS[prefix]; + } + } + return PROVIDER_VOICE_PRESETS.default; +} + +const SPEECH_FORMATS = ["mp3", "wav", "opus", "flac", "pcm"]; + export default function MediaPageClient() { const t = useTranslations("media"); const [activeTab, setActiveTab] = useState("image"); @@ -47,9 +110,17 @@ export default function MediaPageClient() { const [loadingModels, setLoadingModels] = useState(false); const [result, setResult] = useState(null); const [error, setError] = useState(null); + // Speech-specific state + const [speechVoice, setSpeechVoice] = useState("alloy"); + const [speechFormat, setSpeechFormat] = useState("mp3"); // Fetch available models for each modality const fetchModels = async (modality: Modality) => { + if (modality === "speech") { + // Models come from /v1/models filtered by speech providers + setModels([]); + return; + } setLoadingModels(true); try { const res = await fetch(MODALITY_CONFIG[modality].endpoint); @@ -81,6 +152,37 @@ export default function MediaPageClient() { try { const config = MODALITY_CONFIG[activeTab]; + + if (activeTab === "speech") { + // TTS: returns binary audio stream — create a Blob URL for