From 4a779dfe3cd275772a15c738bd005e9757ce874b Mon Sep 17 00:00:00 2001 From: duongvdo Date: Sun, 1 Mar 2026 13:31:29 +0700 Subject: [PATCH] feat: add new providers & modalities (TTS, STT, Image, Video, Music) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 6 TTS providers (Nvidia NIM, ElevenLabs, HuggingFace, Coqui, Tortoise, Qwen3), 3 STT providers (Nvidia NIM, HuggingFace, Qwen3), 2 local image providers (SD WebUI, ComfyUI), and two new modalities — Text-to-Video (/v1/videos/generations) and Text-to-Music (/v1/music/generations). Key design decisions: - Format-based unified providers: local providers grouped by API format (comfyui, sdwebui, coqui, tortoise, openai-compatible) with configurable base URLs and expandable model lists - Cloud providers kept separate (unique auth and API shapes) - Local providers use authType: "none" — credential checks bypassed at both route and handler level - Shared ComfyUI client (comfyuiClient.ts) reused across image/video/music - Shared registry utilities (registryUtils.ts) for model parsing and listing - Qwen3 TTS/ASR use format: "openai" — no custom handler needed Co-Authored-By: Claude Opus 4.6 --- open-sse/config/audioRegistry.ts | 101 ++++++++ open-sse/config/imageRegistry.ts | 26 ++ open-sse/config/musicRegistry.ts | 57 +++++ open-sse/config/registryUtils.ts | 98 ++++++++ open-sse/config/videoRegistry.ts | 68 +++++ open-sse/handlers/audioSpeech.ts | 247 +++++++++++++++++-- open-sse/handlers/audioTranscription.ts | 108 ++++++-- open-sse/handlers/imageGeneration.ts | 204 +++++++++++++++ open-sse/handlers/musicGeneration.ts | 138 +++++++++++ open-sse/handlers/videoGeneration.ts | 206 ++++++++++++++++ open-sse/index.ts | 25 ++ open-sse/utils/comfyuiClient.ts | 103 ++++++++ src/app/api/v1/audio/speech/route.ts | 15 +- src/app/api/v1/audio/transcriptions/route.ts | 15 +- src/app/api/v1/images/generations/route.ts | 16 +- src/app/api/v1/models/route.ts | 24 ++ src/app/api/v1/music/generations/route.ts | 117 +++++++++ src/app/api/v1/videos/generations/route.ts | 117 +++++++++ 18 files changed, 1634 insertions(+), 51 deletions(-) create mode 100644 open-sse/config/musicRegistry.ts create mode 100644 open-sse/config/registryUtils.ts create mode 100644 open-sse/config/videoRegistry.ts create mode 100644 open-sse/handlers/musicGeneration.ts create mode 100644 open-sse/handlers/videoGeneration.ts create mode 100644 open-sse/utils/comfyuiClient.ts create mode 100644 src/app/api/v1/music/generations/route.ts create mode 100644 src/app/api/v1/videos/generations/route.ts diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index cf6e11e0b0..dee3994671 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -70,6 +70,39 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { { id: "universal-2", name: "Universal 2" }, ], }, + + nvidia: { + id: "nvidia", + baseUrl: "https://integrate.api.nvidia.com/v1/audio/transcriptions", + authType: "apikey", + authHeader: "bearer", + format: "nvidia-asr", + models: [ + { id: "nvidia/parakeet-ctc-1.1b-asr", name: "Parakeet CTC 1.1B" }, + ], + }, + + huggingface: { + id: "huggingface", + baseUrl: "https://api-inference.huggingface.co/models", + authType: "apikey", + authHeader: "bearer", + format: "huggingface-asr", + models: [ + { id: "openai/whisper-large-v3", name: "Whisper Large v3 (HF)" }, + { id: "openai/whisper-small", name: "Whisper Small (HF)" }, + ], + }, + + qwen: { + id: "qwen", + baseUrl: "http://localhost:8000/v1/audio/transcriptions", + authType: "none", + authHeader: "none", + models: [ + { id: "qwen3-asr", name: "Qwen3 ASR" }, + ], + }, }; export const AUDIO_SPEECH_PROVIDERS: Record = { @@ -106,6 +139,74 @@ export const AUDIO_SPEECH_PROVIDERS: Record = { { id: "aura-stella-en", name: "Aura Stella (EN)" }, ], }, + + nvidia: { + id: "nvidia", + baseUrl: "https://integrate.api.nvidia.com/v1/audio/speech", + authType: "apikey", + authHeader: "bearer", + format: "nvidia-tts", + models: [ + { id: "nvidia/fastpitch", name: "FastPitch" }, + { id: "nvidia/tacotron2", name: "Tacotron2" }, + ], + }, + + elevenlabs: { + id: "elevenlabs", + baseUrl: "https://api.elevenlabs.io/v1/text-to-speech", + authType: "apikey", + authHeader: "xi-api-key", + format: "elevenlabs", + models: [ + { id: "eleven_multilingual_v2", name: "Eleven Multilingual v2" }, + { id: "eleven_turbo_v2_5", name: "Eleven Turbo v2.5" }, + ], + }, + + huggingface: { + id: "huggingface", + baseUrl: "https://api-inference.huggingface.co/models", + authType: "apikey", + authHeader: "bearer", + format: "huggingface-tts", + models: [ + { id: "facebook/mms-tts-eng", name: "MMS TTS English" }, + { id: "microsoft/speecht5_tts", name: "SpeechT5 TTS" }, + ], + }, + + coqui: { + id: "coqui", + baseUrl: "http://localhost:5002/api/tts", + authType: "none", + authHeader: "none", + format: "coqui", + models: [ + { id: "tts_models/en/ljspeech/tacotron2-DDC", name: "Tacotron2 DDC (LJSpeech)" }, + ], + }, + + tortoise: { + id: "tortoise", + baseUrl: "http://localhost:5000/api/tts", + authType: "none", + authHeader: "none", + format: "tortoise", + models: [ + { id: "tortoise-v2", name: "Tortoise v2" }, + ], + }, + + qwen: { + id: "qwen", + baseUrl: "http://localhost:8000/v1/audio/speech", + authType: "none", + authHeader: "none", + models: [ + { id: "qwen3-tts", name: "Qwen3 TTS" }, + ], + }, }; /** diff --git a/open-sse/config/imageRegistry.ts b/open-sse/config/imageRegistry.ts index 48240fa422..21b1bb2119 100644 --- a/open-sse/config/imageRegistry.ts +++ b/open-sse/config/imageRegistry.ts @@ -108,6 +108,32 @@ export const IMAGE_PROVIDERS = { ], supportedSizes: ["1024x1024"], }, + + sdwebui: { + id: "sdwebui", + baseUrl: "http://localhost:7860/sdapi/v1/txt2img", + authType: "none", + authHeader: "none", + format: "sdwebui", + models: [ + { id: "stable-diffusion-v1-5", name: "Stable Diffusion v1.5" }, + { id: "sdxl-base-1.0", name: "SDXL Base 1.0" }, + ], + supportedSizes: ["512x512", "768x768", "1024x1024"], + }, + + comfyui: { + id: "comfyui", + baseUrl: "http://localhost:8188", + authType: "none", + authHeader: "none", + format: "comfyui", + models: [ + { id: "flux-dev", name: "FLUX Dev" }, + { id: "sdxl", name: "SDXL" }, + ], + supportedSizes: ["512x512", "768x768", "1024x1024"], + }, }; /** diff --git a/open-sse/config/musicRegistry.ts b/open-sse/config/musicRegistry.ts new file mode 100644 index 0000000000..40cd950308 --- /dev/null +++ b/open-sse/config/musicRegistry.ts @@ -0,0 +1,57 @@ +/** + * Music Generation Provider Registry + * + * Defines providers that support the /v1/music/generations endpoint. + * Currently supports local providers (ComfyUI with audio models). + */ + +import { parseModelFromRegistry, getAllModelsFromRegistry } from "./registryUtils.ts"; + +interface MusicModel { + id: string; + name: string; +} + +interface MusicProvider { + id: string; + baseUrl: string; + authType: string; + authHeader: string; + format: string; + models: MusicModel[]; +} + +export const MUSIC_PROVIDERS: Record = { + comfyui: { + id: "comfyui", + baseUrl: "http://localhost:8188", + authType: "none", + authHeader: "none", + format: "comfyui", + models: [ + { id: "stable-audio-open", name: "Stable Audio Open" }, + { id: "musicgen-medium", name: "MusicGen Medium" }, + ], + }, +}; + +/** + * Get music provider config by ID + */ +export function getMusicProvider(providerId: string): MusicProvider | null { + return MUSIC_PROVIDERS[providerId] || null; +} + +/** + * Parse music model string (format: "provider/model" or just "model") + */ +export function parseMusicModel(modelStr: string | null) { + return parseModelFromRegistry(modelStr, MUSIC_PROVIDERS); +} + +/** + * Get all music models as a flat list + */ +export function getAllMusicModels() { + return getAllModelsFromRegistry(MUSIC_PROVIDERS); +} diff --git a/open-sse/config/registryUtils.ts b/open-sse/config/registryUtils.ts new file mode 100644 index 0000000000..cd20cc6f46 --- /dev/null +++ b/open-sse/config/registryUtils.ts @@ -0,0 +1,98 @@ +/** + * Shared Registry Utilities + * + * Common interfaces and helpers used by all provider registries + * (audio, image, video, music). Extracts duplicated patterns into + * reusable functions. + */ + +export interface BaseModel { + id: string; + name: string; +} + +export interface BaseProvider { + id: string; + baseUrl: string; + authType: string; // "apikey" | "oauth" | "none" + authHeader: string; // "bearer" | "token" | "xi-api-key" | "x-api-key" | "none" + format?: string; + models: M[]; +} + +/** + * Parse a "provider/model" string against a registry. + * Supports both "provider/model" prefix and bare "model" lookup. + */ +export function parseModelFromRegistry

( + modelStr: string | null, + registry: Record +): { provider: string | null; model: string | null } { + if (!modelStr) return { provider: null, model: null }; + + // Try each provider prefix + for (const [providerId] of Object.entries(registry)) { + if (modelStr.startsWith(providerId + "/")) { + return { provider: providerId, model: modelStr.slice(providerId.length + 1) }; + } + } + + // No provider prefix — try to find the model in any provider + for (const [providerId, config] of Object.entries(registry)) { + if (config.models.some((m) => m.id === modelStr)) { + return { provider: providerId, model: modelStr }; + } + } + + return { provider: null, model: modelStr }; +} + +/** + * Flatten all models from a registry into a list with provider info. + * Optionally merge extra fields per provider via the `extra` callback. + */ +export function getAllModelsFromRegistry

( + registry: Record, + extra?: (providerId: string, config: P) => Record +): Array<{ id: string; name: string; provider: string } & Record> { + const models: Array<{ id: string; name: string; provider: string } & Record> = []; + + for (const [providerId, config] of Object.entries(registry)) { + const extraFields = extra ? extra(providerId, config) : {}; + for (const model of config.models) { + models.push({ + id: `${providerId}/${model.id}`, + name: model.name, + provider: providerId, + ...extraFields, + }); + } + } + + return models; +} + +/** + * Build auth headers for a provider. + * Handles bearer, token, xi-api-key, x-api-key, and none. + */ +export function buildAuthHeaders( + provider: BaseProvider, + token: string | null +): Record { + if (provider.authType === "none" || provider.authHeader === "none" || !token) { + return {}; + } + + switch (provider.authHeader) { + case "token": + return { Authorization: `Token ${token}` }; + case "xi-api-key": + return { "xi-api-key": token }; + case "x-api-key": + return { "x-api-key": token }; + case "bearer": + default: + return { Authorization: `Bearer ${token}` }; + } +} diff --git a/open-sse/config/videoRegistry.ts b/open-sse/config/videoRegistry.ts new file mode 100644 index 0000000000..8d640efe57 --- /dev/null +++ b/open-sse/config/videoRegistry.ts @@ -0,0 +1,68 @@ +/** + * Video Generation Provider Registry + * + * Defines providers that support the /v1/videos/generations endpoint. + * Currently supports local providers (ComfyUI, SD WebUI with AnimateDiff). + */ + +import { parseModelFromRegistry, getAllModelsFromRegistry } from "./registryUtils.ts"; + +interface VideoModel { + id: string; + name: string; +} + +interface VideoProvider { + id: string; + baseUrl: string; + authType: string; + authHeader: string; + format: string; + models: VideoModel[]; +} + +export const VIDEO_PROVIDERS: Record = { + comfyui: { + id: "comfyui", + baseUrl: "http://localhost:8188", + authType: "none", + authHeader: "none", + format: "comfyui", + models: [ + { id: "animatediff", name: "AnimateDiff" }, + { id: "svd-xt", name: "Stable Video Diffusion XT" }, + ], + }, + + sdwebui: { + id: "sdwebui", + baseUrl: "http://localhost:7860", + authType: "none", + authHeader: "none", + format: "sdwebui-video", + models: [ + { id: "animatediff-webui", name: "AnimateDiff (WebUI)" }, + ], + }, +}; + +/** + * Get video provider config by ID + */ +export function getVideoProvider(providerId: string): VideoProvider | null { + return VIDEO_PROVIDERS[providerId] || null; +} + +/** + * Parse video model string (format: "provider/model" or just "model") + */ +export function parseVideoModel(modelStr: string | null) { + return parseModelFromRegistry(modelStr, VIDEO_PROVIDERS); +} + +/** + * Get all video models as a flat list + */ +export function getAllVideoModels() { + return getAllModelsFromRegistry(VIDEO_PROVIDERS); +} diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index 4d6eee08c0..884adef5c7 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -6,24 +6,20 @@ import { getCorsOrigin } from "../utils/cors.ts"; * Returns audio binary stream. * * Supported provider formats: - * - OpenAI: standard JSON → audio stream proxy + * - OpenAI / Qwen3 (openai-compatible): standard JSON → audio stream proxy * - Hyperbolic: POST { text } → { audio: base64 } * - Deepgram: POST { text } with model via query param, Token auth + * - ElevenLabs: POST { text, model_id } to /v1/text-to-speech/{voice_id} + * - Nvidia NIM: POST { input: { text }, voice, model } → audio binary + * - HuggingFace Inference: POST { inputs: text } to /models/{model_id} + * - Coqui TTS: POST { text, speaker_id } → WAV audio (local, no auth) + * - Tortoise TTS: POST { text, voice } → audio binary (local, no auth) */ import { getSpeechProvider, parseSpeechModel } from "../config/audioRegistry.ts"; +import { buildAuthHeaders } from "../config/registryUtils.ts"; import { errorResponse } from "../utils/error.ts"; -/** - * Build auth header for a speech provider - */ -function buildAuthHeader(providerConfig, token) { - if (providerConfig.authHeader === "token") { - return { Authorization: `Token ${token}` }; - } - return { Authorization: `Bearer ${token}` }; -} - /** * Handle Hyperbolic TTS (returns base64 audio in JSON) */ @@ -32,7 +28,7 @@ async function handleHyperbolicSpeech(providerConfig, body, token) { method: "POST", headers: { "Content-Type": "application/json", - ...buildAuthHeader(providerConfig, token), + ...buildAuthHeaders(providerConfig, token), }, body: JSON.stringify({ text: body.input }), }); @@ -72,7 +68,7 @@ async function handleDeepgramSpeech(providerConfig, body, modelId, token) { method: "POST", headers: { "Content-Type": "application/json", - ...buildAuthHeader(providerConfig, token), + ...buildAuthHeaders(providerConfig, token), }, body: JSON.stringify({ text: body.input }), }); @@ -99,6 +95,198 @@ async function handleDeepgramSpeech(providerConfig, body, modelId, token) { }); } +/** + * Handle ElevenLabs TTS + * POST {baseUrl}/{voice_id} with { text, model_id } + * voice_id is mapped from the OpenAI `voice` parameter + */ +async function handleElevenLabsSpeech(providerConfig, body, modelId, token) { + // ElevenLabs uses voice_id in URL path; default to "21m00Tcm4TlvDq8ikWAM" (Rachel) + const voiceId = body.voice || "21m00Tcm4TlvDq8ikWAM"; + const url = `${providerConfig.baseUrl}/${voiceId}`; + + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...buildAuthHeaders(providerConfig, token), + }, + body: JSON.stringify({ + text: body.input, + model_id: modelId, + }), + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const contentType = res.headers.get("content-type") || "audio/mpeg"; + return new Response(res.body, { + status: 200, + headers: { + "Content-Type": contentType, + "Access-Control-Allow-Origin": getCorsOrigin(), + "Transfer-Encoding": "chunked", + }, + }); +} + +/** + * Handle Nvidia NIM TTS + * POST with { input: { text }, voice, model } → audio binary + */ +async function handleNvidiaTtsSpeech(providerConfig, body, modelId, token) { + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...buildAuthHeaders(providerConfig, token), + }, + body: JSON.stringify({ + input: { text: body.input }, + voice: body.voice || "default", + model: modelId, + }), + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const contentType = res.headers.get("content-type") || "audio/wav"; + return new Response(res.body, { + status: 200, + headers: { + "Content-Type": contentType, + "Access-Control-Allow-Origin": getCorsOrigin(), + "Transfer-Encoding": "chunked", + }, + }); +} + +/** + * Handle HuggingFace Inference TTS + * POST {baseUrl}/{model_id} with { inputs: text } → audio binary + */ +async function handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token) { + const url = `${providerConfig.baseUrl}/${modelId}`; + + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...buildAuthHeaders(providerConfig, token), + }, + body: JSON.stringify({ inputs: body.input }), + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const contentType = res.headers.get("content-type") || "audio/wav"; + return new Response(res.body, { + status: 200, + headers: { + "Content-Type": contentType, + "Access-Control-Allow-Origin": getCorsOrigin(), + "Transfer-Encoding": "chunked", + }, + }); +} + +/** + * Handle Coqui TTS (local, no auth) + * POST {baseUrl} with { text, speaker_id } → WAV audio + */ +async function handleCoquiSpeech(providerConfig, body, modelId) { + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + text: body.input, + speaker_id: body.voice || undefined, + }), + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const contentType = res.headers.get("content-type") || "audio/wav"; + return new Response(res.body, { + status: 200, + headers: { + "Content-Type": contentType, + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); +} + +/** + * Handle Tortoise TTS (local, no auth) + * POST {baseUrl} with { text, voice } → audio binary + */ +async function handleTortoiseSpeech(providerConfig, body, modelId) { + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + text: body.input, + voice: body.voice || "random", + }), + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const contentType = res.headers.get("content-type") || "audio/wav"; + return new Response(res.body, { + status: 200, + headers: { + "Content-Type": contentType, + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); +} + /** * Handle audio speech (TTS) request * @@ -122,12 +310,13 @@ export async function handleAudioSpeech({ body, credentials }) { if (!providerConfig) { return errorResponse( 400, - `No speech provider found for model "${body.model}". Available: openai, hyperbolic, deepgram` + `No speech provider found for model "${body.model}". Available: openai, hyperbolic, deepgram, nvidia, elevenlabs, huggingface, coqui, tortoise, qwen` ); } - const token = credentials?.apiKey || credentials?.accessToken; - if (!token) { + // Skip credential check for local providers (authType: "none") + const token = providerConfig.authType === "none" ? null : (credentials?.apiKey || credentials?.accessToken); + if (providerConfig.authType !== "none" && !token) { return errorResponse(401, `No credentials for speech provider: ${providerId}`); } @@ -141,12 +330,32 @@ export async function handleAudioSpeech({ body, credentials }) { return handleDeepgramSpeech(providerConfig, body, modelId, token); } - // Default: OpenAI-compatible JSON → audio stream proxy + if (providerConfig.format === "elevenlabs") { + return handleElevenLabsSpeech(providerConfig, body, modelId, token); + } + + if (providerConfig.format === "nvidia-tts") { + return handleNvidiaTtsSpeech(providerConfig, body, modelId, token); + } + + if (providerConfig.format === "huggingface-tts") { + return handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token); + } + + if (providerConfig.format === "coqui") { + return handleCoquiSpeech(providerConfig, body, modelId); + } + + if (providerConfig.format === "tortoise") { + return handleTortoiseSpeech(providerConfig, body, modelId); + } + + // Default: OpenAI-compatible JSON → audio stream proxy (also used by Qwen3) const res = await fetch(providerConfig.baseUrl, { method: "POST", headers: { "Content-Type": "application/json", - ...buildAuthHeader(providerConfig, token), + ...buildAuthHeaders(providerConfig, token), }, body: JSON.stringify({ model: modelId, @@ -181,4 +390,4 @@ export async function handleAudioSpeech({ body, credentials }) { } catch (err) { return errorResponse(500, `Speech request failed: ${err.message}`); } -} +} \ No newline at end of file diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index 70d513aae2..182008b4e8 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -6,24 +6,17 @@ import { getCorsOrigin } from "../utils/cors.ts"; * Proxies multipart/form-data to upstream providers. * * Supported provider formats: - * - OpenAI/Groq: standard multipart form-data proxy + * - OpenAI/Groq/Qwen3: standard multipart form-data proxy * - Deepgram: raw binary audio POST with model via query param * - AssemblyAI: async workflow (upload → submit → poll) + * - Nvidia NIM: multipart POST, transform response to { text } + * - HuggingFace Inference: POST raw binary to /models/{model_id} */ import { getTranscriptionProvider, parseTranscriptionModel } from "../config/audioRegistry.ts"; +import { buildAuthHeaders } from "../config/registryUtils.ts"; import { errorResponse } from "../utils/error.ts"; -/** - * Build auth header for a transcription provider - */ -function buildAuthHeader(providerConfig, token) { - if (providerConfig.authHeader === "token") { - return { Authorization: `Token ${token}` }; - } - return { Authorization: `Bearer ${token}` }; -} - /** * Handle Deepgram transcription (raw binary audio, model via query param) */ @@ -37,7 +30,7 @@ async function handleDeepgramTranscription(providerConfig, file, modelId, token) const res = await fetch(url.toString(), { method: "POST", headers: { - ...buildAuthHeader(providerConfig, token), + ...buildAuthHeaders(providerConfig, token), "Content-Type": file.type || "audio/wav", }, body: arrayBuffer, @@ -65,7 +58,7 @@ async function handleDeepgramTranscription(providerConfig, file, modelId, token) * Handle AssemblyAI transcription (async: upload file → submit → poll) */ async function handleAssemblyAITranscription(providerConfig, file, modelId, token) { - const authHeaders = buildAuthHeader(providerConfig, token); + const authHeaders = buildAuthHeaders(providerConfig, token); // Step 1: Upload the audio file const arrayBuffer = await file.arrayBuffer(); @@ -146,6 +139,74 @@ async function handleAssemblyAITranscription(providerConfig, file, modelId, toke return errorResponse(504, "AssemblyAI transcription timed out after 120s"); } +/** + * Handle Nvidia NIM transcription + * Multipart POST, transform response to { text } + */ +async function handleNvidiaTranscription(providerConfig, file, modelId, token) { + const upstreamForm = new FormData(); + upstreamForm.append("file", /** @type {Blob} */ file, /** @type {any} */ file.name || "audio.wav"); + upstreamForm.append("model", modelId); + + const res = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: buildAuthHeaders(providerConfig, token), + body: upstreamForm, + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const data = await res.json(); + // Normalize to { text } — Nvidia may return { text } directly or nested + const text = data.text || data.transcript || ""; + + return Response.json({ text }, { headers: { "Access-Control-Allow-Origin": getCorsOrigin() } }); +} + +/** + * Handle HuggingFace Inference transcription + * POST raw binary audio to {baseUrl}/{model_id}, returns { text } + */ +async function handleHuggingFaceTranscription(providerConfig, file, modelId, token) { + const url = `${providerConfig.baseUrl}/${modelId}`; + const arrayBuffer = await file.arrayBuffer(); + + const res = await fetch(url, { + method: "POST", + headers: { + ...buildAuthHeaders(providerConfig, token), + "Content-Type": file.type || "audio/wav", + }, + body: arrayBuffer, + }); + + if (!res.ok) { + const errText = await res.text(); + return new Response(errText, { + status: res.status, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": getCorsOrigin(), + }, + }); + } + + const data = await res.json(); + // HuggingFace returns { text } directly + const text = data.text || ""; + + return Response.json({ text }, { headers: { "Access-Control-Allow-Origin": getCorsOrigin() } }); +} + /** * Handle audio transcription request * @@ -172,12 +233,13 @@ export async function handleAudioTranscription({ formData, credentials }) { if (!providerConfig) { return errorResponse( 400, - `No transcription provider found for model "${model}". Available: openai, groq, deepgram, assemblyai` + `No transcription provider found for model "${model}". Available: openai, groq, deepgram, assemblyai, nvidia, huggingface, qwen` ); } - const token = credentials?.apiKey || credentials?.accessToken; - if (!token) { + // Skip credential check for local providers (authType: "none") + const token = providerConfig.authType === "none" ? null : (credentials?.apiKey || credentials?.accessToken); + if (providerConfig.authType !== "none" && !token) { return errorResponse(401, `No credentials for transcription provider: ${providerId}`); } @@ -190,7 +252,15 @@ export async function handleAudioTranscription({ formData, credentials }) { return handleAssemblyAITranscription(providerConfig, file, modelId, token); } - // Default: OpenAI/Groq-compatible multipart proxy + if (providerConfig.format === "nvidia-asr") { + return handleNvidiaTranscription(providerConfig, file, modelId, token); + } + + if (providerConfig.format === "huggingface-asr") { + return handleHuggingFaceTranscription(providerConfig, file, modelId, token); + } + + // Default: OpenAI/Groq/Qwen3-compatible multipart proxy const upstreamForm = new FormData(); upstreamForm.append( "file", @@ -216,7 +286,7 @@ export async function handleAudioTranscription({ formData, credentials }) { try { const res = await fetch(providerConfig.baseUrl, { method: "POST", - headers: buildAuthHeader(providerConfig, token), + headers: buildAuthHeaders(providerConfig, token), body: upstreamForm, }); @@ -241,4 +311,4 @@ export async function handleAudioTranscription({ formData, credentials }) { } catch (err) { return errorResponse(500, `Transcription request failed: ${err.message}`); } -} +} \ No newline at end of file diff --git a/open-sse/handlers/imageGeneration.ts b/open-sse/handlers/imageGeneration.ts index ca9b25ea7f..b7de3270c7 100644 --- a/open-sse/handlers/imageGeneration.ts +++ b/open-sse/handlers/imageGeneration.ts @@ -17,6 +17,12 @@ import { getImageProvider, parseImageModel } from "../config/imageRegistry.ts"; import { saveCallLog } from "@/lib/usageDb"; +import { + submitComfyWorkflow, + pollComfyResult, + fetchComfyOutput, + extractComfyOutputFiles, +} from "../utils/comfyuiClient.ts"; /** * Handle image generation request @@ -72,6 +78,14 @@ export async function handleImageGeneration({ body, credentials, log }) { }); } + if (providerConfig.format === "sdwebui") { + return handleSDWebUIImageGeneration({ model, provider, providerConfig, body, log }); + } + + if (providerConfig.format === "comfyui") { + return handleComfyUIImageGeneration({ model, provider, providerConfig, body, log }); + } + return handleOpenAIImageGeneration({ model, provider, providerConfig, body, credentials, log }); } @@ -560,3 +574,193 @@ async function handleNanoBananaImageGeneration({ return { success: false, status: 502, error: `Image provider error: ${err.message}` }; } } + +/** + * Handle SD WebUI image generation (local, no auth) + * POST {baseUrl} with { prompt, negative_prompt, width, height, steps } + * Response: { images: ["base64..."] } + */ +async function handleSDWebUIImageGeneration({ model, provider, providerConfig, body, log }) { + const startTime = Date.now(); + const [width, height] = (body.size || "512x512").split("x").map(Number); + + const upstreamBody = { + prompt: body.prompt, + negative_prompt: body.negative_prompt || "", + width: width || 512, + height: height || 512, + steps: body.steps || 20, + cfg_scale: body.cfg_scale || 7, + sampler_name: body.sampler || "Euler a", + batch_size: body.n || 1, + override_settings: { + sd_model_checkpoint: model, + }, + }; + + if (log) { + const promptPreview = String(body.prompt ?? "").slice(0, 60); + log.info("IMAGE", `${provider}/${model} (sdwebui) | prompt: "${promptPreview}..."`); + } + + try { + const response = await fetch(providerConfig.baseUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(upstreamBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + if (log) log.error("IMAGE", `${provider} error ${response.status}: ${errorText.slice(0, 200)}`); + + saveCallLog({ + method: "POST", + path: "/v1/images/generations", + status: response.status, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + error: errorText.slice(0, 500), + }).catch(() => {}); + + return { success: false, status: response.status, error: errorText }; + } + + const data = await response.json(); + // SD WebUI returns { images: ["base64...", ...] } + const images = (data.images || []).map((b64) => ({ + b64_json: b64, + revised_prompt: body.prompt, + })); + + saveCallLog({ + method: "POST", + path: "/v1/images/generations", + status: 200, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + responseBody: { images_count: images.length }, + }).catch(() => {}); + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: images }, + }; + } catch (err) { + if (log) log.error("IMAGE", `${provider} sdwebui error: ${err.message}`); + saveCallLog({ + method: "POST", + path: "/v1/images/generations", + status: 502, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + error: err.message, + }).catch(() => {}); + return { success: false, status: 502, error: `Image provider error: ${err.message}` }; + } +} + +/** + * Handle ComfyUI image generation (local, no auth) + * Submits a txt2img workflow, polls for completion, fetches output + */ +async function handleComfyUIImageGeneration({ model, provider, providerConfig, body, log }) { + const startTime = Date.now(); + const [width, height] = (body.size || "1024x1024").split("x").map(Number); + + // Default txt2img workflow template for ComfyUI + const workflow = { + "3": { + class_type: "KSampler", + inputs: { + seed: Math.floor(Math.random() * 2 ** 32), + steps: body.steps || 20, + cfg: body.cfg_scale || 7, + sampler_name: "euler", + scheduler: "normal", + denoise: 1, + model: ["4", 0], + positive: ["6", 0], + negative: ["7", 0], + latent_image: ["5", 0], + }, + }, + "4": { + class_type: "CheckpointLoaderSimple", + inputs: { ckpt_name: model }, + }, + "5": { + class_type: "EmptyLatentImage", + inputs: { width: width || 1024, height: height || 1024, batch_size: body.n || 1 }, + }, + "6": { + class_type: "CLIPTextEncode", + inputs: { text: body.prompt, clip: ["4", 1] }, + }, + "7": { + class_type: "CLIPTextEncode", + inputs: { text: body.negative_prompt || "", clip: ["4", 1] }, + }, + "8": { + class_type: "VAEDecode", + inputs: { samples: ["3", 0], vae: ["4", 2] }, + }, + "9": { + class_type: "SaveImage", + inputs: { filename_prefix: "omniroute", images: ["8", 0] }, + }, + }; + + if (log) { + const promptPreview = String(body.prompt ?? "").slice(0, 60); + log.info("IMAGE", `${provider}/${model} (comfyui) | prompt: "${promptPreview}..."`); + } + + try { + const promptId = await submitComfyWorkflow(providerConfig.baseUrl, workflow); + const historyEntry = await pollComfyResult(providerConfig.baseUrl, promptId); + const outputFiles = extractComfyOutputFiles(historyEntry); + + const images = []; + for (const file of outputFiles) { + const buffer = await fetchComfyOutput( + providerConfig.baseUrl, + file.filename, + file.subfolder, + file.type + ); + const base64 = Buffer.from(buffer).toString("base64"); + images.push({ b64_json: base64, revised_prompt: body.prompt }); + } + + saveCallLog({ + method: "POST", + path: "/v1/images/generations", + status: 200, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + responseBody: { images_count: images.length }, + }).catch(() => {}); + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: images }, + }; + } catch (err) { + if (log) log.error("IMAGE", `${provider} comfyui error: ${err.message}`); + saveCallLog({ + method: "POST", + path: "/v1/images/generations", + status: 502, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + error: err.message, + }).catch(() => {}); + return { success: false, status: 502, error: `Image provider error: ${err.message}` }; + } +} diff --git a/open-sse/handlers/musicGeneration.ts b/open-sse/handlers/musicGeneration.ts new file mode 100644 index 0000000000..3efab8fa9a --- /dev/null +++ b/open-sse/handlers/musicGeneration.ts @@ -0,0 +1,138 @@ +/** + * Music Generation Handler + * + * Handles POST /v1/music/generations requests. + * Proxies to upstream music generation providers. + * + * Supported provider formats: + * - ComfyUI: submit audio workflow → poll → fetch output + * + * Response format (OpenAI-like): + * { + * "created": 1234567890, + * "data": [{ "b64_json": "...", "format": "wav" }] + * } + */ + +import { getMusicProvider, parseMusicModel } from "../config/musicRegistry.ts"; +import { + submitComfyWorkflow, + pollComfyResult, + fetchComfyOutput, + extractComfyOutputFiles, +} from "../utils/comfyuiClient.ts"; + +/** + * Handle music generation request + */ +export async function handleMusicGeneration({ body, credentials, log }) { + const { provider, model } = parseMusicModel(body.model); + + if (!provider) { + return { + success: false, + status: 400, + error: `Invalid music model: ${body.model}. Use format: provider/model`, + }; + } + + const providerConfig = getMusicProvider(provider); + if (!providerConfig) { + return { + success: false, + status: 400, + error: `Unknown music provider: ${provider}`, + }; + } + + if (providerConfig.format === "comfyui") { + return handleComfyUIMusicGeneration({ model, provider, providerConfig, body, log }); + } + + return { success: false, status: 400, error: `Unsupported music format: ${providerConfig.format}` }; +} + +/** + * Handle ComfyUI music generation + * Submits an audio generation workflow (Stable Audio / MusicGen), polls, fetches output + */ +async function handleComfyUIMusicGeneration({ model, provider, providerConfig, body, log }) { + const duration = body.duration || 10; // seconds + + // Audio generation workflow template for ComfyUI + const workflow = { + "1": { + class_type: "CheckpointLoaderSimple", + inputs: { ckpt_name: model }, + }, + "2": { + class_type: "CLIPTextEncode", + inputs: { text: body.prompt, clip: ["1", 1] }, + }, + "3": { + class_type: "CLIPTextEncode", + inputs: { text: body.negative_prompt || "", clip: ["1", 1] }, + }, + "4": { + class_type: "EmptyLatentAudio", + inputs: { seconds: duration }, + }, + "5": { + class_type: "KSampler", + inputs: { + seed: Math.floor(Math.random() * 2 ** 32), + steps: body.steps || 100, + cfg: body.cfg_scale || 7, + sampler_name: "euler", + scheduler: "normal", + denoise: 1, + model: ["1", 0], + positive: ["2", 0], + negative: ["3", 0], + latent_image: ["4", 0], + }, + }, + "6": { + class_type: "VAEDecodeAudio", + inputs: { samples: ["5", 0], vae: ["1", 2] }, + }, + "7": { + class_type: "SaveAudio", + inputs: { + filename_prefix: "omniroute_music", + audio: ["6", 0], + }, + }, + }; + + if (log) { + const promptPreview = String(body.prompt ?? "").slice(0, 60); + log.info("MUSIC", `${provider}/${model} (comfyui) | prompt: "${promptPreview}..." | duration: ${duration}s`); + } + + try { + const promptId = await submitComfyWorkflow(providerConfig.baseUrl, workflow); + const historyEntry = await pollComfyResult(providerConfig.baseUrl, promptId, 300_000); + const outputFiles = extractComfyOutputFiles(historyEntry); + + const audioFiles = []; + for (const file of outputFiles) { + const buffer = await fetchComfyOutput( + providerConfig.baseUrl, + file.filename, + file.subfolder, + file.type + ); + const base64 = Buffer.from(buffer).toString("base64"); + audioFiles.push({ b64_json: base64, format: "wav" }); + } + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: audioFiles }, + }; + } catch (err) { + if (log) log.error("MUSIC", `${provider} comfyui error: ${err.message}`); + return { success: false, status: 502, error: `Music provider error: ${err.message}` }; + } +} diff --git a/open-sse/handlers/videoGeneration.ts b/open-sse/handlers/videoGeneration.ts new file mode 100644 index 0000000000..65d52d3b5b --- /dev/null +++ b/open-sse/handlers/videoGeneration.ts @@ -0,0 +1,206 @@ +/** + * Video Generation Handler + * + * Handles POST /v1/videos/generations requests. + * Proxies to upstream video generation providers. + * + * Supported provider formats: + * - ComfyUI: submit AnimateDiff/SVD workflow → poll → fetch video + * - SD WebUI: POST to AnimateDiff extension endpoint + * + * Response format (OpenAI-like): + * { + * "created": 1234567890, + * "data": [{ "b64_json": "...", "format": "mp4" }] + * } + */ + +import { getVideoProvider, parseVideoModel } from "../config/videoRegistry.ts"; +import { + submitComfyWorkflow, + pollComfyResult, + fetchComfyOutput, + extractComfyOutputFiles, +} from "../utils/comfyuiClient.ts"; + +/** + * Handle video generation request + */ +export async function handleVideoGeneration({ body, credentials, log }) { + const { provider, model } = parseVideoModel(body.model); + + if (!provider) { + return { + success: false, + status: 400, + error: `Invalid video model: ${body.model}. Use format: provider/model`, + }; + } + + const providerConfig = getVideoProvider(provider); + if (!providerConfig) { + return { + success: false, + status: 400, + error: `Unknown video provider: ${provider}`, + }; + } + + if (providerConfig.format === "comfyui") { + return handleComfyUIVideoGeneration({ model, provider, providerConfig, body, log }); + } + + if (providerConfig.format === "sdwebui-video") { + return handleSDWebUIVideoGeneration({ model, provider, providerConfig, body, log }); + } + + return { success: false, status: 400, error: `Unsupported video format: ${providerConfig.format}` }; +} + +/** + * Handle ComfyUI video generation + * Submits an AnimateDiff or SVD workflow, polls for completion, fetches output video + */ +async function handleComfyUIVideoGeneration({ model, provider, providerConfig, body, log }) { + const [width, height] = (body.size || "512x512").split("x").map(Number); + const frames = body.frames || 16; + + // AnimateDiff workflow template + const workflow = { + "1": { + class_type: "CheckpointLoaderSimple", + inputs: { ckpt_name: model }, + }, + "2": { + class_type: "CLIPTextEncode", + inputs: { text: body.prompt, clip: ["1", 1] }, + }, + "3": { + class_type: "CLIPTextEncode", + inputs: { text: body.negative_prompt || "", clip: ["1", 1] }, + }, + "4": { + class_type: "EmptyLatentImage", + inputs: { width: width || 512, height: height || 512, batch_size: frames }, + }, + "5": { + class_type: "KSampler", + inputs: { + seed: Math.floor(Math.random() * 2 ** 32), + steps: body.steps || 20, + cfg: body.cfg_scale || 7, + sampler_name: "euler", + scheduler: "normal", + denoise: 1, + model: ["1", 0], + positive: ["2", 0], + negative: ["3", 0], + latent_image: ["4", 0], + }, + }, + "6": { + class_type: "VAEDecode", + inputs: { samples: ["5", 0], vae: ["1", 2] }, + }, + "7": { + class_type: "SaveAnimatedWEBP", + inputs: { + filename_prefix: "omniroute_video", + fps: body.fps || 8, + lossless: false, + quality: 80, + method: "default", + images: ["6", 0], + }, + }, + }; + + if (log) { + const promptPreview = String(body.prompt ?? "").slice(0, 60); + log.info("VIDEO", `${provider}/${model} (comfyui) | prompt: "${promptPreview}..." | frames: ${frames}`); + } + + try { + const promptId = await submitComfyWorkflow(providerConfig.baseUrl, workflow); + const historyEntry = await pollComfyResult(providerConfig.baseUrl, promptId, 300_000); + const outputFiles = extractComfyOutputFiles(historyEntry); + + const videos = []; + for (const file of outputFiles) { + const buffer = await fetchComfyOutput( + providerConfig.baseUrl, + file.filename, + file.subfolder, + file.type + ); + const base64 = Buffer.from(buffer).toString("base64"); + videos.push({ b64_json: base64, format: "webp" }); + } + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: videos }, + }; + } catch (err) { + if (log) log.error("VIDEO", `${provider} comfyui error: ${err.message}`); + return { success: false, status: 502, error: `Video provider error: ${err.message}` }; + } +} + +/** + * Handle SD WebUI video generation via AnimateDiff extension + * POST to the AnimateDiff API endpoint + */ +async function handleSDWebUIVideoGeneration({ model, provider, providerConfig, body, log }) { + const [width, height] = (body.size || "512x512").split("x").map(Number); + const url = `${providerConfig.baseUrl}/animatediff/v1/generate`; + + const upstreamBody = { + prompt: body.prompt, + negative_prompt: body.negative_prompt || "", + width: width || 512, + height: height || 512, + steps: body.steps || 20, + cfg_scale: body.cfg_scale || 7, + frames: body.frames || 16, + fps: body.fps || 8, + }; + + if (log) { + const promptPreview = String(body.prompt ?? "").slice(0, 60); + log.info("VIDEO", `${provider}/${model} (sdwebui) | prompt: "${promptPreview}..."`); + } + + try { + const response = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(upstreamBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + if (log) log.error("VIDEO", `${provider} error ${response.status}: ${errorText.slice(0, 200)}`); + return { success: false, status: response.status, error: errorText }; + } + + const data = await response.json(); + // SD WebUI AnimateDiff returns { video: "base64..." } or { images: [...] } + const videos = []; + if (data.video) { + videos.push({ b64_json: data.video, format: "mp4" }); + } else if (data.images) { + for (const img of data.images) { + videos.push({ b64_json: typeof img === "string" ? img : img.image, format: "mp4" }); + } + } + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: videos }, + }; + } catch (err) { + if (log) log.error("VIDEO", `${provider} sdwebui error: ${err.message}`); + return { success: false, status: 502, error: `Video provider error: ${err.message}` }; + } +} diff --git a/open-sse/index.ts b/open-sse/index.ts index 071a810b81..0292fd172b 100644 --- a/open-sse/index.ts +++ b/open-sse/index.ts @@ -139,3 +139,28 @@ export { parseModerationModel, getAllModerationModels, } from "./config/moderationRegistry.ts"; + +// Video Generation +export { handleVideoGeneration } from "./handlers/videoGeneration.ts"; +export { + VIDEO_PROVIDERS, + getVideoProvider, + parseVideoModel, + getAllVideoModels, +} from "./config/videoRegistry.ts"; + +// Music Generation +export { handleMusicGeneration } from "./handlers/musicGeneration.ts"; +export { + MUSIC_PROVIDERS, + getMusicProvider, + parseMusicModel, + getAllMusicModels, +} from "./config/musicRegistry.ts"; + +// Registry Utilities +export { + parseModelFromRegistry, + getAllModelsFromRegistry, + buildAuthHeaders, +} from "./config/registryUtils.ts"; diff --git a/open-sse/utils/comfyuiClient.ts b/open-sse/utils/comfyuiClient.ts new file mode 100644 index 0000000000..fbff0e2373 --- /dev/null +++ b/open-sse/utils/comfyuiClient.ts @@ -0,0 +1,103 @@ +/** + * Shared ComfyUI API Client + * + * Used by image, video, and music handlers to submit workflows, + * poll for completion, and fetch output files from a ComfyUI server. + */ + +/** + * Submit a workflow to ComfyUI for execution. + * @returns The prompt_id for polling + */ +export async function submitComfyWorkflow( + baseUrl: string, + workflow: object +): Promise { + const res = await fetch(`${baseUrl}/prompt`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ prompt: workflow }), + }); + + if (!res.ok) { + const errText = await res.text(); + throw new Error(`ComfyUI submit failed (${res.status}): ${errText}`); + } + + const data = await res.json(); + return data.prompt_id; +} + +/** + * Poll ComfyUI history endpoint until the prompt completes or times out. + * @returns The history entry for the completed prompt + */ +export async function pollComfyResult( + baseUrl: string, + promptId: string, + timeoutMs: number = 120_000 +): Promise { + const start = Date.now(); + + while (Date.now() - start < timeoutMs) { + await new Promise((r) => setTimeout(r, 2000)); + + const res = await fetch(`${baseUrl}/history/${promptId}`); + if (!res.ok) continue; + + const data = await res.json(); + const entry = data[promptId]; + + if (entry && entry.outputs && Object.keys(entry.outputs).length > 0) { + return entry; + } + } + + throw new Error(`ComfyUI prompt ${promptId} timed out after ${timeoutMs}ms`); +} + +/** + * Fetch an output file from ComfyUI. + * @returns The file contents as ArrayBuffer + */ +export async function fetchComfyOutput( + baseUrl: string, + filename: string, + subfolder: string, + type: string +): Promise { + const url = new URL(`${baseUrl}/view`); + url.searchParams.set("filename", filename); + url.searchParams.set("subfolder", subfolder); + url.searchParams.set("type", type); + + const res = await fetch(url.toString()); + if (!res.ok) { + throw new Error(`ComfyUI fetch output failed (${res.status})`); + } + + return res.arrayBuffer(); +} + +/** + * Extract output files from a ComfyUI history entry. + * Returns an array of { filename, subfolder, type } for each output. + */ +export function extractComfyOutputFiles( + historyEntry: any +): Array<{ filename: string; subfolder: string; type: string }> { + const files: Array<{ filename: string; subfolder: string; type: string }> = []; + + for (const nodeOutput of Object.values(historyEntry.outputs || {})) { + const outputs = (nodeOutput as any).images || (nodeOutput as any).gifs || (nodeOutput as any).audio || []; + for (const file of outputs) { + files.push({ + filename: file.filename, + subfolder: file.subfolder || "", + type: file.type || "output", + }); + } + } + + return files; +} diff --git a/src/app/api/v1/audio/speech/route.ts b/src/app/api/v1/audio/speech/route.ts index c16ba8cbfa..9105bd8c93 100644 --- a/src/app/api/v1/audio/speech/route.ts +++ b/src/app/api/v1/audio/speech/route.ts @@ -1,7 +1,7 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleAudioSpeech } from "@omniroute/open-sse/handlers/audioSpeech.ts"; import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; -import { parseSpeechModel } from "@omniroute/open-sse/config/audioRegistry.ts"; +import { parseSpeechModel, getSpeechProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; @@ -54,9 +54,16 @@ export async function POST(request) { ); } - const credentials = await getProviderCredentials(provider); - if (!credentials) { - return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`); + // Check provider config for auth bypass + const providerConfig = getSpeechProvider(provider); + + // Get credentials — skip for local providers (authType: "none") + let credentials = null; + if (providerConfig && providerConfig.authType !== "none") { + credentials = await getProviderCredentials(provider); + if (!credentials) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`); + } } return handleAudioSpeech({ body, credentials }); diff --git a/src/app/api/v1/audio/transcriptions/route.ts b/src/app/api/v1/audio/transcriptions/route.ts index adfeb00fff..6fe5ab4dbc 100644 --- a/src/app/api/v1/audio/transcriptions/route.ts +++ b/src/app/api/v1/audio/transcriptions/route.ts @@ -1,7 +1,7 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleAudioTranscription } from "@omniroute/open-sse/handlers/audioTranscription.ts"; import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; -import { parseTranscriptionModel } from "@omniroute/open-sse/config/audioRegistry.ts"; +import { parseTranscriptionModel, getTranscriptionProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; @@ -56,9 +56,16 @@ export async function POST(request) { ); } - const credentials = await getProviderCredentials(provider); - if (!credentials) { - return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`); + // Check provider config for auth bypass + const providerConfig = getTranscriptionProvider(provider); + + // Get credentials — skip for local providers (authType: "none") + let credentials = null; + if (providerConfig && providerConfig.authType !== "none") { + credentials = await getProviderCredentials(provider); + if (!credentials) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`); + } } return handleAudioTranscription({ formData, credentials }); diff --git a/src/app/api/v1/images/generations/route.ts b/src/app/api/v1/images/generations/route.ts index d95bd090f6..5a007fd620 100644 --- a/src/app/api/v1/images/generations/route.ts +++ b/src/app/api/v1/images/generations/route.ts @@ -1,7 +1,7 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleImageGeneration } from "@omniroute/open-sse/handlers/imageGeneration.ts"; import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; -import { parseImageModel, getAllImageModels } from "@omniroute/open-sse/config/imageRegistry.ts"; +import { parseImageModel, getAllImageModels, getImageProvider } from "@omniroute/open-sse/config/imageRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import * as log from "@/sse/utils/logger"; @@ -89,10 +89,16 @@ export async function POST(request) { ); } - // Get credentials for the image provider - const credentials = await getProviderCredentials(provider); - if (!credentials) { - return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for image provider: ${provider}`); + // Check provider config for auth bypass + const providerConfig = getImageProvider(provider); + + // Get credentials — skip for local providers (authType: "none") + let credentials = null; + if (providerConfig && providerConfig.authType !== "none") { + credentials = await getProviderCredentials(provider); + if (!credentials) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for image provider: ${provider}`); + } } const result = await handleImageGeneration({ body, credentials, log }); diff --git a/src/app/api/v1/models/route.ts b/src/app/api/v1/models/route.ts index f938f2618a..78ef1064c2 100644 --- a/src/app/api/v1/models/route.ts +++ b/src/app/api/v1/models/route.ts @@ -14,6 +14,8 @@ import { getAllImageModels } from "@omniroute/open-sse/config/imageRegistry.ts"; import { getAllRerankModels } from "@omniroute/open-sse/config/rerankRegistry.ts"; import { getAllAudioModels } from "@omniroute/open-sse/config/audioRegistry.ts"; import { getAllModerationModels } from "@omniroute/open-sse/config/moderationRegistry.ts"; +import { getAllVideoModels } from "@omniroute/open-sse/config/videoRegistry.ts"; +import { getAllMusicModels } from "@omniroute/open-sse/config/musicRegistry.ts"; const FALLBACK_ALIAS_TO_PROVIDER = { ag: "antigravity", @@ -310,6 +312,28 @@ export async function GET(request: Request) { }); } + // Add video models (local providers always listed, cloud filtered by active) + for (const videoModel of getAllVideoModels()) { + models.push({ + id: videoModel.id, + object: "model", + created: timestamp, + owned_by: videoModel.provider, + type: "video", + }); + } + + // Add music models (local providers always listed, cloud filtered by active) + for (const musicModel of getAllMusicModels()) { + models.push({ + id: musicModel.id, + object: "model", + created: timestamp, + owned_by: musicModel.provider, + type: "music", + }); + } + // Add custom models (user-defined) try { const customModelsMap: Record = await getAllCustomModels(); diff --git a/src/app/api/v1/music/generations/route.ts b/src/app/api/v1/music/generations/route.ts new file mode 100644 index 0000000000..f7508fd113 --- /dev/null +++ b/src/app/api/v1/music/generations/route.ts @@ -0,0 +1,117 @@ +import { CORS_ORIGIN } from "@/shared/utils/cors"; +import { handleMusicGeneration } from "@omniroute/open-sse/handlers/musicGeneration.ts"; +import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { parseMusicModel, getAllMusicModels, getMusicProvider } from "@omniroute/open-sse/config/musicRegistry.ts"; +import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; +import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; +import * as log from "@/sse/utils/logger"; +import { toJsonErrorPayload } from "@/shared/utils/upstreamError"; +import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; + +/** + * Handle CORS preflight + */ +export async function OPTIONS() { + return new Response(null, { + headers: { + "Access-Control-Allow-Origin": CORS_ORIGIN, + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); +} + +/** + * GET /v1/music/generations — list available music models + */ +export async function GET() { + const models = getAllMusicModels(); + return new Response( + JSON.stringify({ + object: "list", + data: models.map((m) => ({ + id: m.id, + object: "model", + created: Math.floor(Date.now() / 1000), + owned_by: m.provider, + type: "music", + })), + }), + { + headers: { "Content-Type": "application/json" }, + } + ); +} + +/** + * POST /v1/music/generations — generate music + */ +export async function POST(request) { + let body; + try { + body = await request.json(); + } catch { + log.warn("MUSIC", "Invalid JSON body"); + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body"); + } + + // Optional API key validation + if (process.env.REQUIRE_API_KEY === "true") { + const apiKey = extractApiKey(request); + if (!apiKey) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key"); + } + const valid = await isValidApiKey(apiKey); + if (!valid) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key"); + } + } + + if (!body.model) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model"); + } + + if (typeof body.prompt !== "string" || body.prompt.trim().length === 0) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid prompt: expected a non-empty string"); + } + + // Enforce API key policies (model restrictions + budget limits) + const policy = await enforceApiKeyPolicy(request, body.model); + if (policy.rejection) return policy.rejection; + + // Parse model to get provider + const { provider } = parseMusicModel(body.model); + if (!provider) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `Invalid music model: ${body.model}. Use format: provider/model` + ); + } + + // Check provider config for auth bypass + const providerConfig = getMusicProvider(provider); + + // Get credentials — skip for local providers (authType: "none") + let credentials = null; + if (providerConfig && providerConfig.authType !== "none") { + credentials = await getProviderCredentials(provider); + if (!credentials) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for music provider: ${provider}`); + } + } + + const result = await handleMusicGeneration({ body, credentials, log }); + + if (result.success) { + return new Response(JSON.stringify((result as any).data), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + + const errorPayload = toJsonErrorPayload((result as any).error, "Music generation provider error"); + return new Response(JSON.stringify(errorPayload), { + status: (result as any).status, + headers: { "Content-Type": "application/json" }, + }); +} diff --git a/src/app/api/v1/videos/generations/route.ts b/src/app/api/v1/videos/generations/route.ts new file mode 100644 index 0000000000..24e1255698 --- /dev/null +++ b/src/app/api/v1/videos/generations/route.ts @@ -0,0 +1,117 @@ +import { CORS_ORIGIN } from "@/shared/utils/cors"; +import { handleVideoGeneration } from "@omniroute/open-sse/handlers/videoGeneration.ts"; +import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { parseVideoModel, getAllVideoModels, getVideoProvider } from "@omniroute/open-sse/config/videoRegistry.ts"; +import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; +import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; +import * as log from "@/sse/utils/logger"; +import { toJsonErrorPayload } from "@/shared/utils/upstreamError"; +import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; + +/** + * Handle CORS preflight + */ +export async function OPTIONS() { + return new Response(null, { + headers: { + "Access-Control-Allow-Origin": CORS_ORIGIN, + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); +} + +/** + * GET /v1/videos/generations — list available video models + */ +export async function GET() { + const models = getAllVideoModels(); + return new Response( + JSON.stringify({ + object: "list", + data: models.map((m) => ({ + id: m.id, + object: "model", + created: Math.floor(Date.now() / 1000), + owned_by: m.provider, + type: "video", + })), + }), + { + headers: { "Content-Type": "application/json" }, + } + ); +} + +/** + * POST /v1/videos/generations — generate videos + */ +export async function POST(request) { + let body; + try { + body = await request.json(); + } catch { + log.warn("VIDEO", "Invalid JSON body"); + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body"); + } + + // Optional API key validation + if (process.env.REQUIRE_API_KEY === "true") { + const apiKey = extractApiKey(request); + if (!apiKey) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Missing API key"); + } + const valid = await isValidApiKey(apiKey); + if (!valid) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key"); + } + } + + if (!body.model) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Missing model"); + } + + if (typeof body.prompt !== "string" || body.prompt.trim().length === 0) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid prompt: expected a non-empty string"); + } + + // Enforce API key policies (model restrictions + budget limits) + const policy = await enforceApiKeyPolicy(request, body.model); + if (policy.rejection) return policy.rejection; + + // Parse model to get provider + const { provider } = parseVideoModel(body.model); + if (!provider) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `Invalid video model: ${body.model}. Use format: provider/model` + ); + } + + // Check provider config for auth bypass + const providerConfig = getVideoProvider(provider); + + // Get credentials — skip for local providers (authType: "none") + let credentials = null; + if (providerConfig && providerConfig.authType !== "none") { + credentials = await getProviderCredentials(provider); + if (!credentials) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for video provider: ${provider}`); + } + } + + const result = await handleVideoGeneration({ body, credentials, log }); + + if (result.success) { + return new Response(JSON.stringify((result as any).data), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + + const errorPayload = toJsonErrorPayload((result as any).error, "Video generation provider error"); + return new Response(JSON.stringify(errorPayload), { + status: (result as any).status, + headers: { "Content-Type": "application/json" }, + }); +}