feat(content): extend providers with video, audio, TTS, music capabilities (#2369)

Integrated into release/v3.8.0

open-sse/config/audioRegistry.ts

@@ -67,6 +67,27 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record<string, AudioProvider> = {
     ],
   },
 
+  pollinations: {
+    id: "pollinations",
+    baseUrl: "https://gen.pollinations.ai/v1/audio/transcriptions",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "openai",
+    models: [{ id: "whisper", name: "Pollinations Whisper (Free)" }],
+  },
+
+  together: {
+    id: "together",
+    baseUrl: "https://api.together.xyz/v1/audio/transcriptions",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "openai",
+    models: [
+      { id: "openai/whisper-large-v3", name: "Whisper Large v3" },
+      { id: "openai/whisper-large-v3-turbo", name: "Whisper Large v3 Turbo" },
+    ],
+  },
+
   assemblyai: {
     id: "assemblyai",
     baseUrl: "https://api.assemblyai.com/v2/transcript",
@@ -302,6 +323,37 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
       { id: "generative", name: "Polly Generative" },
     ],
   },
+  pollinations: {
+    id: "pollinations",
+    baseUrl: "https://gen.pollinations.ai/v1/audio/speech",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "openai",
+    models: [{ id: "default", name: "Pollinations TTS (Free)" }],
+  },
+
+  minimax: {
+    id: "minimax",
+    baseUrl: "https://api.minimax.io/v1/t2a_v2",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "minimax-tts",
+    models: [{ id: "speech-2.8-hd", name: "Speech 2.8 HD" }],
+  },
+
+  together: {
+    id: "together",
+    baseUrl: "https://api.together.xyz/v1/audio/speech",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "openai",
+    models: [
+      { id: "cartesia/sonic-2", name: "Cartesia Sonic 2" },
+      { id: "hexgrad/Kokoro-82M", name: "Kokoro 82M" },
+      { id: "canopylabs/orpheus-3b-0.1-ft", name: "Orpheus 3B" },
+    ],
+  },
+
   "xiaomi-mimo": {
     id: "xiaomi-mimo",
     baseUrl: "https://api.xiaomimimo.com/v1/chat/completions",

open-sse/config/musicRegistry.ts

@@ -37,6 +37,20 @@ export const MUSIC_PROVIDERS: Record<string, MusicProvider> = {
     ],
   },
 
+  minimax: {
+    id: "minimax",
+    baseUrl: "https://api.minimax.io/v1/music_generation",
+    statusUrl: "https://api.minimax.io/v1/query/music_generation",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "minimax-music",
+    models: [
+      { id: "music-2.6", name: "Music 2.6" },
+      { id: "music-2.6-free", name: "Music 2.6 Free" },
+      { id: "music-cover", name: "Music Cover" },
+    ],
+  },
+
   comfyui: {
     id: "comfyui",
     baseUrl: "http://localhost:8188",

open-sse/config/videoRegistry.ts

@@ -52,6 +52,56 @@ export const VIDEO_PROVIDERS: Record<string, VideoProvider> = {
     ],
   },
 
+  pollinations: {
+    id: "pollinations",
+    baseUrl: "https://gen.pollinations.ai/video",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "pollinations-video",
+    models: [{ id: "default", name: "Pollinations Video (Free)" }],
+  },
+
+  minimax: {
+    id: "minimax",
+    baseUrl: "https://api.minimax.io/v1/video_generation",
+    statusUrl: "https://api.minimax.io/v1/query/video_generation",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "minimax-video",
+    models: [
+      { id: "MiniMax-Hailuo-2.3", name: "Hailuo 2.3" },
+      { id: "MiniMax-Hailuo-02", name: "Hailuo 02" },
+      { id: "T2V-01-Director", name: "T2V 01 Director" },
+    ],
+  },
+
+  together: {
+    id: "together",
+    baseUrl: "https://api.together.xyz/videos",
+    statusUrl: "https://api.together.xyz/videos",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "together-video",
+    models: [
+      { id: "wan-ai/wan2.1-t2v-480p", name: "Wan 2.1 T2V 480p" },
+      { id: "wan-ai/wan2.7-t2v", name: "Wan 2.7 T2V" },
+    ],
+  },
+
+  replicate: {
+    id: "replicate",
+    baseUrl: "https://api.replicate.com/v1/predictions",
+    statusUrl: "https://api.replicate.com/v1/predictions",
+    authType: "apikey",
+    authHeader: "bearer",
+    format: "replicate-video",
+    models: [
+      { id: "minimax/video-01", name: "MiniMax Video 01" },
+      { id: "wan-ai/wan2.1-t2v-480p", name: "Wan 2.1 T2V" },
+      { id: "tencent/hunyuan-video", name: "Hunyuan Video" },
+    ],
+  },
+
   comfyui: {
     id: "comfyui",
     baseUrl: "http://localhost:8188",

src/shared/constants/providers.ts

@@ -1660,7 +1660,14 @@ export const ENTERPRISE_CLOUD_PROVIDER_IDS = new Set([
   "modal",
 ]);
 
-export const VIDEO_PROVIDER_IDS = new Set(["runwayml", "veoaifree-web"]);
+export const VIDEO_PROVIDER_IDS = new Set([
+  "runwayml",
+  "veoaifree-web",
+  "pollinations",
+  "minimax",
+  "together",
+  "replicate",
+]);
 
 export const EMBEDDING_RERANK_PROVIDER_IDS = new Set(["voyage-ai", "jina-ai"]);