From 25aab8c55c6fb8a221b8f326917a32c4faa205dd Mon Sep 17 00:00:00 2001 From: Regis <92858615+Regis-RCR@users.noreply.github.com> Date: Mon, 16 Mar 2026 19:55:52 +0100 Subject: [PATCH] feat(audio): route audio requests to local provider_nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audio endpoints (/v1/audio/speech and /v1/audio/transcriptions) only supported hardcoded providers from audioRegistry.ts. Local inference backends configured as provider_nodes (e.g., MLX-Audio, oMLX) could not serve audio through OmniRoute. This adds a Phase 3 fallback in the audio model parser that consults provider_nodes from the database. Local providers with api_type=openai are automatically available for audio routing via their prefix (e.g., mlx-audio/tts-model, omlx/whisper-large-v3-turbo). Design: injection pattern — Next.js route handlers load provider_nodes (async DB query) and pass them to the sync parser as a parameter. No cross-workspace imports, no breaking changes to existing parsers. Changes: - Add buildDynamicAudioProvider() in audioRegistry.ts - Add Phase 3 (provider_nodes prefix match) to parseAudioModel() - Extend parseSpeechModel/parseTranscriptionModel with optional dynamicProviders parameter (backward compatible) - Load and inject provider_nodes in speech/transcription route handlers - Dynamic providers use authType=none (local, no credentials needed) --- open-sse/config/audioRegistry.ts | 54 +++++++++++++++++--- open-sse/handlers/audioSpeech.ts | 20 ++++++-- open-sse/handlers/audioTranscription.ts | 22 ++++++-- src/app/api/v1/audio/speech/route.ts | 46 +++++++++++++++-- src/app/api/v1/audio/transcriptions/route.ts | 49 ++++++++++++++++-- 5 files changed, 165 insertions(+), 26 deletions(-) diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index 4a2c8157bc..906bec7100 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -11,7 +11,7 @@ interface AudioModel { name: string; } -interface AudioProvider { +export interface AudioProvider { id: string; baseUrl: string; authType: string; @@ -262,36 +262,74 @@ export function getSpeechProvider(providerId: string): AudioProvider | null { return AUDIO_SPEECH_PROVIDERS[providerId] || null; } +export interface ProviderNodeRow { + prefix: string; + name: string; + baseUrl: string; + apiType?: string; +} + /** - * Parse audio model string (format: "provider/model" or just "model") + * Build a dynamic AudioProvider from a provider_node DB entry. + * Only used for local providers (localhost/127.0.0.1) — remote nodes are + * excluded by the caller to prevent auth bypass and SSRF. */ +export function buildDynamicAudioProvider(node: ProviderNodeRow, audioPath: string): AudioProvider { + if (!node.prefix || !node.baseUrl) { + throw new Error(`Invalid provider_node: missing prefix or baseUrl`); + } + const baseUrl = node.baseUrl.replace(/\/+$/, ""); + return { + id: node.prefix, + baseUrl: `${baseUrl}${audioPath}`, + authType: "none", + authHeader: "none", + models: [], + }; +} + function parseAudioModel( modelStr: string | null, - registry: Record + registry: Record, + dynamicProviders?: AudioProvider[] ): { provider: string | null; model: string | null } { if (!modelStr) return { provider: null, model: null }; - for (const [providerId, config] of Object.entries(registry)) { + // Phase 1: prefix match in hardcoded registry + for (const [providerId] of Object.entries(registry)) { if (modelStr.startsWith(providerId + "/")) { return { provider: providerId, model: modelStr.slice(providerId.length + 1) }; } } + // Phase 2: bare model lookup in hardcoded registry for (const [providerId, config] of Object.entries(registry)) { if (config.models.some((m) => m.id === modelStr)) { return { provider: providerId, model: modelStr }; } } + // Phase 3: prefix match in dynamic providers (provider_nodes) + if (dynamicProviders) { + for (const dp of dynamicProviders) { + if (modelStr.startsWith(dp.id + "/")) { + return { provider: dp.id, model: modelStr.slice(dp.id.length + 1) }; + } + } + } + return { provider: null, model: modelStr }; } -export function parseTranscriptionModel(modelStr: string | null) { - return parseAudioModel(modelStr, AUDIO_TRANSCRIPTION_PROVIDERS); +export function parseTranscriptionModel( + modelStr: string | null, + dynamicProviders?: AudioProvider[] +) { + return parseAudioModel(modelStr, AUDIO_TRANSCRIPTION_PROVIDERS, dynamicProviders); } -export function parseSpeechModel(modelStr: string | null) { - return parseAudioModel(modelStr, AUDIO_SPEECH_PROVIDERS); +export function parseSpeechModel(modelStr: string | null, dynamicProviders?: AudioProvider[]) { + return parseAudioModel(modelStr, AUDIO_SPEECH_PROVIDERS, dynamicProviders); } /** diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index abf4817a7f..e7f97d64a0 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -381,7 +381,12 @@ async function handleTortoiseSpeech(providerConfig, body) { * @returns {Response} */ /** @returns {Promise} */ -export async function handleAudioSpeech({ body, credentials }) { +export async function handleAudioSpeech({ + body, + credentials, + resolvedProvider = null, + resolvedModel = null, +}) { if (!body.model) { return errorResponse(400, "model is required"); } @@ -389,8 +394,15 @@ export async function handleAudioSpeech({ body, credentials }) { return errorResponse(400, "input is required"); } - const { provider: providerId, model: modelId } = parseSpeechModel(body.model); - const providerConfig = providerId ? getSpeechProvider(providerId) : null; + // Use pre-resolved provider/model from route handler if available (supports dynamic provider_nodes). + // Falls back to hardcoded registry lookup for backward compatibility. + let providerConfig = resolvedProvider; + let modelId = resolvedModel; + if (!providerConfig) { + const parsed = parseSpeechModel(body.model); + providerConfig = parsed.provider ? getSpeechProvider(parsed.provider) : null; + modelId = parsed.model; + } if (!providerConfig) { return errorResponse( @@ -403,7 +415,7 @@ export async function handleAudioSpeech({ body, credentials }) { const token = providerConfig.authType === "none" ? null : credentials?.apiKey || credentials?.accessToken; if (providerConfig.authType !== "none" && !token) { - return errorResponse(401, `No credentials for speech provider: ${providerId}`); + return errorResponse(401, `No credentials for speech provider: ${providerConfig.id}`); } try { diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index c7148f1af5..97f08e415e 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -13,7 +13,11 @@ import { getCorsOrigin } from "../utils/cors.ts"; * - HuggingFace Inference: POST raw binary to /models/{model_id} */ -import { getTranscriptionProvider, parseTranscriptionModel } from "../config/audioRegistry.ts"; +import { + getTranscriptionProvider, + parseTranscriptionModel, + type AudioProvider, +} from "../config/audioRegistry.ts"; import { buildAuthHeaders } from "../config/registryUtils.ts"; import { errorResponse } from "../utils/error.ts"; @@ -235,9 +239,13 @@ async function handleHuggingFaceTranscription(providerConfig, file, modelId, tok export async function handleAudioTranscription({ formData, credentials, + resolvedProvider = null, + resolvedModel = null, }: { formData: FormData; credentials?: TranscriptionCredentials | null; + resolvedProvider?: AudioProvider | null; + resolvedModel?: string | null; }): Promise { const model = formData.get("model"); if (typeof model !== "string" || !model) { @@ -250,8 +258,14 @@ export async function handleAudioTranscription({ } const file = fileEntry as Blob & { name?: unknown }; - const { provider: providerId, model: modelId } = parseTranscriptionModel(model); - const providerConfig = providerId ? getTranscriptionProvider(providerId) : null; + // Use pre-resolved provider/model from route handler if available (supports dynamic provider_nodes). + let providerConfig = resolvedProvider; + let modelId = resolvedModel; + if (!providerConfig) { + const parsed = parseTranscriptionModel(model); + providerConfig = parsed.provider ? getTranscriptionProvider(parsed.provider) : null; + modelId = parsed.model; + } if (!providerConfig) { return errorResponse( @@ -264,7 +278,7 @@ export async function handleAudioTranscription({ const token = providerConfig.authType === "none" ? null : credentials?.apiKey || credentials?.accessToken; if (providerConfig.authType !== "none" && !token) { - return errorResponse(401, `No credentials for transcription provider: ${providerId}`); + return errorResponse(401, `No credentials for transcription provider: ${providerConfig.id}`); } // Route to provider-specific handler diff --git a/src/app/api/v1/audio/speech/route.ts b/src/app/api/v1/audio/speech/route.ts index 7cdbf75424..f53e218067 100644 --- a/src/app/api/v1/audio/speech/route.ts +++ b/src/app/api/v1/audio/speech/route.ts @@ -6,10 +6,16 @@ import { extractApiKey, isValidApiKey, } from "@/sse/services/auth"; -import { parseSpeechModel, getSpeechProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; +import { + parseSpeechModel, + getSpeechProvider, + buildDynamicAudioProvider, + type ProviderNodeRow, +} from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; +import { getProviderNodes } from "@/lib/localDb"; import { v1AudioSpeechSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; @@ -55,7 +61,31 @@ export async function POST(request) { const policy = await enforceApiKeyPolicy(request, body.model); if (policy.rejection) return policy.rejection; - const { provider } = parseSpeechModel(body.model); + // Load local provider_nodes for audio routing (only localhost — prevents auth bypass/SSRF) + let dynamicProviders: ReturnType[] = []; + try { + const nodes = await getProviderNodes(); + dynamicProviders = (Array.isArray(nodes) ? nodes : []) + .filter((n: ProviderNodeRow) => { + if (n.apiType !== "chat" && n.apiType !== "responses") return false; + try { + const hostname = new URL(n.baseUrl).hostname; + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); + } catch { + return false; + } + }) + .map((n) => buildDynamicAudioProvider(n, "/audio/speech")); + } catch { + // DB error — fall back to hardcoded providers only + } + + const { provider, model: resolvedModel } = parseSpeechModel(body.model, dynamicProviders); if (!provider) { return errorResponse( HTTP_STATUS.BAD_REQUEST, @@ -63,8 +93,9 @@ export async function POST(request) { ); } - // Check provider config for auth bypass - const providerConfig = getSpeechProvider(provider); + // Check provider config — hardcoded first, then dynamic + const providerConfig = + getSpeechProvider(provider) || dynamicProviders.find((dp) => dp.id === provider) || null; // Get credentials — skip for local providers (authType: "none") let credentials = null; @@ -75,7 +106,12 @@ export async function POST(request) { } } - const response = await handleAudioSpeech({ body, credentials }); + const response = await handleAudioSpeech({ + body, + credentials, + resolvedProvider: providerConfig, + resolvedModel, + }); if (response?.ok) { await clearRecoveredProviderState(credentials); } diff --git a/src/app/api/v1/audio/transcriptions/route.ts b/src/app/api/v1/audio/transcriptions/route.ts index 93e594b24c..c0392b646f 100644 --- a/src/app/api/v1/audio/transcriptions/route.ts +++ b/src/app/api/v1/audio/transcriptions/route.ts @@ -6,10 +6,16 @@ import { extractApiKey, isValidApiKey, } from "@/sse/services/auth"; -import { parseTranscriptionModel, getTranscriptionProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; +import { + parseTranscriptionModel, + getTranscriptionProvider, + buildDynamicAudioProvider, + type ProviderNodeRow, +} from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; +import { getProviderNodes } from "@/lib/localDb"; /** * Handle CORS preflight @@ -53,7 +59,34 @@ export async function POST(request) { const policy = await enforceApiKeyPolicy(request, model as string); if (policy.rejection) return policy.rejection; - const { provider } = parseTranscriptionModel(model); + // Load local provider_nodes for audio routing (only localhost — prevents auth bypass/SSRF) + let dynamicProviders: ReturnType[] = []; + try { + const nodes = await getProviderNodes(); + dynamicProviders = (Array.isArray(nodes) ? nodes : []) + .filter((n: ProviderNodeRow) => { + if (n.apiType !== "chat" && n.apiType !== "responses") return false; + try { + const hostname = new URL(n.baseUrl).hostname; + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); + } catch { + return false; + } + }) + .map((n) => buildDynamicAudioProvider(n, "/audio/transcriptions")); + } catch { + // DB error — fall back to hardcoded providers only + } + + const { provider, model: resolvedModel } = parseTranscriptionModel( + model as string, + dynamicProviders + ); if (!provider) { return errorResponse( HTTP_STATUS.BAD_REQUEST, @@ -61,8 +94,9 @@ export async function POST(request) { ); } - // Check provider config for auth bypass - const providerConfig = getTranscriptionProvider(provider); + // Check provider config — hardcoded first, then dynamic + const providerConfig = + getTranscriptionProvider(provider) || dynamicProviders.find((dp) => dp.id === provider) || null; // Get credentials — skip for local providers (authType: "none") let credentials = null; @@ -73,7 +107,12 @@ export async function POST(request) { } } - const response = await handleAudioTranscription({ formData, credentials }); + const response = await handleAudioTranscription({ + formData, + credentials, + resolvedProvider: providerConfig, + resolvedModel, + }); if (response?.ok) { await clearRecoveredProviderState(credentials); }