diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index 3f56c66ec8..7651477369 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -48,6 +48,28 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { ], }, + openrouter: { + id: "openrouter", + baseUrl: "https://openrouter.ai/api/v1/audio/transcriptions", + authType: "apikey", + authHeader: "bearer", + format: "openrouter-stt", + supportedFormats: ["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"], + models: [ + { id: "deepgram/nova-3", name: "Deepgram Nova-3" }, + { id: "microsoft/mai-transcribe-1.5", name: "Microsoft MAI-Transcribe 1.5" }, + { id: "nvidia/parakeet-tdt-0.6b-v3", name: "NVIDIA Parakeet TDT 0.6B v3" }, + { id: "mistralai/voxtral-mini-transcribe", name: "Mistral Voxtral Mini Transcribe" }, + { id: "qwen/qwen3-asr-flash-2026-02-10", name: "Qwen3 ASR Flash 2026-02-10" }, + { id: "google/chirp-3", name: "Google Chirp 3" }, + { id: "openai/gpt-4o-mini-transcribe", name: "OpenAI GPT-4o Mini Transcribe" }, + { id: "openai/whisper-large-v3", name: "OpenAI Whisper Large v3" }, + { id: "openai/whisper-large-v3-turbo", name: "OpenAI Whisper Large v3 Turbo" }, + { id: "openai/whisper-1", name: "OpenAI Whisper 1" }, + { id: "openai/gpt-4o-transcribe", name: "OpenAI GPT-4o Transcribe" }, + ], + }, + cohere: { id: "cohere", baseUrl: "https://api.cohere.com/v2/audio/transcriptions", diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index 692b30df35..55762eb4cf 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -24,6 +24,7 @@ import { buildAuthHeaders } from "../config/registryUtils.ts"; import { kieExecutor } from "../executors/kie.ts"; import { vertexTranscribe } from "../executors/vertexMedia.ts"; import { errorResponse } from "../utils/error.ts"; +import { handleOpenRouterTranscription } from "./openrouterTranscription.ts"; type TranscriptionCredentials = { apiKey?: string; @@ -33,7 +34,7 @@ type TranscriptionCredentials = { /** * Return a CORS error response from an upstream fetch failure */ -function upstreamErrorResponse(res, errText) { +export function upstreamErrorResponse(res, errText) { // Always return JSON so the client can parse the error reliably let errorMessage: string; try { @@ -671,7 +672,7 @@ export async function handleAudioTranscription({ if (!providerConfig) { return errorResponse( 400, - `No transcription provider found for model "${model}". Available: openai, groq, deepgram, assemblyai, nvidia, huggingface, qwen, gladia, rev-ai, speechmatics` + `No transcription provider found for model "${model}". Available: openai, openrouter, groq, deepgram, assemblyai, nvidia, huggingface, qwen, gladia, rev-ai, speechmatics` ); } @@ -741,6 +742,10 @@ export async function handleAudioTranscription({ return handleSpeechmaticsTranscription(providerConfig, file, modelId, token); } + if (providerConfig.format === "openrouter-stt") { + return handleOpenRouterTranscription(providerConfig, file, modelId, token, formData); + } + // Default: OpenAI/Groq/Qwen3-compatible multipart proxy const extraFields: Record = {}; for (const key of [ diff --git a/open-sse/handlers/openrouterTranscription.ts b/open-sse/handlers/openrouterTranscription.ts new file mode 100644 index 0000000000..3c2f7796cb --- /dev/null +++ b/open-sse/handlers/openrouterTranscription.ts @@ -0,0 +1,101 @@ +import { Buffer } from "node:buffer"; +/** + * OpenRouter Transcription Handler + * + * Extracted from audioTranscription.ts to keep that file under the file-size + * cap. Handles the OpenRouter-specific `openrouter-stt` provider format, which + * uses a dedicated JSON STT endpoint (`input_audio` { data: base64, format }) + * rather than the standard Whisper-style multipart proxy. + */ + +import type { AudioProvider } from "../config/audioRegistry.ts"; +import { buildAuthHeaders } from "../config/registryUtils.ts"; +import { errorResponse } from "../utils/error.ts"; +import { upstreamErrorResponse } from "./audioTranscription.ts"; + +/** + * Resolve the audio container format OpenRouter's dedicated STT endpoint + * expects, from the uploaded file's extension first, then its MIME type. + * Falls back to "wav" when neither is recognisable. + */ +export function resolveOpenRouterAudioFormat(file: Blob & { name?: unknown }): string { + const fileName = typeof file.name === "string" ? file.name.toLowerCase() : ""; + const extension = fileName.includes(".") ? fileName.split(".").pop() || "" : ""; + if (["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"].includes(extension)) { + return extension; + } + const mimeFormats: Record = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/flac": "flac", + "audio/x-flac": "flac", + "audio/mp4": "m4a", + "audio/ogg": "ogg", + "audio/webm": "webm", + "audio/aac": "aac", + }; + // Browser-recorded blobs carry codec params (e.g. "audio/webm;codecs=opus"); + // match on the base MIME type only. + const mimeType = (file.type || "").split(";")[0].trim().toLowerCase(); + return mimeFormats[mimeType] || "wav"; +} + +/** + * Handle OpenRouter transcription via its dedicated STT endpoint. + * Converts the multipart audio upload into OpenRouter's JSON + * `input_audio` { data: base64, format } shape and forwards optional + * language / temperature / response_format / timestamp_granularities fields + * when present (temperature is coerced to a number for the JSON payload). + */ +export async function handleOpenRouterTranscription( + provider: AudioProvider, + file: Blob & { name?: unknown }, + model: string | null, + token: string | null, + formData: FormData +): Promise { + const body: Record = { + model, + input_audio: { + data: Buffer.from(await file.arrayBuffer()).toString("base64"), + format: resolveOpenRouterAudioFormat(file), + }, + }; + const language = formData.get("language"); + if (language !== null) body.language = String(language); + + const responseFormat = formData.get("response_format"); + if (responseFormat !== null) body.response_format = String(responseFormat); + + // temperature arrives as a string from multipart form data; the JSON payload + // needs a number or the upstream API rejects it. + const temperature = formData.get("temperature"); + if (temperature !== null) { + const parsed = Number.parseFloat(String(temperature)); + if (!Number.isNaN(parsed)) body.temperature = parsed; + } + + // Forward timestamp granularities as a JSON array (the multipart path sends + // them as repeated `timestamp_granularities[]` fields). + const granularities = formData.getAll("timestamp_granularities[]"); + if (granularities.length > 0) { + body.timestamp_granularities = granularities.map(String); + } + try { + const res = await fetch(provider.baseUrl, { + method: "POST", + headers: { ...buildAuthHeaders(provider, token), "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) return upstreamErrorResponse(res, await res.text()); + return new Response(await res.text(), { + status: res.status, + headers: { "Content-Type": res.headers.get("content-type") || "application/json" }, + }); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + return errorResponse(500, `Transcription request failed: ${error.message}`); + } +} diff --git a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx index c1eed94d7c..125c631f91 100644 --- a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx +++ b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx @@ -117,6 +117,15 @@ export default function MediaProviderPageClient({ return (
+ {activeKind === "stt" && providerId === "openrouter" && ( +
+ info + + Existing connection: OpenRouter speech-to-text models use your + configured OpenRouter connection. No separate credential is required. + +
+ )} m.type === "audio" && m.subtype === "transcription"); + + const firstModel = sttModels[0]?.id ?? "whisper-1"; const [model, setModel] = useState(""); const [file, setFile] = useState(null); const [fileError, setFileError] = useState(null); @@ -53,6 +59,13 @@ export function SttExampleCard({ providerId }: Props) { const effectiveModel = model || firstModel; + // The transcription route resolves the provider from the leading segment of + // the submitted model id, so a bare id (e.g. "deepgram/nova-3" for the + // OpenRouter connection) would misroute. Qualify it with this connection's + // provider id at submit time. Single-token ids (e.g. "whisper-1") already + // lack a slash and pass through unchanged once prefixed. + const qualify = (id: string) => (id.startsWith(`${providerId}/`) ? id : `${providerId}/${id}`); + // cURL is multipart — show a representative snippet const curlSnippet = buildCurl({ endpoint: @@ -62,7 +75,7 @@ export function SttExampleCard({ providerId }: Props) { Authorization: `Bearer ${apiKey || ""}`, }, body: { - model: effectiveModel, + model: qualify(effectiveModel), file: "", }, }); @@ -89,7 +102,7 @@ export function SttExampleCard({ providerId }: Props) { const t0 = performance.now(); try { const formData = new FormData(); - formData.append("model", effectiveModel); + formData.append("model", qualify(effectiveModel)); formData.append("file", file); const res = await fetch(ENDPOINT_PATH, { @@ -115,7 +128,7 @@ export function SttExampleCard({ providerId }: Props) { } }; - const modelOptions = models.length > 0 ? models : [{ id: "whisper-1" }]; + const modelOptions = sttModels.length > 0 ? sttModels : [{ id: "whisper-1" }]; return ( { + const { parseTranscriptionModel } = await import("../../open-sse/config/audioRegistry.ts"); + for (const model of OPENROUTER_TRANSCRIPTION_MODELS) { + assert.deepEqual(parseTranscriptionModel(`openrouter/${model}`), { + provider: "openrouter", + model, + }); + } +}); + +test("OpenRouter transcription converts multipart audio to dedicated STT input_audio", async () => { + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + let capturedInit; + globalThis.fetch = async (url, init) => { + capturedUrl = String(url); + capturedInit = init; + return Response.json({ text: "hello from MAI" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.set("language", "pt"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { text: "hello from MAI" }); + assert.equal(capturedUrl, "https://openrouter.ai/api/v1/audio/transcriptions"); + const headers = capturedInit?.headers as Record; + assert.equal(headers.Authorization, "Bearer or-test-key"); + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.model, "microsoft/mai-transcribe-1.5"); + assert.equal(body.input_audio.format, "flac"); + assert.equal(body.input_audio.data, "AQID"); + assert.equal(body.language, "pt"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription sends a string temperature as a JSON number", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.set("temperature", "0.2"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.strictEqual(body.temperature, 0.2); + assert.strictEqual(typeof body.temperature, "number"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription forwards timestamp_granularities[] as a JSON array", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.append("timestamp_granularities[]", "word"); + form.append("timestamp_granularities[]", "segment"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.ok(Array.isArray(body.timestamp_granularities)); + assert.deepEqual(body.timestamp_granularities, ["word", "segment"]); + // The bracketed multipart field name must not leak into the JSON payload. + assert.equal(body["timestamp_granularities[]"], undefined); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription strips codec params from the MIME type when resolving format", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + // Browser-recorded blob: MIME carries codec params and the filename has no + // recognisable extension, so resolution must fall through to the base MIME. + form.set("file", buildFile([1, 2, 3], "recording", "audio/webm;codecs=opus")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.input_audio.format, "webm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription routes a fully-qualified openrouter// id", async () => { + // The SttExampleCard qualifies bare ids to `openrouter//` + // before submitting. Confirm the handler resolves that shape to the + // OpenRouter STT endpoint with the nested model id preserved. + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + let capturedInit; + globalThis.fetch = async (url, init) => { + capturedUrl = String(url); + capturedInit = init; + return Response.json({ text: "routed" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/deepgram/nova-3"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + assert.equal(capturedUrl, "https://openrouter.ai/api/v1/audio/transcriptions"); + const headers = capturedInit?.headers as Record; + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.model, "deepgram/nova-3"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/provider-models-discovery-split.test.ts b/tests/unit/provider-models-discovery-split.test.ts index bdb9d2d871..3a49bcb25f 100644 --- a/tests/unit/provider-models-discovery-split.test.ts +++ b/tests/unit/provider-models-discovery-split.test.ts @@ -149,6 +149,25 @@ test("providerModelsConfig aimlapi.parseResponse keeps only chat-completion mode assert.deepEqual(parsed, [{ id: "chat-1", name: "Chat 1" }]); }); +test("providerModelsConfig openrouter.parseResponse keeps the full catalog (LLMs not filtered out)", () => { + // Generic OpenRouter discovery must stay unfiltered so sync/import/pickers + // and /v1/models keep every LLM. STT narrowing lives on the STT card, not here. + const data = { + data: [ + { id: "openai/gpt-4o", architecture: { modality: "text->text" } }, + { + id: "openai/whisper-1", + architecture: { modality: "audio->transcription" }, + }, + ], + }; + const parsed = PROVIDER_MODELS_CONFIG.openrouter.parseResponse(data); + assert.deepEqual( + parsed.map((m: { id: string }) => m.id), + ["openai/gpt-4o", "openai/whisper-1"] + ); +}); + // ── codex discovery leaf ──────────────────────────────────────────────────── test.beforeEach(() => {