From a6dafa0ff799f7989a1b1b6e81ed8f571e6fd57f Mon Sep 17 00:00:00 2001 From: Marcus Bearden Date: Mon, 20 Jul 2026 19:56:34 +0100 Subject: [PATCH] feat(providers): add OpenRouter speech-to-text (audio transcription) provider (#7861) * feat(providers): add OpenRouter speech-to-text (audio transcription) provider Adds OpenRouter as a speech-to-text provider for POST /v1/audio/transcriptions. Transcription requests route to OpenRouter's dedicated STT endpoint (https://openrouter.ai/api/v1/audio/transcriptions), converting the multipart audio upload into OpenRouter's JSON input_audio { data, format } shape and forwarding optional language, temperature, and response_format fields. OpenRouter STT reuses your existing OpenRouter connection, so no separate credential is required. Eleven transcription models are available (Deepgram Nova-3, Microsoft MAI-Transcribe 1.5, NVIDIA Parakeet, Mistral Voxtral, Qwen3 ASR, Google Chirp 3, and the OpenAI Whisper / GPT-4o transcribe family). The media-providers STT playground card now narrows its model picker to transcription-only models, so the OpenRouter chat catalog is not shown for speech-to-text, and the provider page shows an existing-connection note for OpenRouter STT. * fix(providers): address OpenRouter STT review feedback - Extract the OpenRouter transcription handler into open-sse/handlers/openrouterTranscription.ts to keep audioTranscription.ts under the file-size cap. - Qualify the STT card's submitted model id with the connection's provider prefix so OpenRouter models route to OpenRouter rather than the model's own vendor (the transcription route resolves the provider from the leading segment of the model id). - Coerce temperature to a number and forward timestamp_granularities on the JSON input_audio payload; match the base MIME type when resolving the audio format so codec parameters (e.g. audio/webm;codecs=opus) do not fall back to wav. - Split the OpenRouter cases into tests/unit/audio-transcription-openrouter.test.ts and add coverage for temperature, timestamp granularities, MIME codec params, and qualified-id routing. --- open-sse/config/audioRegistry.ts | 22 ++ open-sse/handlers/audioTranscription.ts | 9 +- open-sse/handlers/openrouterTranscription.ts | 101 ++++++++++ .../[kind]/[id]/MediaProviderPageClient.tsx | 9 + .../components/SttExampleCard.tsx | 21 +- .../providers/hooks/useProviderModels.ts | 4 + .../unit/audio-transcription-handler.test.ts | 32 +-- .../audio-transcription-openrouter.test.ts | 190 ++++++++++++++++++ .../provider-models-discovery-split.test.ts | 19 ++ 9 files changed, 385 insertions(+), 22 deletions(-) create mode 100644 open-sse/handlers/openrouterTranscription.ts create mode 100644 tests/unit/audio-transcription-openrouter.test.ts diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index 3f56c66ec8..7651477369 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -48,6 +48,28 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { ], }, + openrouter: { + id: "openrouter", + baseUrl: "https://openrouter.ai/api/v1/audio/transcriptions", + authType: "apikey", + authHeader: "bearer", + format: "openrouter-stt", + supportedFormats: ["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"], + models: [ + { id: "deepgram/nova-3", name: "Deepgram Nova-3" }, + { id: "microsoft/mai-transcribe-1.5", name: "Microsoft MAI-Transcribe 1.5" }, + { id: "nvidia/parakeet-tdt-0.6b-v3", name: "NVIDIA Parakeet TDT 0.6B v3" }, + { id: "mistralai/voxtral-mini-transcribe", name: "Mistral Voxtral Mini Transcribe" }, + { id: "qwen/qwen3-asr-flash-2026-02-10", name: "Qwen3 ASR Flash 2026-02-10" }, + { id: "google/chirp-3", name: "Google Chirp 3" }, + { id: "openai/gpt-4o-mini-transcribe", name: "OpenAI GPT-4o Mini Transcribe" }, + { id: "openai/whisper-large-v3", name: "OpenAI Whisper Large v3" }, + { id: "openai/whisper-large-v3-turbo", name: "OpenAI Whisper Large v3 Turbo" }, + { id: "openai/whisper-1", name: "OpenAI Whisper 1" }, + { id: "openai/gpt-4o-transcribe", name: "OpenAI GPT-4o Transcribe" }, + ], + }, + cohere: { id: "cohere", baseUrl: "https://api.cohere.com/v2/audio/transcriptions", diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index 692b30df35..55762eb4cf 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -24,6 +24,7 @@ import { buildAuthHeaders } from "../config/registryUtils.ts"; import { kieExecutor } from "../executors/kie.ts"; import { vertexTranscribe } from "../executors/vertexMedia.ts"; import { errorResponse } from "../utils/error.ts"; +import { handleOpenRouterTranscription } from "./openrouterTranscription.ts"; type TranscriptionCredentials = { apiKey?: string; @@ -33,7 +34,7 @@ type TranscriptionCredentials = { /** * Return a CORS error response from an upstream fetch failure */ -function upstreamErrorResponse(res, errText) { +export function upstreamErrorResponse(res, errText) { // Always return JSON so the client can parse the error reliably let errorMessage: string; try { @@ -671,7 +672,7 @@ export async function handleAudioTranscription({ if (!providerConfig) { return errorResponse( 400, - `No transcription provider found for model "${model}". Available: openai, groq, deepgram, assemblyai, nvidia, huggingface, qwen, gladia, rev-ai, speechmatics` + `No transcription provider found for model "${model}". Available: openai, openrouter, groq, deepgram, assemblyai, nvidia, huggingface, qwen, gladia, rev-ai, speechmatics` ); } @@ -741,6 +742,10 @@ export async function handleAudioTranscription({ return handleSpeechmaticsTranscription(providerConfig, file, modelId, token); } + if (providerConfig.format === "openrouter-stt") { + return handleOpenRouterTranscription(providerConfig, file, modelId, token, formData); + } + // Default: OpenAI/Groq/Qwen3-compatible multipart proxy const extraFields: Record = {}; for (const key of [ diff --git a/open-sse/handlers/openrouterTranscription.ts b/open-sse/handlers/openrouterTranscription.ts new file mode 100644 index 0000000000..3c2f7796cb --- /dev/null +++ b/open-sse/handlers/openrouterTranscription.ts @@ -0,0 +1,101 @@ +import { Buffer } from "node:buffer"; +/** + * OpenRouter Transcription Handler + * + * Extracted from audioTranscription.ts to keep that file under the file-size + * cap. Handles the OpenRouter-specific `openrouter-stt` provider format, which + * uses a dedicated JSON STT endpoint (`input_audio` { data: base64, format }) + * rather than the standard Whisper-style multipart proxy. + */ + +import type { AudioProvider } from "../config/audioRegistry.ts"; +import { buildAuthHeaders } from "../config/registryUtils.ts"; +import { errorResponse } from "../utils/error.ts"; +import { upstreamErrorResponse } from "./audioTranscription.ts"; + +/** + * Resolve the audio container format OpenRouter's dedicated STT endpoint + * expects, from the uploaded file's extension first, then its MIME type. + * Falls back to "wav" when neither is recognisable. + */ +export function resolveOpenRouterAudioFormat(file: Blob & { name?: unknown }): string { + const fileName = typeof file.name === "string" ? file.name.toLowerCase() : ""; + const extension = fileName.includes(".") ? fileName.split(".").pop() || "" : ""; + if (["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"].includes(extension)) { + return extension; + } + const mimeFormats: Record = { + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/flac": "flac", + "audio/x-flac": "flac", + "audio/mp4": "m4a", + "audio/ogg": "ogg", + "audio/webm": "webm", + "audio/aac": "aac", + }; + // Browser-recorded blobs carry codec params (e.g. "audio/webm;codecs=opus"); + // match on the base MIME type only. + const mimeType = (file.type || "").split(";")[0].trim().toLowerCase(); + return mimeFormats[mimeType] || "wav"; +} + +/** + * Handle OpenRouter transcription via its dedicated STT endpoint. + * Converts the multipart audio upload into OpenRouter's JSON + * `input_audio` { data: base64, format } shape and forwards optional + * language / temperature / response_format / timestamp_granularities fields + * when present (temperature is coerced to a number for the JSON payload). + */ +export async function handleOpenRouterTranscription( + provider: AudioProvider, + file: Blob & { name?: unknown }, + model: string | null, + token: string | null, + formData: FormData +): Promise { + const body: Record = { + model, + input_audio: { + data: Buffer.from(await file.arrayBuffer()).toString("base64"), + format: resolveOpenRouterAudioFormat(file), + }, + }; + const language = formData.get("language"); + if (language !== null) body.language = String(language); + + const responseFormat = formData.get("response_format"); + if (responseFormat !== null) body.response_format = String(responseFormat); + + // temperature arrives as a string from multipart form data; the JSON payload + // needs a number or the upstream API rejects it. + const temperature = formData.get("temperature"); + if (temperature !== null) { + const parsed = Number.parseFloat(String(temperature)); + if (!Number.isNaN(parsed)) body.temperature = parsed; + } + + // Forward timestamp granularities as a JSON array (the multipart path sends + // them as repeated `timestamp_granularities[]` fields). + const granularities = formData.getAll("timestamp_granularities[]"); + if (granularities.length > 0) { + body.timestamp_granularities = granularities.map(String); + } + try { + const res = await fetch(provider.baseUrl, { + method: "POST", + headers: { ...buildAuthHeaders(provider, token), "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) return upstreamErrorResponse(res, await res.text()); + return new Response(await res.text(), { + status: res.status, + headers: { "Content-Type": res.headers.get("content-type") || "application/json" }, + }); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + return errorResponse(500, `Transcription request failed: ${error.message}`); + } +} diff --git a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx index c1eed94d7c..125c631f91 100644 --- a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx +++ b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx @@ -117,6 +117,15 @@ export default function MediaProviderPageClient({ return (
+ {activeKind === "stt" && providerId === "openrouter" && ( +
+ info + + Existing connection: OpenRouter speech-to-text models use your + configured OpenRouter connection. No separate credential is required. + +
+ )} m.type === "audio" && m.subtype === "transcription"); + + const firstModel = sttModels[0]?.id ?? "whisper-1"; const [model, setModel] = useState(""); const [file, setFile] = useState(null); const [fileError, setFileError] = useState(null); @@ -53,6 +59,13 @@ export function SttExampleCard({ providerId }: Props) { const effectiveModel = model || firstModel; + // The transcription route resolves the provider from the leading segment of + // the submitted model id, so a bare id (e.g. "deepgram/nova-3" for the + // OpenRouter connection) would misroute. Qualify it with this connection's + // provider id at submit time. Single-token ids (e.g. "whisper-1") already + // lack a slash and pass through unchanged once prefixed. + const qualify = (id: string) => (id.startsWith(`${providerId}/`) ? id : `${providerId}/${id}`); + // cURL is multipart — show a representative snippet const curlSnippet = buildCurl({ endpoint: @@ -62,7 +75,7 @@ export function SttExampleCard({ providerId }: Props) { Authorization: `Bearer ${apiKey || ""}`, }, body: { - model: effectiveModel, + model: qualify(effectiveModel), file: "", }, }); @@ -89,7 +102,7 @@ export function SttExampleCard({ providerId }: Props) { const t0 = performance.now(); try { const formData = new FormData(); - formData.append("model", effectiveModel); + formData.append("model", qualify(effectiveModel)); formData.append("file", file); const res = await fetch(ENDPOINT_PATH, { @@ -115,7 +128,7 @@ export function SttExampleCard({ providerId }: Props) { } }; - const modelOptions = models.length > 0 ? models : [{ id: "whisper-1" }]; + const modelOptions = sttModels.length > 0 ? sttModels : [{ id: "whisper-1" }]; return ( { + const { parseTranscriptionModel } = await import("../../open-sse/config/audioRegistry.ts"); + for (const model of OPENROUTER_TRANSCRIPTION_MODELS) { + assert.deepEqual(parseTranscriptionModel(`openrouter/${model}`), { + provider: "openrouter", + model, + }); + } +}); + +test("OpenRouter transcription converts multipart audio to dedicated STT input_audio", async () => { + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + let capturedInit; + globalThis.fetch = async (url, init) => { + capturedUrl = String(url); + capturedInit = init; + return Response.json({ text: "hello from MAI" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.set("language", "pt"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { text: "hello from MAI" }); + assert.equal(capturedUrl, "https://openrouter.ai/api/v1/audio/transcriptions"); + const headers = capturedInit?.headers as Record; + assert.equal(headers.Authorization, "Bearer or-test-key"); + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.model, "microsoft/mai-transcribe-1.5"); + assert.equal(body.input_audio.format, "flac"); + assert.equal(body.input_audio.data, "AQID"); + assert.equal(body.language, "pt"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription sends a string temperature as a JSON number", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.set("temperature", "0.2"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.strictEqual(body.temperature, 0.2); + assert.strictEqual(typeof body.temperature, "number"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription forwards timestamp_granularities[] as a JSON array", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + form.append("timestamp_granularities[]", "word"); + form.append("timestamp_granularities[]", "segment"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.ok(Array.isArray(body.timestamp_granularities)); + assert.deepEqual(body.timestamp_granularities, ["word", "segment"]); + // The bracketed multipart field name must not leak into the JSON payload. + assert.equal(body["timestamp_granularities[]"], undefined); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription strips codec params from the MIME type when resolving format", async () => { + const originalFetch = globalThis.fetch; + let capturedInit; + globalThis.fetch = async (_url, init) => { + capturedInit = init; + return Response.json({ text: "ok" }); + }; + + try { + const form = new FormData(); + // Browser-recorded blob: MIME carries codec params and the filename has no + // recognisable extension, so resolution must fall through to the base MIME. + form.set("file", buildFile([1, 2, 3], "recording", "audio/webm;codecs=opus")); + form.set("model", "openrouter/microsoft/mai-transcribe-1.5"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.input_audio.format, "webm"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("OpenRouter transcription routes a fully-qualified openrouter// id", async () => { + // The SttExampleCard qualifies bare ids to `openrouter//` + // before submitting. Confirm the handler resolves that shape to the + // OpenRouter STT endpoint with the nested model id preserved. + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + let capturedInit; + globalThis.fetch = async (url, init) => { + capturedUrl = String(url); + capturedInit = init; + return Response.json({ text: "routed" }); + }; + + try { + const form = new FormData(); + form.set("file", buildFile([1, 2, 3], "sample.flac", "audio/flac")); + form.set("model", "openrouter/deepgram/nova-3"); + + const response = await handleAudioTranscription({ + formData: form, + credentials: { apiKey: "or-test-key" }, + }); + + assert.equal(response.status, 200); + assert.equal(capturedUrl, "https://openrouter.ai/api/v1/audio/transcriptions"); + const headers = capturedInit?.headers as Record; + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(String(capturedInit?.body)); + assert.equal(body.model, "deepgram/nova-3"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/provider-models-discovery-split.test.ts b/tests/unit/provider-models-discovery-split.test.ts index bdb9d2d871..3a49bcb25f 100644 --- a/tests/unit/provider-models-discovery-split.test.ts +++ b/tests/unit/provider-models-discovery-split.test.ts @@ -149,6 +149,25 @@ test("providerModelsConfig aimlapi.parseResponse keeps only chat-completion mode assert.deepEqual(parsed, [{ id: "chat-1", name: "Chat 1" }]); }); +test("providerModelsConfig openrouter.parseResponse keeps the full catalog (LLMs not filtered out)", () => { + // Generic OpenRouter discovery must stay unfiltered so sync/import/pickers + // and /v1/models keep every LLM. STT narrowing lives on the STT card, not here. + const data = { + data: [ + { id: "openai/gpt-4o", architecture: { modality: "text->text" } }, + { + id: "openai/whisper-1", + architecture: { modality: "audio->transcription" }, + }, + ], + }; + const parsed = PROVIDER_MODELS_CONFIG.openrouter.parseResponse(data); + assert.deepEqual( + parsed.map((m: { id: string }) => m.id), + ["openai/gpt-4o", "openai/whisper-1"] + ); +}); + // ── codex discovery leaf ──────────────────────────────────────────────────── test.beforeEach(() => {