From a5d603ebc986c05e1dd92a79779b11469118d631 Mon Sep 17 00:00:00 2001 From: Amirreza Kimiyaei Date: Fri, 18 Sep 2026 17:59:46 +0330 Subject: [PATCH] fix(soniox): pass client parameters through and surface speaker diarization (#12948) handleSonioxTranscription took no formData, built the job body from a fixed three-key object, and reduced the transcript to { text }. Every client-supplied parameter was therefore dropped in silence: the request returned 200, the flag had no effect, and from the caller's side an unsupported parameter and a dropped one looked identical. Diarization and Soniox's `context` were both unreachable, and the per-token `speaker` attribution Soniox returns was discarded. The handler now receives the form data (as the Deepgram one already did) and maps what the caller asked for onto the job: diarization under the three spellings callers reach for, `context` verbatim, and `language` as a Soniox language hint. Keys are added only when requested, so a request carrying no options produces byte-identical job bodies and the existing audio-soniox-provider deep-equality assertions still hold. Response shape stays `{ text }` by default. When diarization or response_format=verbose_json is requested, the token stream is collapsed into contiguous single-speaker runs and returned as OpenAI-style `segments` carrying `speaker`, with `words` when word granularity is asked for. Verified against a live Soniox account on a real two-party Persian phone call: default response unchanged, 13 segments over 2 distinct speakers with turn boundaries matching the dialogue, and `context` correcting a proper noun the model otherwise gets wrong. Closes #12947 Co-authored-by: Claude Opus 5 --- open-sse/handlers/audioTranscription.ts | 147 ++++++++++++-- ...niox-diarization-and-client-params.test.ts | 185 ++++++++++++++++++ 2 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 tests/unit/soniox-diarization-and-client-params.test.ts diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index da4e1ccfd4..92db782a01 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -352,11 +352,94 @@ async function handleGladiaTranscription(providerConfig, file, modelId, token) { return errorResponse(504, "Gladia transcription timed out after 120s"); } +type SonioxToken = { + text?: string; + start_ms?: number; + end_ms?: number; + speaker?: string | number | null; + language?: string | null; +}; + +type SonioxOptions = { + diarize: boolean; + context: string; + language: string; + verbose: boolean; + wantWords: boolean; +}; + +/** + * Client-settable Soniox job options, read off the multipart form. + * + * Diarization is accepted under three spellings because callers reach for + * whichever their previous provider used: Soniox's own + * `enable_speaker_diarization`, the generic `diarization`, and Deepgram's + * `speaker_labels`. Anything absent stays absent from the job body so a + * caller who sends nothing produces byte-identical requests to before. + */ +function readSonioxOptions(formData?: FormData): SonioxOptions { + const str = (key: string): string => { + const value = formData?.get(key); + return typeof value === "string" ? value.trim() : ""; + }; + const flag = (...keys: string[]): boolean => + keys.some((key) => /^(1|true|yes|on)$/i.test(str(key))); + + const granularities = (formData?.getAll?.("timestamp_granularities[]") ?? []).map((value) => + String(value).toLowerCase() + ); + const diarize = flag("enable_speaker_diarization", "diarization", "speaker_labels"); + const responseFormat = str("response_format").toLowerCase(); + + return { + diarize, + context: str("context"), + language: str("language"), + // Diarization implies the richer body: a caller who asked who-spoke-when and + // got `{text}` back has no way to tell the flag was honoured. + verbose: diarize || responseFormat === "verbose_json", + wantWords: granularities.includes("word"), + }; +} + +/** + * Collapse a token stream into contiguous single-speaker runs. Soniox labels + * every token, so a turn boundary is simply the point where the label changes. + */ +function groupSonioxTokensBySpeaker(tokens: SonioxToken[]) { + const segments: { speaker: string | null; startMs: number; endMs: number; text: string }[] = []; + + for (const token of tokens) { + const speaker = token.speaker == null ? null : String(token.speaker); + const previous = segments[segments.length - 1]; + if (!previous || previous.speaker !== speaker) { + segments.push({ + speaker, + startMs: token.start_ms ?? 0, + endMs: token.end_ms ?? token.start_ms ?? 0, + text: token.text ?? "", + }); + continue; + } + previous.endMs = token.end_ms ?? previous.endMs; + previous.text += token.text ?? ""; + } + + return segments; +} + /** * Handle Soniox transcription (async: upload file → create job → poll → get transcript) */ -async function handleSonioxTranscription(providerConfig, file, modelId, token) { +async function handleSonioxTranscription( + providerConfig, + file, + modelId, + token, + formData?: FormData +) { const authHeaders = buildAuthHeaders(providerConfig, token); + const options = readSonioxOptions(formData); const { body: uploadBody, contentType: uploadContentType } = await buildMultipartBody(file, {}); const uploadRes = await fetch("https://api.soniox.com/v1/files", { @@ -369,14 +452,21 @@ async function handleSonioxTranscription(providerConfig, file, modelId, token) { } const fileId = (await uploadRes.json()).id; + // Only keys the caller actually asked for are added, so a request with no + // options produces exactly the body this handler has always sent. + const jobBody: Record = { + model: modelId, + file_id: fileId, + enable_language_identification: true, + }; + if (options.diarize) jobBody.enable_speaker_diarization = true; + if (options.context) jobBody.context = options.context; + if (options.language) jobBody.language_hints = [options.language]; + const createRes = await fetch(providerConfig.baseUrl, { method: "POST", headers: { ...authHeaders, "Content-Type": "application/json" }, - body: JSON.stringify({ - model: modelId, - file_id: fileId, - enable_language_identification: true, - }), + body: JSON.stringify(jobBody), }); if (!createRes.ok) { return upstreamErrorResponse(createRes, await createRes.text()); @@ -414,14 +504,49 @@ async function handleSonioxTranscription(providerConfig, file, modelId, token) { return upstreamErrorResponse(transcriptRes, await transcriptRes.text()); } const transcript = await transcriptRes.json(); + const tokens: SonioxToken[] = Array.isArray(transcript.tokens) ? transcript.tokens : []; const text = typeof transcript.text === "string" && transcript.text.length > 0 ? transcript.text - : Array.isArray(transcript.tokens) - ? transcript.tokens.map((t: { text?: string }) => t.text ?? "").join("") - : ""; + : tokens.map((t) => t.text ?? "").join(""); - return Response.json({ text }, { headers: { ...CORS_HEADERS } }); + // Default contract is unchanged: callers who asked for nothing still get + // exactly `{ text }`, which is what every existing client parses. + if (!options.verbose) { + return Response.json({ text }, { headers: { ...CORS_HEADERS } }); + } + + const segments = groupSonioxTokensBySpeaker(tokens).map((segment, index) => ({ + id: index, + start: segment.startMs / 1000, + end: segment.endMs / 1000, + text: segment.text.trim(), + ...(segment.speaker !== null ? { speaker: segment.speaker } : {}), + })); + + const language = tokens.find((t) => typeof t.language === "string" && t.language)?.language; + const durationMs = tokens.length ? (tokens[tokens.length - 1].end_ms ?? 0) : 0; + + return Response.json( + { + task: "transcribe", + ...(language ? { language } : {}), + duration: durationMs / 1000, + text, + segments, + ...(options.wantWords + ? { + words: tokens.map((t) => ({ + word: t.text ?? "", + start: (t.start_ms ?? 0) / 1000, + end: (t.end_ms ?? 0) / 1000, + ...(t.speaker != null ? { speaker: String(t.speaker) } : {}), + })), + } + : {}), + }, + { headers: { ...CORS_HEADERS } } + ); } /** @@ -824,7 +949,7 @@ export async function handleAudioTranscription({ } if (providerConfig.format === "soniox") { - return handleSonioxTranscription(providerConfig, file, modelId, token); + return handleSonioxTranscription(providerConfig, file, modelId, token, formData); } if (providerConfig.format === "nvidia-asr") { diff --git a/tests/unit/soniox-diarization-and-client-params.test.ts b/tests/unit/soniox-diarization-and-client-params.test.ts new file mode 100644 index 0000000000..f41b6c3f25 --- /dev/null +++ b/tests/unit/soniox-diarization-and-client-params.test.ts @@ -0,0 +1,185 @@ +// Regression: no client-supplied parameter reached Soniox. handleSonioxTranscription +// took no formData, the job body was a fixed three-key object, and the transcript was +// reduced to { text } — so `enable_speaker_diarization` was accepted with a 200 and +// silently dropped, and the per-token `speaker` attribution Soniox returns was thrown +// away. Diarization and `context` were both unreachable through the gateway. +// +// The default contract is load-bearing: a caller sending no options must still get +// exactly { text } and must still produce the same three-key job body, because that +// is what tests/unit/audio-soniox-provider.test.ts asserts with strict deep equality +// and what every existing client parses. +import test from "node:test"; +import assert from "node:assert/strict"; + +const { handleAudioTranscription } = await import("../../open-sse/handlers/audioTranscription.ts"); + +function immediateTimeout(callback, _ms, ...args) { + if (typeof callback === "function") callback(...args); + return 0; +} + +function buildFile() { + return new File([Buffer.from("abc")], "clip.wav", { type: "audio/wav" }); +} + +// A two-speaker exchange with a mid-turn token split, so grouping is actually exercised. +const TOKENS = [ + { text: "سلام", start_ms: 0, end_ms: 500, speaker: "1", language: "fa" }, + { text: " وقت بخیر", start_ms: 500, end_ms: 1200, speaker: "1", language: "fa" }, + { text: "بله", start_ms: 1300, end_ms: 1800, speaker: "2", language: "fa" }, + { text: " بفرمایید", start_ms: 1800, end_ms: 2400, speaker: "2", language: "fa" }, + { text: "ممنون", start_ms: 2500, end_ms: 3000, speaker: "1", language: "fa" }, +]; + +/** Runs the handler against a stubbed Soniox and returns the job body + parsed response. */ +async function runSoniox(fields: Record) { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.setTimeout; + let jobBody: Record = {}; + + globalThis.setTimeout = immediateTimeout; + globalThis.fetch = async (url, options: { body?: unknown } = {}) => { + const stringUrl = String(url); + if (stringUrl === "https://api.soniox.com/v1/files") return Response.json({ id: "file-1" }); + if (stringUrl === "https://api.soniox.com/v1/transcriptions") { + jobBody = JSON.parse(String(options.body || "{}")); + return Response.json({ id: "job-1" }); + } + if (stringUrl === "https://api.soniox.com/v1/transcriptions/job-1") + return Response.json({ status: "completed" }); + return Response.json({ tokens: TOKENS }); + }; + + try { + const formData = new FormData(); + formData.append("model", "soniox/stt-async-v5"); + formData.append("file", buildFile()); + for (const [key, value] of Object.entries(fields)) { + for (const item of Array.isArray(value) ? value : [value]) formData.append(key, item); + } + const response = await handleAudioTranscription({ + formData, + credentials: { apiKey: "soniox-key" }, + }); + return { jobBody, status: response.status, payload: await response.json() }; + } finally { + globalThis.fetch = originalFetch; + globalThis.setTimeout = originalSetTimeout; + } +} + +test("no options: job body and response shape are unchanged", async () => { + const { jobBody, payload } = await runSoniox({}); + assert.deepEqual(jobBody, { + model: "stt-async-v5", + file_id: "file-1", + enable_language_identification: true, + }); + assert.deepEqual(Object.keys(payload), ["text"]); + assert.equal(payload.text, "سلام وقت بخیربله بفرماییدممنون"); +}); + +test("diarization reaches Soniox and speaker attribution reaches the client", async () => { + const { jobBody, payload } = await runSoniox({ enable_speaker_diarization: "true" }); + assert.equal(jobBody.enable_speaker_diarization, true); + assert.ok(Array.isArray(payload.segments)); + assert.deepEqual( + payload.segments.map((s) => [s.speaker, s.text]), + [ + ["1", "سلام وقت بخیر"], + ["2", "بله بفرمایید"], + ["1", "ممنون"], + ] + ); + // More than one distinct label — the classic wrong "fix" applies one label to everything. + assert.equal(new Set(payload.segments.map((s) => s.speaker)).size, 2); +}); + +test("segment boundaries carry real timings, in seconds", async () => { + const { payload } = await runSoniox({ diarization: "true" }); + assert.deepEqual(payload.segments[0], { + id: 0, + start: 0, + end: 1.2, + text: "سلام وقت بخیر", + speaker: "1", + }); + assert.equal(payload.segments[1].start, 1.3); + assert.equal(payload.duration, 3); + assert.equal(payload.language, "fa"); + assert.equal(payload.text, "سلام وقت بخیربله بفرماییدممنون"); +}); + +test("diarization is accepted under all three spellings callers use", async () => { + for (const key of ["enable_speaker_diarization", "diarization", "speaker_labels"]) { + const { jobBody } = await runSoniox({ [key]: "true" }); + assert.equal(jobBody.enable_speaker_diarization, true, `spelling ${key} was dropped`); + } + // ...and only for truthy values. + const off = await runSoniox({ diarization: "false" }); + assert.ok(!("enable_speaker_diarization" in off.jobBody)); + assert.deepEqual(Object.keys(off.payload), ["text"]); +}); + +test("context reaches Soniox verbatim without changing the response contract", async () => { + const { jobBody, payload } = await runSoniox({ context: "سانیتل، بانک رفاه" }); + assert.equal(jobBody.context, "سانیتل، بانک رفاه"); + // context alone is not a shape change — the caller did not ask for segments. + assert.deepEqual(Object.keys(payload), ["text"]); +}); + +test("language becomes a Soniox language hint", async () => { + const { jobBody } = await runSoniox({ language: "fa" }); + assert.deepEqual(jobBody.language_hints, ["fa"]); + assert.equal(jobBody.enable_language_identification, true); +}); + +test("response_format=verbose_json alone yields segments without requesting diarization", async () => { + const { jobBody, payload } = await runSoniox({ response_format: "verbose_json" }); + assert.ok(!("enable_speaker_diarization" in jobBody)); + assert.equal(payload.task, "transcribe"); + assert.ok(Array.isArray(payload.segments)); +}); + +test("timestamp_granularities[]=word adds per-word timings", async () => { + const { payload } = await runSoniox({ + diarization: "true", + "timestamp_granularities[]": ["word"], + }); + assert.equal(payload.words.length, TOKENS.length); + assert.deepEqual(payload.words[0], { word: "سلام", start: 0, end: 0.5, speaker: "1" }); +}); + +test("tokens without speaker labels still produce segments, with no speaker key", async () => { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.setTimeout; + globalThis.setTimeout = immediateTimeout; + globalThis.fetch = async (url) => { + const stringUrl = String(url); + if (stringUrl === "https://api.soniox.com/v1/files") return Response.json({ id: "file-1" }); + if (stringUrl === "https://api.soniox.com/v1/transcriptions") + return Response.json({ id: "job-1" }); + if (stringUrl === "https://api.soniox.com/v1/transcriptions/job-1") + return Response.json({ status: "completed" }); + return Response.json({ + tokens: [ + { text: "one ", start_ms: 0, end_ms: 100 }, + { text: "two", start_ms: 100, end_ms: 200 }, + ], + }); + }; + try { + const formData = new FormData(); + formData.append("model", "soniox/stt-async-v5"); + formData.append("file", buildFile()); + formData.append("response_format", "verbose_json"); + const response = await handleAudioTranscription({ formData, credentials: { apiKey: "k" } }); + const payload = await response.json(); + assert.equal(payload.segments.length, 1); + assert.ok(!("speaker" in payload.segments[0])); + assert.equal(payload.text, "one two"); + } finally { + globalThis.fetch = originalFetch; + globalThis.setTimeout = originalSetTimeout; + } +});