mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
* refactor(sse): declare the multipart/gRPC frame bodies as ArrayBuffer-backed
Seven TS2769 across three files, one root cause: a bare `Uint8Array` widens to
`Uint8Array<ArrayBufferLike>`, which admits `SharedArrayBuffer` and so is not
assignable to `BodyInit`. Every one of the seven is a `fetch({ body })` call
rejecting an otherwise-valid payload.
They flow from exactly two producers:
buildMultipartBody() audioTranscription.ts -> 5 sites here + 1 in audioTranslation.ts
grpcWebFrame() windsurf.ts -> 1 site
Both allocate with `new Uint8Array(length)`, which is always ArrayBuffer-backed,
so declaring `Uint8Array<ArrayBuffer>` states what the code already guarantees.
This is a narrowing of the declaration to match reality, not a cast at the call
sites — no `as BodyInit` anywhere, and the seven call sites are untouched.
280 -> 273, zero new, on a line-number-agnostic diff of the full tsc error set.
The runtime diff is two return-type annotations; nothing executable changed.
Tests: buildMultipartBody already has four direct tests, but they assert the
payload's *contents* (boundary, filename sanitization, MIME fallback) — none
assert the backing, which is precisely what the new type guarantees and what a
plausible switch to a pooled `Buffer.concat` would break. Added
multipart-body-arraybuffer-backing.test.ts (3 tests): the buffer is a plain
ArrayBuffer, the view spans it entirely at offset 0 (no pool remainder), and
both hold for a 64 KiB payload past Node's Buffer-pool threshold. 258/258 across
the 17 affected audio/windsurf suites.
Not included: the 3 remaining TS2339 in audioTranscription.ts (`data?.data?.…`
on an untyped poll result). Different root cause — grouped by kind, not by file,
per #8484.
* chore(quality): trim the buildMultipartBody doc so audioTranscription.ts stays under the 800 cap
My doc comment on buildMultipartBody added 9 lines to a file with 8 to spare
(792 -> 801 as check:file-size counts, cap 800), so this PR was failing the gate
on growth I introduced. Condensed the comment to 4 lines; the file lands at 796.
Rebuilt on top of the /green-prs sync merge (9973bf3bf) rather than force-pushing
my own rebase over it — the release-tip sync there is @ikelvingo's work and had
already been validated against this branch.
Delta unchanged: 280 -> 273, 7 fixed, zero new. 68/68 on the audio/windsurf
suites; typecheck:core and eslint clean; check:file-size now OK.
---------
Co-authored-by: backryun <busan011@ormbiz.co.kr>
Co-authored-by: ikelvingo <im.kelvinwong@gmail.com>
797 lines
25 KiB
TypeScript
797 lines
25 KiB
TypeScript
import { CORS_HEADERS } from "../utils/cors.ts";
|
|
import { Buffer } from "node:buffer";
|
|
/**
|
|
* Audio Transcription Handler
|
|
*
|
|
* Handles POST /v1/audio/transcriptions (Whisper API format).
|
|
* Proxies multipart/form-data to upstream providers.
|
|
*
|
|
* Supported provider formats:
|
|
* - OpenAI/Groq/Qwen3: standard multipart form-data proxy
|
|
* - Deepgram: raw binary audio POST with model via query param
|
|
* - AssemblyAI: async workflow (upload → submit → poll)
|
|
* - Gladia: async workflow (upload → submit pre-recorded job → poll result_url)
|
|
* - Nvidia NIM: multipart POST, transform response to { text }
|
|
* - HuggingFace Inference: POST raw binary to /models/{model_id}
|
|
*/
|
|
|
|
import {
|
|
getTranscriptionProvider,
|
|
parseTranscriptionModel,
|
|
type AudioProvider,
|
|
} from "../config/audioRegistry.ts";
|
|
import { buildAuthHeaders } from "../config/registryUtils.ts";
|
|
import { kieExecutor } from "../executors/kie.ts";
|
|
import { vertexTranscribe } from "../executors/vertexMedia.ts";
|
|
import { errorResponse } from "../utils/error.ts";
|
|
import { handleOpenRouterTranscription } from "./openrouterTranscription.ts";
|
|
|
|
type TranscriptionCredentials = {
|
|
apiKey?: string;
|
|
accessToken?: string;
|
|
};
|
|
|
|
/**
|
|
* Return a CORS error response from an upstream fetch failure
|
|
*/
|
|
export function upstreamErrorResponse(res, errText) {
|
|
// Always return JSON so the client can parse the error reliably
|
|
let errorMessage: string;
|
|
try {
|
|
const parsed = JSON.parse(errText);
|
|
// Guard against `parsed.error` or `parsed.detail` being objects
|
|
const raw =
|
|
parsed?.err_msg ||
|
|
parsed?.error?.message ||
|
|
(typeof parsed?.error === "string" ? parsed.error : null) ||
|
|
parsed?.message ||
|
|
(typeof parsed?.detail === "string" ? parsed.detail : parsed?.detail?.message) ||
|
|
null;
|
|
errorMessage = raw ? String(raw) : errText || `Upstream error (${res.status})`;
|
|
} catch {
|
|
errorMessage = errText || `Upstream error (${res.status})`;
|
|
}
|
|
|
|
return Response.json(
|
|
{ error: { message: errorMessage, code: res.status } },
|
|
{
|
|
status: res.status,
|
|
headers: { ...CORS_HEADERS },
|
|
}
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Validate a path segment to prevent path traversal / SSRF.
|
|
*/
|
|
function isValidPathSegment(segment: string): boolean {
|
|
return !segment.includes("..") && !segment.includes("//");
|
|
}
|
|
|
|
function getUploadedFileName(file: Blob & { name?: unknown }): string {
|
|
return typeof file.name === "string" && file.name.length > 0 ? file.name : "audio.wav";
|
|
}
|
|
|
|
/**
|
|
* `body` is `Uint8Array<ArrayBuffer>`, not bare `Uint8Array`: `new Uint8Array(n)`
|
|
* is always ArrayBuffer-backed, and only that narrower form satisfies `BodyInit`
|
|
* (the bare type widens to `ArrayBufferLike`, which admits `SharedArrayBuffer`).
|
|
*/
|
|
export async function buildMultipartBody(
|
|
file: Blob & { name?: unknown },
|
|
fields: Record<string, string>,
|
|
fileFieldName = "file"
|
|
): Promise<{ body: Uint8Array<ArrayBuffer>; contentType: string }> {
|
|
const boundary = "----OmniRouteAudioBoundary" + Date.now().toString(36);
|
|
const parts: Uint8Array[] = [];
|
|
const encoder = new TextEncoder();
|
|
|
|
for (const [name, value] of Object.entries(fields)) {
|
|
parts.push(
|
|
encoder.encode(
|
|
`--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`
|
|
)
|
|
);
|
|
}
|
|
|
|
const fileName = getUploadedFileName(file)
|
|
.replace(/["]/g, "_")
|
|
.replace(/[\r\n]/g, "_");
|
|
const fileBytes = new Uint8Array(await file.arrayBuffer());
|
|
parts.push(
|
|
encoder.encode(
|
|
`--${boundary}\r\nContent-Disposition: form-data; name="${fileFieldName}"; filename="${fileName}"\r\nContent-Type: ${file.type || "application/octet-stream"}\r\n\r\n`
|
|
)
|
|
);
|
|
parts.push(fileBytes);
|
|
parts.push(encoder.encode(`\r\n--${boundary}--\r\n`));
|
|
|
|
const totalLength = parts.reduce((sum, p) => sum + p.length, 0);
|
|
const body = new Uint8Array(totalLength);
|
|
let offset = 0;
|
|
for (const part of parts) {
|
|
body.set(part, offset);
|
|
offset += part.length;
|
|
}
|
|
|
|
return { body, contentType: "multipart/form-data; boundary=" + boundary };
|
|
}
|
|
|
|
/**
|
|
* Infer a suitable Content-Type for Deepgram from the browser-provided MIME
|
|
* type and the original filename. Deepgram accepts `audio/*` and many raw
|
|
* formats, but `video/*` causes it to silently fail with "no speech detected".
|
|
*
|
|
* Strategy:
|
|
* 1. If the browser says `audio/*`, keep it as-is.
|
|
* 2. If it's `video/*` (e.g. `.mp4`), remap to the audio equivalent so
|
|
* Deepgram extracts the audio track. `.mp4` → `audio/mp4`, etc.
|
|
* 3. Fall back to `application/octet-stream` which tells Deepgram to
|
|
* auto-detect from the raw bytes (most reliable for unknown formats).
|
|
*/
|
|
function resolveAudioContentType(file: Blob & { name?: unknown }): string {
|
|
const browserType = (file.type || "").toLowerCase();
|
|
const fileName = typeof file.name === "string" ? file.name.toLowerCase() : "";
|
|
|
|
// 1) Browser already says it's audio — trust it
|
|
if (browserType.startsWith("audio/")) return browserType;
|
|
|
|
// 2) Derive from file extension (covers video/* and empty MIME)
|
|
const ext = fileName.includes(".") ? fileName.split(".").pop() : "";
|
|
const EXT_TO_MIME: Record<string, string> = {
|
|
mp3: "audio/mpeg",
|
|
mp4: "audio/mp4",
|
|
m4a: "audio/mp4",
|
|
wav: "audio/wav",
|
|
ogg: "audio/ogg",
|
|
flac: "audio/flac",
|
|
webm: "audio/webm",
|
|
aac: "audio/aac",
|
|
wma: "audio/x-ms-wma",
|
|
opus: "audio/opus",
|
|
};
|
|
if (ext && EXT_TO_MIME[ext]) return EXT_TO_MIME[ext];
|
|
|
|
// 3) Fallback — let Deepgram auto-detect from raw bytes
|
|
return "application/octet-stream";
|
|
}
|
|
|
|
/**
|
|
* Handle Deepgram transcription (raw binary audio, model via query param)
|
|
*/
|
|
async function handleDeepgramTranscription(
|
|
providerConfig,
|
|
file,
|
|
modelId,
|
|
token,
|
|
formData?: FormData
|
|
) {
|
|
const url = new URL(providerConfig.baseUrl);
|
|
url.searchParams.set("model", modelId);
|
|
url.searchParams.set("smart_format", "true");
|
|
url.searchParams.set("punctuate", "true");
|
|
|
|
// Language: if caller specified one, use it; otherwise let Deepgram auto-detect
|
|
const langParam = formData?.get("language");
|
|
if (typeof langParam === "string" && langParam.trim()) {
|
|
url.searchParams.set("language", langParam.trim());
|
|
} else {
|
|
url.searchParams.set("detect_language", "true");
|
|
}
|
|
|
|
const arrayBuffer = await file.arrayBuffer();
|
|
|
|
const res = await fetch(url.toString(), {
|
|
method: "POST",
|
|
headers: {
|
|
...buildAuthHeaders(providerConfig, token),
|
|
"Content-Type": resolveAudioContentType(file),
|
|
},
|
|
body: arrayBuffer,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
return upstreamErrorResponse(res, await res.text());
|
|
}
|
|
|
|
const data = await res.json();
|
|
// Transform Deepgram response to OpenAI Whisper format
|
|
const text = data.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? null;
|
|
|
|
// null means the audio had no recognizable speech (music, silence, etc.)
|
|
// Return it explicitly so the client can distinguish from a credentials error
|
|
return Response.json(
|
|
{ text: text ?? "", noSpeechDetected: text === null || text === "" },
|
|
{ headers: { ...CORS_HEADERS } }
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Handle AssemblyAI transcription (async: upload file → submit → poll)
|
|
*/
|
|
async function handleAssemblyAITranscription(providerConfig, file, modelId, token) {
|
|
const authHeaders = buildAuthHeaders(providerConfig, token);
|
|
|
|
// Step 1: Upload the audio file
|
|
const arrayBuffer = await file.arrayBuffer();
|
|
const uploadRes = await fetch("https://api.assemblyai.com/v2/upload", {
|
|
method: "POST",
|
|
headers: {
|
|
...authHeaders,
|
|
"Content-Type": "application/octet-stream",
|
|
},
|
|
body: arrayBuffer,
|
|
});
|
|
|
|
if (!uploadRes.ok) {
|
|
return upstreamErrorResponse(uploadRes, await uploadRes.text());
|
|
}
|
|
|
|
const { upload_url } = await uploadRes.json();
|
|
|
|
// Step 2: Submit transcription request
|
|
const submitRes = await fetch(providerConfig.baseUrl, {
|
|
method: "POST",
|
|
headers: {
|
|
...authHeaders,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
audio_url: upload_url,
|
|
speech_models: [modelId],
|
|
language_detection: true,
|
|
}),
|
|
});
|
|
|
|
if (!submitRes.ok) {
|
|
return upstreamErrorResponse(submitRes, await submitRes.text());
|
|
}
|
|
|
|
const { id: transcriptId } = await submitRes.json();
|
|
|
|
// Step 3: Poll for completion (max 120s)
|
|
const pollUrl = `${providerConfig.baseUrl}/${transcriptId}`;
|
|
const maxWait = 120_000;
|
|
const start = Date.now();
|
|
|
|
while (Date.now() - start < maxWait) {
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
|
const pollRes = await fetch(pollUrl, { headers: authHeaders });
|
|
if (!pollRes.ok) continue;
|
|
|
|
const result = await pollRes.json();
|
|
|
|
if (result.status === "completed") {
|
|
return Response.json({ text: result.text || "" }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
if (result.status === "error") {
|
|
return errorResponse(500, result.error || "AssemblyAI transcription failed");
|
|
}
|
|
}
|
|
|
|
return errorResponse(504, "AssemblyAI transcription timed out after 120s");
|
|
}
|
|
|
|
/**
|
|
* Handle Gladia transcription (async: upload file → submit pre-recorded job → poll result_url)
|
|
*/
|
|
async function handleGladiaTranscription(providerConfig, file, modelId, token) {
|
|
const authHeaders = buildAuthHeaders(providerConfig, token);
|
|
|
|
// Step 1: Upload the audio file (multipart/form-data)
|
|
const { body: uploadBody, contentType: uploadCT } = await buildMultipartBody(file, {});
|
|
const uploadRes = await fetch("https://api.gladia.io/v2/upload", {
|
|
method: "POST",
|
|
headers: { ...authHeaders, "Content-Type": uploadCT },
|
|
body: uploadBody,
|
|
});
|
|
|
|
if (!uploadRes.ok) {
|
|
return upstreamErrorResponse(uploadRes, await uploadRes.text());
|
|
}
|
|
|
|
const { audio_url } = await uploadRes.json();
|
|
|
|
// Step 2: Submit the pre-recorded transcription job
|
|
const submitRes = await fetch(providerConfig.baseUrl, {
|
|
method: "POST",
|
|
headers: { ...authHeaders, "Content-Type": "application/json" },
|
|
body: JSON.stringify({ audio_url, model: modelId }),
|
|
});
|
|
|
|
if (!submitRes.ok) {
|
|
return upstreamErrorResponse(submitRes, await submitRes.text());
|
|
}
|
|
|
|
const { result_url: resultUrl } = await submitRes.json();
|
|
if (!resultUrl) {
|
|
return errorResponse(502, "Gladia did not return a result_url");
|
|
}
|
|
|
|
// Step 3: Poll for completion (max 120s)
|
|
const maxWait = 120_000;
|
|
const start = Date.now();
|
|
|
|
while (Date.now() - start < maxWait) {
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
|
const pollRes = await fetch(resultUrl, { headers: authHeaders });
|
|
if (!pollRes.ok) continue;
|
|
|
|
const result = await pollRes.json();
|
|
|
|
if (result.status === "done") {
|
|
const text = result.result?.transcription?.full_transcript || "";
|
|
return Response.json({ text }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
if (result.status === "error") {
|
|
return errorResponse(500, result.error_code || result.error || "Gladia transcription failed");
|
|
}
|
|
}
|
|
|
|
return errorResponse(504, "Gladia transcription timed out after 120s");
|
|
}
|
|
|
|
/**
|
|
* Handle Nvidia NIM transcription
|
|
* Multipart POST, transform response to { text }
|
|
*/
|
|
async function handleNvidiaTranscription(providerConfig, file, modelId, token) {
|
|
const { body, contentType } = await buildMultipartBody(file, { model: modelId });
|
|
|
|
const res = await fetch(providerConfig.baseUrl, {
|
|
method: "POST",
|
|
headers: { ...buildAuthHeaders(providerConfig, token), "Content-Type": contentType },
|
|
body,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
return upstreamErrorResponse(res, await res.text());
|
|
}
|
|
|
|
const data = await res.json();
|
|
// Normalize to { text } — Nvidia may return { text } directly or nested
|
|
const text = data.text || data.transcript || "";
|
|
|
|
return Response.json({ text }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
/**
|
|
* Handle HuggingFace Inference transcription
|
|
* POST raw binary audio to {baseUrl}/{model_id}, returns { text }
|
|
*/
|
|
async function handleHuggingFaceTranscription(providerConfig, file, modelId, token) {
|
|
if (!isValidPathSegment(modelId)) {
|
|
return errorResponse(400, "Invalid model ID");
|
|
}
|
|
const url = `${providerConfig.baseUrl}/${modelId}`;
|
|
const arrayBuffer = await file.arrayBuffer();
|
|
|
|
const res = await fetch(url, {
|
|
method: "POST",
|
|
headers: {
|
|
...buildAuthHeaders(providerConfig, token),
|
|
"Content-Type": resolveAudioContentType(file),
|
|
},
|
|
body: arrayBuffer,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
return upstreamErrorResponse(res, await res.text());
|
|
}
|
|
|
|
const data = await res.json();
|
|
// HuggingFace returns { text } directly
|
|
const text = data.text || "";
|
|
|
|
return Response.json({ text }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
/**
|
|
* Handle Kie.ai transcription
|
|
*/
|
|
async function handleKieAudioTranscription(providerConfig, file, modelId, token) {
|
|
const baseUrl = providerConfig.baseUrl.replace(/\/$/, "");
|
|
const fileBuffer = await file.arrayBuffer();
|
|
const fileBase64 = Buffer.from(fileBuffer).toString("base64");
|
|
let data;
|
|
try {
|
|
data = await kieExecutor.createTask({
|
|
baseUrl,
|
|
token,
|
|
payload: {
|
|
model: modelId,
|
|
input: {
|
|
file_name: getUploadedFileName(file),
|
|
file_base64: fileBase64,
|
|
},
|
|
},
|
|
});
|
|
} catch (err: unknown) {
|
|
const status =
|
|
typeof err === "object" && err !== null && "status" in err
|
|
? Number((err as { status?: unknown }).status) || 502
|
|
: 502;
|
|
return Response.json(
|
|
{
|
|
error: {
|
|
message: err instanceof Error ? err.message : "Kie transcription createTask failed",
|
|
code: status,
|
|
},
|
|
},
|
|
{
|
|
status,
|
|
headers: { ...CORS_HEADERS },
|
|
}
|
|
);
|
|
}
|
|
const taskId = data?.data?.taskId || data?.taskId;
|
|
|
|
if (taskId) {
|
|
return pollKieTranscriptionResult(baseUrl, modelId, taskId, token);
|
|
}
|
|
|
|
return Response.json(
|
|
{ text: data?.data?.text || data?.text || "" },
|
|
{ headers: { ...CORS_HEADERS } }
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Internal polling for Kie.ai async transcription tasks
|
|
*/
|
|
async function pollKieTranscriptionResult(baseUrl, modelId, taskId, token) {
|
|
void modelId;
|
|
const statusUrl = kieExecutor.getTaskStatusUrl(baseUrl);
|
|
try {
|
|
const { data, state } = await kieExecutor.pollTask({
|
|
statusUrl,
|
|
taskId: String(taskId),
|
|
token,
|
|
timeoutMs: 120000,
|
|
pollIntervalMs: 2000,
|
|
});
|
|
|
|
if (state === "success") {
|
|
const text =
|
|
data?.data?.response?.text ||
|
|
data?.data?.resultText ||
|
|
data?.data?.text ||
|
|
data?.text ||
|
|
"";
|
|
return Response.json({ text }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
} catch (err: unknown) {
|
|
const status =
|
|
typeof err === "object" && err !== null && "status" in err
|
|
? Number((err as { status?: unknown }).status) || 504
|
|
: 504;
|
|
return errorResponse(
|
|
status,
|
|
err instanceof Error ? err.message : "Kie transcription generation timed out or failed"
|
|
);
|
|
}
|
|
|
|
return errorResponse(504, "Kie transcription generation timed out or failed");
|
|
}
|
|
|
|
/**
|
|
* Handle Rev AI transcription (async: submit job with media upload → poll → fetch transcript)
|
|
*
|
|
* Rev AI accepts the audio file directly in the job-submission multipart body
|
|
* (field "media"), avoiding AssemblyAI's separate upload step. Once the job
|
|
* reaches a terminal state we fetch the plain-text transcript.
|
|
*/
|
|
async function handleRevAiTranscription(providerConfig, file, modelId, token) {
|
|
const authHeaders = buildAuthHeaders(providerConfig, token);
|
|
const baseUrl = providerConfig.baseUrl.replace(/\/$/, "");
|
|
|
|
// Step 1: submit the job — multipart body with "media" (file) + "options" (JSON)
|
|
const options = JSON.stringify({ transcriber: modelId });
|
|
const { body, contentType } = await buildMultipartBody(file, { options }, "media");
|
|
|
|
const submitRes = await fetch(`${baseUrl}/jobs`, {
|
|
method: "POST",
|
|
headers: { ...authHeaders, "Content-Type": contentType },
|
|
body,
|
|
});
|
|
|
|
if (!submitRes.ok) {
|
|
return upstreamErrorResponse(submitRes, await submitRes.text());
|
|
}
|
|
|
|
const { id: jobId } = await submitRes.json();
|
|
|
|
// Step 2: poll for completion (max 120s)
|
|
const jobUrl = `${baseUrl}/jobs/${jobId}`;
|
|
const maxWait = 120_000;
|
|
const start = Date.now();
|
|
|
|
while (Date.now() - start < maxWait) {
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
|
const pollRes = await fetch(jobUrl, { headers: authHeaders });
|
|
if (!pollRes.ok) continue;
|
|
|
|
const result = await pollRes.json();
|
|
|
|
if (result.status === "transcribed") {
|
|
const transcriptRes = await fetch(`${jobUrl}/transcript`, {
|
|
headers: { ...authHeaders, Accept: "text/plain" },
|
|
});
|
|
if (!transcriptRes.ok) {
|
|
return upstreamErrorResponse(transcriptRes, await transcriptRes.text());
|
|
}
|
|
const text = await transcriptRes.text();
|
|
return Response.json({ text: text || "" }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
if (result.status === "failed") {
|
|
return errorResponse(500, result.failure_detail || "Rev AI transcription failed");
|
|
}
|
|
}
|
|
|
|
return errorResponse(504, "Rev AI transcription timed out after 120s");
|
|
}
|
|
|
|
/**
|
|
* Speechmatics operating point (accuracy tier). Catalog model ids are the
|
|
* real Speechmatics `operating_point` values ("standard", "enhanced",
|
|
* "melia-1"), so this passes straight through — kept as a named seam in
|
|
* case a future catalog id needs remapping.
|
|
*/
|
|
function speechmaticsOperatingPoint(modelId: string): string {
|
|
return modelId;
|
|
}
|
|
|
|
/**
|
|
* Fetch and return the finished Speechmatics transcript once a job reaches
|
|
* the "done" state.
|
|
*/
|
|
async function fetchSpeechmaticsTranscript(jobUrl, authHeaders) {
|
|
const transcriptRes = await fetch(`${jobUrl}/transcript?format=txt`, {
|
|
headers: { ...authHeaders, Accept: "text/plain" },
|
|
});
|
|
if (!transcriptRes.ok) {
|
|
return upstreamErrorResponse(transcriptRes, await transcriptRes.text());
|
|
}
|
|
const text = await transcriptRes.text();
|
|
return Response.json({ text: text || "" }, { headers: { ...CORS_HEADERS } });
|
|
}
|
|
|
|
function speechmaticsJobErrorMessage(result): string {
|
|
const errors = result?.job?.errors;
|
|
const first = Array.isArray(errors) ? errors[0] : null;
|
|
return first?.message || "Speechmatics transcription failed";
|
|
}
|
|
|
|
/**
|
|
* Poll a submitted Speechmatics job until it reaches a terminal state
|
|
* (max 120s), then fetch its transcript.
|
|
*/
|
|
async function pollSpeechmaticsJob(jobUrl, authHeaders) {
|
|
const maxWait = 120_000;
|
|
const start = Date.now();
|
|
|
|
while (Date.now() - start < maxWait) {
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
|
const pollRes = await fetch(jobUrl, { headers: authHeaders });
|
|
if (!pollRes.ok) continue;
|
|
|
|
const result = await pollRes.json();
|
|
const status = result?.job?.status;
|
|
|
|
if (status === "done") {
|
|
return fetchSpeechmaticsTranscript(jobUrl, authHeaders);
|
|
}
|
|
|
|
if (status === "rejected") {
|
|
return errorResponse(500, speechmaticsJobErrorMessage(result));
|
|
}
|
|
}
|
|
|
|
return errorResponse(504, "Speechmatics transcription timed out after 120s");
|
|
}
|
|
|
|
/**
|
|
* Handle Speechmatics transcription (async batch: submit multipart job → poll → fetch transcript)
|
|
*
|
|
* Speechmatics batch mode accepts the audio file directly in the job-submission
|
|
* multipart body (field "data_file") alongside a JSON "config" field describing
|
|
* the requested transcription options. Streaming (real-time WebSocket) mode is
|
|
* out of scope for v1 — this handler only implements batch (REST) transcription.
|
|
*/
|
|
async function handleSpeechmaticsTranscription(providerConfig, file, modelId, token) {
|
|
const authHeaders = buildAuthHeaders(providerConfig, token);
|
|
const baseUrl = providerConfig.baseUrl.replace(/\/$/, "");
|
|
|
|
// Step 1: submit the job — multipart body with "data_file" (audio) + "config" (JSON)
|
|
const config = JSON.stringify({
|
|
type: "transcription",
|
|
transcription_config: { operating_point: speechmaticsOperatingPoint(modelId) },
|
|
});
|
|
const { body, contentType } = await buildMultipartBody(file, { config }, "data_file");
|
|
|
|
const submitRes = await fetch(baseUrl, {
|
|
method: "POST",
|
|
headers: { ...authHeaders, "Content-Type": contentType },
|
|
body,
|
|
});
|
|
|
|
if (!submitRes.ok) {
|
|
return upstreamErrorResponse(submitRes, await submitRes.text());
|
|
}
|
|
|
|
const { id: jobId } = await submitRes.json();
|
|
if (!jobId) {
|
|
return errorResponse(502, "Speechmatics did not return a job id");
|
|
}
|
|
|
|
// Step 2: poll for completion (max 120s)
|
|
return pollSpeechmaticsJob(`${baseUrl}/${jobId}`, authHeaders);
|
|
}
|
|
|
|
/**
|
|
* Handle audio transcription request
|
|
*
|
|
* @param {Object} options
|
|
* @param {FormData} options.formData - Multipart form data with file + model
|
|
* @param {Object} options.credentials - Provider credentials { apiKey }
|
|
* @returns {Response}
|
|
*/
|
|
export async function handleAudioTranscription({
|
|
formData,
|
|
credentials,
|
|
resolvedProvider = null,
|
|
resolvedModel = null,
|
|
}: {
|
|
formData: FormData;
|
|
credentials?: TranscriptionCredentials | null;
|
|
resolvedProvider?: AudioProvider | null;
|
|
resolvedModel?: string | null;
|
|
}): Promise<Response> {
|
|
const model = formData.get("model");
|
|
if (typeof model !== "string" || !model) {
|
|
return errorResponse(400, "model is required");
|
|
}
|
|
|
|
const fileEntry = formData.get("file");
|
|
if (!(fileEntry instanceof Blob)) {
|
|
return errorResponse(400, "file is required");
|
|
}
|
|
const file = fileEntry as Blob & { name?: unknown };
|
|
|
|
// Use pre-resolved provider/model from route handler if available (supports dynamic provider_nodes).
|
|
let providerConfig = resolvedProvider;
|
|
let modelId = resolvedModel;
|
|
if (!providerConfig) {
|
|
const parsed = parseTranscriptionModel(model);
|
|
providerConfig = parsed.provider ? getTranscriptionProvider(parsed.provider) : null;
|
|
modelId = parsed.model;
|
|
}
|
|
|
|
if (!providerConfig) {
|
|
return errorResponse(
|
|
400,
|
|
`No transcription provider found for model "${model}". Available: openai, openrouter, groq, deepgram, assemblyai, nvidia, huggingface, qwen, gladia, rev-ai, speechmatics`
|
|
);
|
|
}
|
|
|
|
// Skip credential check for local providers (authType: "none")
|
|
const token =
|
|
providerConfig.authType === "none" ? null : credentials?.apiKey || credentials?.accessToken;
|
|
if (providerConfig.authType !== "none" && !token) {
|
|
return errorResponse(401, `No credentials for transcription provider: ${providerConfig.id}`);
|
|
}
|
|
|
|
// Route to provider-specific handler
|
|
if (providerConfig.format === "vertex-gemini") {
|
|
try {
|
|
const buffer = Buffer.from(await file.arrayBuffer());
|
|
const uploadedType =
|
|
typeof (file as { type?: unknown }).type === "string" && (file as { type?: string }).type
|
|
? (file as { type: string }).type
|
|
: "audio/wav";
|
|
const languageValue = formData.get("language");
|
|
const promptValue = formData.get("prompt");
|
|
const text = await vertexTranscribe(credentials ?? {}, {
|
|
model: modelId as string,
|
|
audioBase64: buffer.toString("base64"),
|
|
mimeType: uploadedType,
|
|
prompt: typeof promptValue === "string" ? promptValue : undefined,
|
|
language: typeof languageValue === "string" ? languageValue : undefined,
|
|
});
|
|
return Response.json({ text }, { headers: { ...CORS_HEADERS } });
|
|
} catch (err) {
|
|
const error = err as { message?: string; status?: number };
|
|
return errorResponse(
|
|
typeof error?.status === "number" ? error.status : 500,
|
|
`Vertex transcription failed: ${error?.message || "unknown error"}`
|
|
);
|
|
}
|
|
}
|
|
|
|
if (providerConfig.format === "deepgram") {
|
|
return handleDeepgramTranscription(providerConfig, file, modelId, token, formData);
|
|
}
|
|
|
|
if (providerConfig.format === "assemblyai") {
|
|
return handleAssemblyAITranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "gladia") {
|
|
return handleGladiaTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "nvidia-asr") {
|
|
return handleNvidiaTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "huggingface-asr") {
|
|
return handleHuggingFaceTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "kie-audio") {
|
|
return handleKieAudioTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "rev-ai") {
|
|
return handleRevAiTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "speechmatics") {
|
|
return handleSpeechmaticsTranscription(providerConfig, file, modelId, token);
|
|
}
|
|
|
|
if (providerConfig.format === "openrouter-stt") {
|
|
return handleOpenRouterTranscription(providerConfig, file, modelId, token, formData);
|
|
}
|
|
|
|
// Default: OpenAI/Groq/Qwen3-compatible multipart proxy
|
|
const extraFields: Record<string, string> = {};
|
|
for (const key of [
|
|
"language",
|
|
"prompt",
|
|
"response_format",
|
|
"temperature",
|
|
"timestamp_granularities[]",
|
|
]) {
|
|
const val = formData.get(key);
|
|
if (val !== null && val !== undefined) {
|
|
extraFields[key] = String(val);
|
|
}
|
|
}
|
|
|
|
const { body: multipartBody, contentType: multipartCT } = await buildMultipartBody(file, {
|
|
model: modelId,
|
|
...extraFields,
|
|
});
|
|
|
|
try {
|
|
const res = await fetch(providerConfig.baseUrl, {
|
|
method: "POST",
|
|
headers: { ...buildAuthHeaders(providerConfig, token), "Content-Type": multipartCT },
|
|
body: multipartBody,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
return upstreamErrorResponse(res, await res.text());
|
|
}
|
|
|
|
const data = await res.text();
|
|
const respContentType = res.headers.get("content-type") || "application/json";
|
|
|
|
return new Response(data, {
|
|
status: 200,
|
|
headers: { "Content-Type": respContentType },
|
|
});
|
|
} catch (err) {
|
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
return errorResponse(500, `Transcription request failed: ${error.message}`);
|
|
}
|
|
}
|