feat(audio): expand Fish Audio S2.1 and voice cloning (#13090)

* feat(audio): expand Fish Audio S2.1 and voice cloning

* fix(audio): type Node streaming request init

* test(audio): align Fish Audio CI expectations

---------

Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
Bl0ck
2026-09-18 18:23:31 +03:00
committed by GitHub
parent 45fa62a18e
commit 62d14a7fc7
10 changed files with 546 additions and 37 deletions

View File

@@ -0,0 +1 @@
- feat(providers): update Fish Audio for S2.1 Pro Free, validated advanced TTS controls, and provider-scoped persistent voice-clone management.

View File

@@ -463,9 +463,14 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
authHeader: "bearer",
format: "fishaudio",
models: [
{ id: "s2.1-pro-free", name: "Fish Speech S2.1 Pro Free" },
{ id: "s2.1-pro", name: "Fish Speech S2.1 Pro" },
{ id: "s2-pro", name: "Fish Speech S2 Pro" },
{ id: "s1", name: "Fish Speech S1" },
{ id: "speech-1.6", name: "Fish Speech 1.6" },
{ id: "speech-1.5", name: "Fish Speech 1.5" },
// Legacy ids kept for existing clients even though Fish no longer lists them
// in the current public model enum.
{ id: "speech-1.6", name: "Fish Speech 1.6 (legacy)" },
{ id: "speech-1.5", name: "Fish Speech 1.5 (legacy)" },
],
},

View File

@@ -0,0 +1,215 @@
import { errorResponse } from "../utils/error.ts";
import { audioStreamResponse, upstreamErrorResponse } from "../utils/audioResponse.ts";
const FISH_AUDIO_FORMATS = new Set(["wav", "pcm", "mp3", "opus"]);
const FISH_AUDIO_LATENCY = new Set(["low", "normal", "balanced"]);
const FISH_AUDIO_SAMPLE_RATES = new Set([8000, 16000, 24000, 32000, 44100, 48000]);
const FISH_AUDIO_MP3_BITRATES = new Set([64, 128, 192]);
const FISH_AUDIO_OPUS_BITRATES = new Set([-1000, 24000, 32000, 48000, 64000]);
type JsonRecord = Record<string, unknown>;
type FishAudioPayloadResult =
| { payload: JsonRecord; error?: never }
| { payload?: never; error: string };
function isJsonObject(value: unknown): value is JsonRecord {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function normalizeResponseFormat(value: unknown): string {
if (typeof value !== "string" || !value) return "mp3";
const lower = value.toLowerCase();
return lower === "ogg" ? "opus" : lower;
}
function fishAudioOptions(body: JsonRecord): JsonRecord {
const providerOptions = isJsonObject(body.provider_options) ? body.provider_options : {};
return isJsonObject(providerOptions.fishaudio) ? providerOptions.fishaudio : {};
}
function numberOption(
value: unknown,
name: string,
options: { min?: number; max?: number; integer?: boolean } = {}
): number | undefined {
if (value === undefined) return undefined;
if (typeof value !== "number" || !Number.isFinite(value)) {
throw new Error(`${name} must be a finite number`);
}
if (options.integer && !Number.isInteger(value)) {
throw new Error(`${name} must be an integer`);
}
if (options.min !== undefined && value < options.min) {
throw new Error(`${name} must be >= ${options.min}`);
}
if (options.max !== undefined && value > options.max) {
throw new Error(`${name} must be <= ${options.max}`);
}
return value;
}
function booleanOption(value: unknown, name: string): boolean | undefined {
if (value === undefined) return undefined;
if (typeof value !== "boolean") throw new Error(`${name} must be a boolean`);
return value;
}
function enumOption<T extends string | number>(
value: unknown,
name: string,
allowed: Set<T>
): T | undefined {
if (value === undefined) return undefined;
if (!allowed.has(value as T)) {
throw new Error(`${name} must be one of: ${Array.from(allowed).join(", ")}`);
}
return value as T;
}
function setReferenceId(payload: JsonRecord, referenceId: unknown): void {
if (referenceId === undefined) return;
if (typeof referenceId === "string" && referenceId.trim()) {
payload.reference_id = referenceId.trim();
return;
}
if (
Array.isArray(referenceId) &&
referenceId.length > 0 &&
referenceId.every((item) => typeof item === "string" && item.trim().length > 0)
) {
payload.reference_id = referenceId.map((item) => item.trim());
return;
}
throw new Error("reference_id must be a non-empty string or array of non-empty strings");
}
function applyGenerationOptions(payload: JsonRecord, options: JsonRecord): void {
const values: Array<[string, unknown]> = [
["temperature", numberOption(options.temperature, "temperature", { min: 0, max: 1 })],
["top_p", numberOption(options.top_p, "top_p", { min: 0, max: 1 })],
[
"chunk_length",
numberOption(options.chunk_length, "chunk_length", { min: 100, max: 300, integer: true }),
],
["normalize", booleanOption(options.normalize, "normalize")],
["sample_rate", enumOption(options.sample_rate, "sample_rate", FISH_AUDIO_SAMPLE_RATES)],
["mp3_bitrate", enumOption(options.mp3_bitrate, "mp3_bitrate", FISH_AUDIO_MP3_BITRATES)],
["opus_bitrate", enumOption(options.opus_bitrate, "opus_bitrate", FISH_AUDIO_OPUS_BITRATES)],
["latency", enumOption(options.latency, "latency", FISH_AUDIO_LATENCY)],
[
"max_new_tokens",
numberOption(options.max_new_tokens, "max_new_tokens", { min: 1, integer: true }),
],
[
"repetition_penalty",
numberOption(options.repetition_penalty, "repetition_penalty", { min: 0 }),
],
[
"min_chunk_length",
numberOption(options.min_chunk_length, "min_chunk_length", { min: 0, max: 100, integer: true }),
],
[
"condition_on_previous_chunks",
booleanOption(options.condition_on_previous_chunks, "condition_on_previous_chunks"),
],
[
"early_stop_threshold",
numberOption(options.early_stop_threshold, "early_stop_threshold", { min: 0, max: 1 }),
],
];
for (const [key, value] of values) {
if (value !== undefined) payload[key] = value;
}
if (options.features !== undefined) {
if (
!Array.isArray(options.features) ||
options.features.some((feature) => typeof feature !== "string" || !feature.trim())
) {
throw new Error("features must be an array of non-empty strings");
}
payload.features = options.features.map((feature) => feature.trim());
}
}
function applyProsody(payload: JsonRecord, body: JsonRecord, options: JsonRecord): void {
const rawProsody = options.prosody;
if (rawProsody !== undefined && rawProsody !== null && !isJsonObject(rawProsody)) {
throw new Error("prosody must be an object");
}
const prosody = isJsonObject(rawProsody) ? rawProsody : {};
const speed = numberOption(body.speed ?? prosody.speed, "prosody.speed", { min: 0.5, max: 2 });
const volume = numberOption(prosody.volume, "prosody.volume", { min: -20, max: 20 });
const normalizeLoudness = booleanOption(
prosody.normalize_loudness,
"prosody.normalize_loudness"
);
if (speed !== undefined || volume !== undefined || normalizeLoudness !== undefined) {
payload.prosody = {
...(speed !== undefined ? { speed } : {}),
...(volume !== undefined ? { volume } : {}),
...(normalizeLoudness !== undefined ? { normalize_loudness: normalizeLoudness } : {}),
};
}
}
/**
* Build Fish Audio's JSON TTS payload while keeping provider-specific controls
* namespaced under `provider_options.fishaudio` in the OpenAI-compatible request.
* Inline reference audio is intentionally not accepted here: Fish requires
* MessagePack for that path. Use a persistent /model clone and pass its id.
*/
export function buildFishAudioSpeechPayload(body: JsonRecord): FishAudioPayloadResult {
try {
const options = fishAudioOptions(body);
if (options.references !== undefined) {
throw new Error(
"inline references require Fish Audio MessagePack; create a persistent voice via /v1/providers/fishaudio/voices and pass its id as voice/reference_id"
);
}
const format = enumOption(
normalizeResponseFormat(body.response_format),
"response_format",
FISH_AUDIO_FORMATS
);
const payload: JsonRecord = {
text: body.input,
format: format || "mp3",
};
setReferenceId(payload, options.reference_id ?? body.voice);
applyGenerationOptions(payload, options);
applyProsody(payload, body, options);
return { payload };
} catch (error) {
return { error: error instanceof Error ? error.message : "Invalid Fish Audio provider options" };
}
}
/** Fish Audio TTS adapter for /v1/audio/speech. */
export async function handleFishAudioSpeech(
providerConfig: { baseUrl: string },
body: JsonRecord,
modelId: string,
token: string
): Promise<Response> {
const built = buildFishAudioSpeechPayload(body);
if (built.error) return errorResponse(400, `Fish Audio: ${built.error}`);
const res = await fetch(providerConfig.baseUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
model: modelId,
},
body: JSON.stringify(built.payload),
});
if (!res.ok) return upstreamErrorResponse(res, await res.text());
return audioStreamResponse(res);
}

View File

@@ -24,6 +24,7 @@ import { vertexGenerateSpeech } from "../executors/vertexMedia.ts";
import { handleGeminiTtsSpeech } from "../executors/geminiTts.ts";
import { handleAwsPollySpeech } from "../executors/awsPollyTts.ts";
import { GttsUpstreamError, normalizeGttsLang, synthesizeGtts } from "../executors/gtts.ts";
import { handleFishAudioSpeech } from "../executors/fishAudioTts.ts";
import { errorResponse } from "../utils/error.ts";
import { resolveElevenLabsVoiceId } from "./elevenLabsVoiceMap.ts";
import { audioStreamResponse, upstreamErrorResponse } from "../utils/audioResponse.ts";
@@ -452,35 +453,6 @@ async function handleCartesiaSpeech(providerConfig, body, modelId, token) {
return audioStreamResponse(res);
}
/**
* Handle Fish Audio TTS
* POST { text, format, reference_id, prosody } → binary audio bytes
* Auth: Authorization: Bearer <api-key>, model as an HTTP header
* Docs: https://docs.fish.audio/api-reference/endpoint/openapi-v1/text-to-speech
*/
async function handleFishAudioSpeech(providerConfig, body, modelId, token) {
const res = await fetch(providerConfig.baseUrl, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
model: modelId,
},
body: JSON.stringify({
text: body.input,
format: body.response_format || "mp3",
...(body.voice ? { reference_id: body.voice } : {}),
...(body.speed ? { prosody: { speed: body.speed } } : {}),
}),
});
if (!res.ok) {
return upstreamErrorResponse(res, await res.text());
}
return audioStreamResponse(res);
}
/**
* Handle PlayHT TTS
* POST { text, voice, voice_engine, output_format } → audio stream

View File

@@ -0,0 +1,101 @@
import {
clearRecoveredProviderState,
getProviderCredentialsWithQuotaPreflight,
} from "@/sse/services/auth";
import {
isAllRateLimitedCredentials,
rateLimitedProviderResponse,
} from "@/app/api/v1/_shared/rateLimit";
import { buildErrorBody, sanitizeErrorMessage } from "@omniroute/open-sse/utils/error.ts";
import { CORS_HEADERS, handleCorsOptions } from "@/shared/utils/cors";
const FISH_AUDIO_API_BASE = "https://api.fish.audio";
const ALLOWED_RESPONSE_HEADERS = ["content-type", "content-disposition", "retry-after", "x-request-id"] as const;
type FishAudioCredentials = {
apiKey?: string | null;
accessToken?: string | null;
allExpired?: boolean;
};
type FishAudioRequestInit = Omit<RequestInit, "headers"> & {
/** Required by Node fetch when forwarding a streaming Request body. */
duplex?: "half";
};
export function fishAudioOptionsResponse(): Response {
return handleCorsOptions();
}
export function isSafeFishAudioVoiceId(value: string): boolean {
return /^[A-Za-z0-9_-]+$/.test(value);
}
export function isFishAudioVoiceProvider(value: string): boolean {
return value === "fishaudio";
}
function proxyResponseHeaders(upstream: Response): Headers {
const headers = new Headers(CORS_HEADERS);
for (const name of ALLOWED_RESPONSE_HEADERS) {
const value = upstream.headers.get(name);
if (value) headers.set(name, value);
}
return headers;
}
export async function proxyFishAudioRequest(
request: Request,
pathname: string,
init: FishAudioRequestInit = {}
): Promise<Response> {
const credentials = (await getProviderCredentialsWithQuotaPreflight(
"fishaudio"
)) as FishAudioCredentials | null;
if (credentials && isAllRateLimitedCredentials(credentials)) {
return rateLimitedProviderResponse("fishaudio", credentials);
}
const apiKey = credentials?.apiKey || credentials?.accessToken;
if (!apiKey || credentials?.allExpired) {
return new Response(JSON.stringify(buildErrorBody(401, "No credentials for provider: fishaudio")), {
status: 401,
headers: { ...CORS_HEADERS, "Content-Type": "application/json" },
});
}
const incomingUrl = new URL(request.url);
const upstreamUrl = new URL(`${FISH_AUDIO_API_BASE}${pathname}`);
upstreamUrl.search = incomingUrl.search;
const headers = new Headers();
headers.set("Authorization", `Bearer ${apiKey}`);
const contentType = request.headers.get("content-type");
if (contentType) headers.set("content-type", contentType);
const accept = request.headers.get("accept");
if (accept) headers.set("accept", accept);
try {
const upstream = await fetch(upstreamUrl, { ...init, headers });
if (upstream.ok) {
await clearRecoveredProviderState(credentials as Record<string, unknown>);
}
return new Response(upstream.body, {
status: upstream.status,
statusText: upstream.statusText,
headers: proxyResponseHeaders(upstream),
});
} catch (error) {
return new Response(
JSON.stringify(
buildErrorBody(
502,
sanitizeErrorMessage(error instanceof Error ? error.message : "Fish Audio request failed")
)
),
{
status: 502,
headers: { ...CORS_HEADERS, "Content-Type": "application/json" },
}
);
}
}

View File

@@ -0,0 +1,65 @@
import {
fishAudioOptionsResponse,
isFishAudioVoiceProvider,
isSafeFishAudioVoiceId,
proxyFishAudioRequest,
} from "@/app/api/v1/_shared/fishAudioProxy";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { enforceClientApiRouteAuth } from "@/shared/utils/clientApiRouteAuth";
export async function OPTIONS() {
return fishAudioOptionsResponse();
}
async function resolveVoiceRequest(
request: Request,
params: Promise<{ provider: string; voiceId: string }>
): Promise<{ pathname: string } | { rejection: Response }> {
const { provider, voiceId } = await params;
if (!isFishAudioVoiceProvider(provider)) {
return {
rejection: errorResponse(
HTTP_STATUS.BAD_REQUEST,
`Voice-model management is not supported for provider: ${provider}`
),
};
}
if (!isSafeFishAudioVoiceId(voiceId)) {
return { rejection: errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid Fish Audio voice ID") };
}
const authRejection = await enforceClientApiRouteAuth(request);
if (authRejection) return { rejection: authRejection };
return { pathname: `/model/${voiceId}` };
}
export async function GET(
request: Request,
{ params }: { params: Promise<{ provider: string; voiceId: string }> }
) {
const resolved = await resolveVoiceRequest(request, params);
if ("rejection" in resolved) return resolved.rejection;
return proxyFishAudioRequest(request, resolved.pathname, { method: "GET" });
}
export async function PATCH(
request: Request,
{ params }: { params: Promise<{ provider: string; voiceId: string }> }
) {
const resolved = await resolveVoiceRequest(request, params);
if ("rejection" in resolved) return resolved.rejection;
return proxyFishAudioRequest(request, resolved.pathname, {
method: "PATCH",
body: request.body,
duplex: "half",
});
}
export async function DELETE(
request: Request,
{ params }: { params: Promise<{ provider: string; voiceId: string }> }
) {
const resolved = await resolveVoiceRequest(request, params);
if ("rejection" in resolved) return resolved.rejection;
return proxyFishAudioRequest(request, resolved.pathname, { method: "DELETE" });
}

View File

@@ -0,0 +1,55 @@
import {
fishAudioOptionsResponse,
isFishAudioVoiceProvider,
proxyFishAudioRequest,
} from "@/app/api/v1/_shared/fishAudioProxy";
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { enforceClientApiRouteAuth } from "@/shared/utils/clientApiRouteAuth";
export async function OPTIONS() {
return fishAudioOptionsResponse();
}
async function validateProviderAndAuth(request: Request, rawProvider: string): Promise<Response | null> {
if (!isFishAudioVoiceProvider(rawProvider)) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
`Voice-model management is not supported for provider: ${rawProvider}`
);
}
return enforceClientApiRouteAuth(request);
}
/** GET /v1/providers/fishaudio/voices — proxy Fish Audio voice-model listing. */
export async function GET(request: Request, { params }: { params: Promise<{ provider: string }> }) {
const { provider } = await params;
const rejection = await validateProviderAndAuth(request, provider);
if (rejection) return rejection;
return proxyFishAudioRequest(request, "/model", { method: "GET" });
}
/**
* POST /v1/providers/fishaudio/voices — create a persistent Fish Audio clone.
* Send Fish's native multipart/form-data fields (`type=tts`, `title`,
* `train_mode=fast`, one or more `voices` files, optional `texts`, etc.).
*/
export async function POST(request: Request, { params }: { params: Promise<{ provider: string }> }) {
const { provider } = await params;
const rejection = await validateProviderAndAuth(request, provider);
if (rejection) return rejection;
const contentType = request.headers.get("content-type") || "";
if (!contentType.toLowerCase().startsWith("multipart/form-data")) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
"Fish Audio voice creation requires multipart/form-data"
);
}
return proxyFishAudioRequest(request, "/model", {
method: "POST",
body: request.body,
duplex: "half",
});
}

View File

@@ -3,7 +3,7 @@ import assert from "node:assert/strict";
const { handleAudioSpeech } = await import("../../open-sse/handlers/audioSpeech.ts");
test("handleAudioSpeech maps Fish Audio headers and body, and passes audio through", async () => {
test("handleAudioSpeech maps Fish S2.1 free model and validated provider options", async () => {
const originalFetch = globalThis.fetch;
let captured;
@@ -21,22 +21,53 @@ test("handleAudioSpeech maps Fish Audio headers and body, and passes audio throu
try {
const response = await handleAudioSpeech({
body: {
model: "fishaudio/s1",
input: "hi",
model: "fishaudio/s2.1-pro-free",
input: "Привіт",
voice: "ref-123",
response_format: "mp3",
speed: 1.2,
provider_options: {
fishaudio: {
temperature: 0.8,
top_p: 0.6,
chunk_length: 240,
normalize: true,
sample_rate: 44100,
mp3_bitrate: 192,
latency: "normal",
max_new_tokens: 1024,
repetition_penalty: 1.2,
min_chunk_length: 50,
condition_on_previous_chunks: true,
early_stop_threshold: 0.9,
features: ["quality-guard"],
prosody: { volume: 3, normalize_loudness: true },
},
},
},
credentials: { apiKey: "fk" },
});
assert.equal(captured.headers.Authorization, "Bearer fk");
assert.equal(captured.headers.model, "s1");
assert.equal(captured.headers.model, "s2.1-pro-free");
assert.deepEqual(captured.body, {
text: "hi",
text: "Привіт",
format: "mp3",
reference_id: "ref-123",
prosody: { speed: 1.2 },
temperature: 0.8,
top_p: 0.6,
chunk_length: 240,
normalize: true,
sample_rate: 44100,
mp3_bitrate: 192,
latency: "normal",
max_new_tokens: 1024,
repetition_penalty: 1.2,
min_chunk_length: 50,
condition_on_previous_chunks: true,
early_stop_threshold: 0.9,
features: ["quality-guard"],
prosody: { speed: 1.2, volume: 3, normalize_loudness: true },
});
assert.equal(response.status, 200);
assert.equal(response.headers.get("content-type"), "audio/mpeg");
@@ -86,3 +117,47 @@ test("handleAudioSpeech requires credentials for Fish Audio", async () => {
assert.equal(response.status, 401);
assert.equal(payload.error.message, "No credentials for speech provider: fishaudio");
});
test("handleAudioSpeech rejects invalid Fish provider options before fetch", async () => {
const originalFetch = globalThis.fetch;
let called = false;
globalThis.fetch = async () => {
called = true;
return new Response();
};
try {
const response = await handleAudioSpeech({
body: {
model: "fishaudio/s2.1-pro-free",
input: "hi",
provider_options: { fishaudio: { temperature: 1.5 } },
},
credentials: { apiKey: "fk" },
});
const payload = (await response.json()) as { error: { message: string } };
assert.equal(response.status, 400);
assert.match(payload.error.message, /temperature must be <= 1/);
assert.equal(called, false);
} finally {
globalThis.fetch = originalFetch;
}
});
test("handleAudioSpeech explains that inline Fish references need MessagePack", async () => {
const response = await handleAudioSpeech({
body: {
model: "fishaudio/s2.1-pro-free",
input: "hi",
provider_options: { fishaudio: { references: [{ audio: "ignored", text: "hi" }] } },
},
credentials: { apiKey: "fk" },
});
const payload = (await response.json()) as { error: { message: string } };
assert.equal(response.status, 400);
assert.match(payload.error.message, /Fish Audio MessagePack/);
assert.match(payload.error.message, /persistent voice/i);
});

View File

@@ -0,0 +1,19 @@
import test from "node:test";
import assert from "node:assert/strict";
const { isFishAudioVoiceProvider, isSafeFishAudioVoiceId } = await import(
"../../src/app/api/v1/_shared/fishAudioProxy.ts"
);
test("Fish Audio voice management only accepts the Fish provider", () => {
assert.equal(isFishAudioVoiceProvider("fishaudio"), true);
assert.equal(isFishAudioVoiceProvider("elevenlabs"), false);
assert.equal(isFishAudioVoiceProvider("fishaudio/../evil"), false);
});
test("Fish Audio voice IDs reject traversal and URL-like input", () => {
assert.equal(isSafeFishAudioVoiceId("abc_DEF-123"), true);
assert.equal(isSafeFishAudioVoiceId("../secret"), false);
assert.equal(isSafeFishAudioVoiceId("https://example.com"), false);
assert.equal(isSafeFishAudioVoiceId("voice/id"), false);
});

View File

@@ -30,6 +30,7 @@ const EXPECTED: Record<InventoryKind, Record<string, number>> = {
"src/app/api/memory/rerank-providers/route.ts": 1,
"src/app/api/search/providers/route.ts": 3,
"src/app/api/v1/_shared/elevenLabsProxy.ts": 1,
"src/app/api/v1/_shared/fishAudioProxy.ts": 1,
"src/app/api/v1/audio/speech/route.ts": 1,
"src/app/api/v1/_shared/videoModelResolution.ts": 1,
"src/app/api/v1/audio/transcriptions/route.ts": 2,