mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-11 17:52:31 +03:00
fix(media): support Gemini Omni Flash video (#9982)
* feat(media): add provider-neutral video and music generation * fix(db): clean audit tables by created timestamp * fix(media): support Fal-hosted Grok video * fix(media): route Fal video references to Grok * fix(media): support Gemini Omni Flash video * fix(media): use Gemini Omni Flash Fal endpoint --------- Co-authored-by: rinseaid <rinseaid@rinseaid.net>
This commit is contained in:
rinseaid
@@ -520,18 +520,18 @@ export const IMAGE_PROVIDERS: Record<string, ImageProviderConfig> = {
|
||||
authHeader: "key",
|
||||
format: "fal-ai",
|
||||
models: [
|
||||
{ id: "fal-ai/flux-2-max", name: "FLUX.2 Max" },
|
||||
{ id: "fal-ai/flux-2-pro", name: "FLUX.2 Pro" },
|
||||
{ id: "fal-ai/flux-2-flex", name: "FLUX.2 Flex" },
|
||||
{ id: "flux-2-max", name: "FLUX.2 Max" },
|
||||
{ id: "flux-2-pro", name: "FLUX.2 Pro" },
|
||||
{ id: "flux-2-flex", name: "FLUX.2 Flex" },
|
||||
{ id: "bria/text-to-image/3.2", name: "Bria 3.2" },
|
||||
{ id: "fal-ai/bytedance/seedream/v4.5/text-to-image", name: "SeeDream V4.5" },
|
||||
{ id: "fal-ai/bytedance/dreamina/v3.1/text-to-image", name: "Dreamina V3.1" },
|
||||
{ id: "fal-ai/ideogram/v3", name: "Ideogram V3" },
|
||||
{ id: "fal-ai/nano-banana-pro", name: "Nano Banana Pro" },
|
||||
{ id: "fal-ai/nano-banana-2", name: "Nano Banana 2" },
|
||||
{ id: "fal-ai/recraft/v4/pro/text-to-image", name: "Recraft V4 Pro via Fal" },
|
||||
{ id: "fal-ai/recraft/v4/text-to-image", name: "Recraft V4 via Fal" },
|
||||
{ id: "fal-ai/stable-diffusion-v35-medium", name: "Stable Diffusion v3.5 Medium" },
|
||||
{ id: "bytedance/seedream/v4.5/text-to-image", name: "SeeDream V4.5" },
|
||||
{ id: "bytedance/dreamina/v3.1/text-to-image", name: "Dreamina V3.1" },
|
||||
{ id: "ideogram/v3", name: "Ideogram V3" },
|
||||
{ id: "nano-banana-pro", name: "Nano Banana Pro" },
|
||||
{ id: "nano-banana-2", name: "Nano Banana 2" },
|
||||
{ id: "recraft/v4/pro/text-to-image", name: "Recraft V4 Pro via Fal" },
|
||||
{ id: "recraft/v4/text-to-image", name: "Recraft V4 via Fal" },
|
||||
{ id: "stable-diffusion-v35-medium", name: "Stable Diffusion v3.5 Medium" },
|
||||
],
|
||||
supportedSizes: ["1024x1024", "1024x1280", "1280x1024"],
|
||||
},
|
||||
@@ -852,15 +852,13 @@ export function parseImageModel(modelStr) {
|
||||
for (const [providerId, config] of Object.entries(IMAGE_PROVIDERS)) {
|
||||
if (modelStr.startsWith(providerId + "/")) {
|
||||
const model = modelStr.slice(providerId.length + 1);
|
||||
const aliased =
|
||||
resolveImageModelAlias(`${providerId}/${model}`) || resolveImageModelAlias(model);
|
||||
const aliased = resolveImageModelAlias(`${providerId}/${model}`);
|
||||
return aliased || { provider: providerId, model };
|
||||
}
|
||||
// Check alias if available
|
||||
if (config.alias && modelStr.startsWith(config.alias + "/")) {
|
||||
const model = modelStr.slice(config.alias.length + 1);
|
||||
const aliased =
|
||||
resolveImageModelAlias(`${providerId}/${model}`) || resolveImageModelAlias(model);
|
||||
const aliased = resolveImageModelAlias(`${providerId}/${model}`);
|
||||
return aliased || { provider: providerId, model };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,15 @@ export const MUSIC_PROVIDERS: Record<string, MusicProvider> = {
|
||||
models: [{ id: "lyria-002", name: "Lyria 2 (Vertex)" }],
|
||||
},
|
||||
|
||||
"fal-ai": {
|
||||
id: "fal-ai",
|
||||
baseUrl: "https://queue.fal.run",
|
||||
authType: "apikey",
|
||||
authHeader: "key",
|
||||
format: "fal-ai-music",
|
||||
models: [{ id: "ace-step", name: "ACE-Step" }],
|
||||
},
|
||||
|
||||
kie: {
|
||||
id: "kie",
|
||||
baseUrl: "https://api.kie.ai",
|
||||
|
||||
@@ -96,6 +96,7 @@ export const VIDEO_PROVIDERS: Record<string, VideoProvider> = {
|
||||
format: "fal-ai-video",
|
||||
models: [
|
||||
{ id: "veo3.1/lite", name: "Veo 3.1 Lite" },
|
||||
{ id: "google/gemini-omni-flash", name: "Gemini Omni Flash" },
|
||||
{
|
||||
id: "xai/grok-imagine-video/text-to-video",
|
||||
name: "Grok Imagine Video",
|
||||
|
||||
@@ -1498,6 +1498,7 @@ async function handleFalAIImageGeneration({
|
||||
}) {
|
||||
const startTime = Date.now();
|
||||
const token = credentials.apiKey || credentials.accessToken;
|
||||
const falModel = model.startsWith("fal-ai/") ? model : `fal-ai/${model}`;
|
||||
const { imageUrl, imageUrls } = extractImageInputs(body);
|
||||
const upstreamBody: Record<string, unknown> = {
|
||||
prompt: body.prompt,
|
||||
@@ -1543,7 +1544,7 @@ async function handleFalAIImageGeneration({
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${providerConfig.baseUrl.replace(/\/$/, "")}/${model}`, {
|
||||
const response = await fetch(`${providerConfig.baseUrl.replace(/\/$/, "")}/${falModel}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
|
||||
359
open-sse/handlers/mediaGeneration/fal.test.ts
Normal file
359
open-sse/handlers/mediaGeneration/fal.test.ts
Normal file
@@ -0,0 +1,359 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
buildFalMusicRequestBody,
|
||||
buildFalVideoRequestBody,
|
||||
handleFalMusicGeneration,
|
||||
handleFalVideoGeneration,
|
||||
normalizeFalMediaResult,
|
||||
} from "./fal.ts";
|
||||
import { parseImageModel } from "../../config/imageRegistry.ts";
|
||||
import { parseMusicModel } from "../../config/musicRegistry.ts";
|
||||
import { parseVideoModel } from "../../config/videoRegistry.ts";
|
||||
|
||||
test("buildFalVideoRequestBody maps the OpenAI-compatible request", () => {
|
||||
assert.deepEqual(
|
||||
buildFalVideoRequestBody({
|
||||
prompt: "A quiet train crossing a snowy bridge",
|
||||
aspect_ratio: "9:16",
|
||||
duration: 6,
|
||||
resolution: "1080p",
|
||||
generate_audio: false,
|
||||
negative_prompt: "text overlays",
|
||||
seed: 42,
|
||||
}),
|
||||
{
|
||||
prompt: "A quiet train crossing a snowy bridge",
|
||||
aspect_ratio: "9:16",
|
||||
duration: "6s",
|
||||
resolution: "1080p",
|
||||
generate_audio: false,
|
||||
negative_prompt: "text overlays",
|
||||
seed: 42,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("buildFalVideoRequestBody maps the Fal-hosted Grok endpoint schema", () => {
|
||||
assert.deepEqual(
|
||||
buildFalVideoRequestBody(
|
||||
{
|
||||
prompt: "A realistic dog walking through a park",
|
||||
aspect_ratio: "16:9",
|
||||
duration: "8s",
|
||||
resolution: "720p",
|
||||
generate_audio: true,
|
||||
},
|
||||
"xai/grok-imagine-video/text-to-video"
|
||||
),
|
||||
{
|
||||
prompt: "A realistic dog walking through a park",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 8,
|
||||
resolution: "720p",
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("buildFalVideoRequestBody maps one provider-neutral image reference", () => {
|
||||
assert.deepEqual(
|
||||
buildFalVideoRequestBody(
|
||||
{
|
||||
prompt: "Animate this dog",
|
||||
image_urls: ["data:image/png;base64,ZmFrZQ=="],
|
||||
},
|
||||
"xai/grok-imagine-video/text-to-video"
|
||||
),
|
||||
{
|
||||
prompt: "Animate this dog",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 6,
|
||||
resolution: "720p",
|
||||
image_url: "data:image/png;base64,ZmFrZQ==",
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("buildFalVideoRequestBody maps multiple provider-neutral image references", () => {
|
||||
assert.deepEqual(
|
||||
buildFalVideoRequestBody(
|
||||
{
|
||||
prompt: "Combine these references",
|
||||
image_urls: ["data:image/png;base64,YQ==", "data:image/png;base64,Yg=="],
|
||||
},
|
||||
"xai/grok-imagine-video/text-to-video"
|
||||
),
|
||||
{
|
||||
prompt: "Combine these references",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 6,
|
||||
resolution: "720p",
|
||||
reference_image_urls: ["data:image/png;base64,YQ==", "data:image/png;base64,Yg=="],
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("buildFalVideoRequestBody maps the Gemini Omni Flash video schema", () => {
|
||||
assert.deepEqual(
|
||||
buildFalVideoRequestBody(
|
||||
{
|
||||
prompt: "A realistic dog walking through a park",
|
||||
aspect_ratio: "9:16",
|
||||
duration: 10,
|
||||
resolution: "1080p",
|
||||
generate_audio: false,
|
||||
},
|
||||
"google/gemini-omni-flash"
|
||||
),
|
||||
{
|
||||
prompt: "A realistic dog walking through a park",
|
||||
aspect_ratio: "9:16",
|
||||
duration: 10,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("handleFalVideoGeneration selects Gemini Omni Flash image-to-video", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
requests.push({ url: String(input), body: String(init?.body) });
|
||||
return new Response(JSON.stringify({ video: { url: "https://cdn.example/gemini.mp4" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleFalVideoGeneration({
|
||||
model: "google/gemini-omni-flash",
|
||||
provider: "fal-ai",
|
||||
providerConfig: { baseUrl: "https://queue.fal.run" },
|
||||
body: {
|
||||
prompt: "Animate this dog",
|
||||
image_urls: ["data:image/png;base64,ZmFrZQ=="],
|
||||
duration: 8,
|
||||
},
|
||||
credentials: { apiKey: "test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(requests[0]?.url, "https://queue.fal.run/google/gemini-omni-flash/image-to-video");
|
||||
assert.deepEqual(JSON.parse(requests[0]?.body || "{}"), {
|
||||
prompt: "Animate this dog",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 8,
|
||||
image_url: "data:image/png;base64,ZmFrZQ==",
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("buildFalMusicRequestBody uses prompt as tags and supports lyrics", () => {
|
||||
assert.deepEqual(
|
||||
buildFalMusicRequestBody({
|
||||
prompt: "warm analog synthwave",
|
||||
lyrics: "[verse] Drive through the night",
|
||||
duration: 30,
|
||||
seed: 7,
|
||||
}),
|
||||
{
|
||||
tags: "warm analog synthwave",
|
||||
lyrics: "[verse] Drive through the night",
|
||||
duration: 30,
|
||||
seed: 7,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("normalizeFalMediaResult returns typed media URLs", () => {
|
||||
assert.deepEqual(
|
||||
normalizeFalMediaResult(
|
||||
{
|
||||
video: {
|
||||
url: "https://cdn.example/video.mp4",
|
||||
content_type: "video/mp4",
|
||||
},
|
||||
},
|
||||
"video"
|
||||
),
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
created: 0,
|
||||
data: [{ url: "https://cdn.example/video.mp4", format: "mp4" }],
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
normalizeFalMediaResult({ audio: { url: "https://cdn.example/song.wav" } }, "music"),
|
||||
{
|
||||
success: true,
|
||||
data: {
|
||||
created: 0,
|
||||
data: [{ url: "https://cdn.example/song.wav", format: "wav" }],
|
||||
},
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("normalizeFalMediaResult rejects a missing artifact", () => {
|
||||
assert.deepEqual(normalizeFalMediaResult({}, "video"), {
|
||||
success: false,
|
||||
status: 502,
|
||||
error: "Fal video generation returned no media URL",
|
||||
});
|
||||
});
|
||||
|
||||
test("media registries expose provider-neutral model IDs", () => {
|
||||
assert.deepEqual(parseImageModel("fal-ai/flux-2-pro"), {
|
||||
provider: "fal-ai",
|
||||
model: "flux-2-pro",
|
||||
});
|
||||
assert.deepEqual(parseVideoModel("fal-ai/veo3.1/lite"), {
|
||||
provider: "fal-ai",
|
||||
model: "veo3.1/lite",
|
||||
});
|
||||
assert.deepEqual(parseMusicModel("fal-ai/ace-step"), {
|
||||
provider: "fal-ai",
|
||||
model: "ace-step",
|
||||
});
|
||||
});
|
||||
|
||||
test("handleFalVideoGeneration uses the provider-neutral queue contract", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
requests.push({ url: String(input), body: String(init?.body) });
|
||||
return new Response(JSON.stringify({ video: { url: "https://cdn.example/video.mp4" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleFalVideoGeneration({
|
||||
model: "veo3.1/lite",
|
||||
provider: "fal-ai",
|
||||
providerConfig: { baseUrl: "https://queue.fal.run" },
|
||||
body: { prompt: "a slow pan across a forest", duration: 4 },
|
||||
credentials: { apiKey: "test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(requests[0]?.url, "https://queue.fal.run/fal-ai/veo3.1/lite");
|
||||
assert.deepEqual(JSON.parse(requests[0]?.body || "{}"), {
|
||||
prompt: "a slow pan across a forest",
|
||||
aspect_ratio: "16:9",
|
||||
duration: "4s",
|
||||
resolution: "720p",
|
||||
generate_audio: true,
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleFalVideoGeneration preserves Fal model paths outside the fal-ai namespace", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
requests.push({ url: String(input), body: String(init?.body) });
|
||||
return new Response(JSON.stringify({ video: { url: "https://cdn.example/grok.mp4" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleFalVideoGeneration({
|
||||
model: "xai/grok-imagine-video/text-to-video",
|
||||
provider: "fal-ai",
|
||||
providerConfig: { baseUrl: "https://queue.fal.run" },
|
||||
body: { prompt: "a dog walking through a park", duration: 8 },
|
||||
credentials: { apiKey: "test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(requests[0]?.url, "https://queue.fal.run/xai/grok-imagine-video/text-to-video");
|
||||
assert.deepEqual(JSON.parse(requests[0]?.body || "{}"), {
|
||||
prompt: "a dog walking through a park",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 8,
|
||||
resolution: "720p",
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleFalVideoGeneration selects Grok image-to-video for one reference image", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
requests.push({ url: String(input), body: String(init?.body) });
|
||||
return new Response(JSON.stringify({ video: { url: "https://cdn.example/grok-i2v.mp4" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleFalVideoGeneration({
|
||||
model: "xai/grok-imagine-video/text-to-video",
|
||||
provider: "fal-ai",
|
||||
providerConfig: { baseUrl: "https://queue.fal.run" },
|
||||
body: {
|
||||
prompt: "Animate this dog",
|
||||
image_urls: ["data:image/png;base64,ZmFrZQ=="],
|
||||
},
|
||||
credentials: { apiKey: "test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(requests[0]?.url, "https://queue.fal.run/xai/grok-imagine-video/image-to-video");
|
||||
assert.deepEqual(JSON.parse(requests[0]?.body || "{}"), {
|
||||
prompt: "Animate this dog",
|
||||
aspect_ratio: "16:9",
|
||||
duration: 6,
|
||||
resolution: "720p",
|
||||
image_url: "data:image/png;base64,ZmFrZQ==",
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleFalMusicGeneration uses the provider-neutral queue contract", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const requests: Array<{ url: string; body: string }> = [];
|
||||
globalThis.fetch = async (input, init) => {
|
||||
requests.push({ url: String(input), body: String(init?.body) });
|
||||
return new Response(JSON.stringify({ audio: { url: "https://cdn.example/music.wav" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleFalMusicGeneration({
|
||||
model: "ace-step",
|
||||
provider: "fal-ai",
|
||||
providerConfig: { baseUrl: "https://queue.fal.run" },
|
||||
body: { prompt: "ambient synthwave", lyrics: "stay awake", duration: 30 },
|
||||
credentials: { apiKey: "test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(requests[0]?.url, "https://queue.fal.run/fal-ai/ace-step");
|
||||
assert.deepEqual(JSON.parse(requests[0]?.body || "{}"), {
|
||||
tags: "ambient synthwave",
|
||||
lyrics: "stay awake",
|
||||
duration: 30,
|
||||
});
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
396
open-sse/handlers/mediaGeneration/fal.ts
Normal file
396
open-sse/handlers/mediaGeneration/fal.ts
Normal file
@@ -0,0 +1,396 @@
|
||||
import { saveCallLog } from "@/lib/usageDb";
|
||||
import {
|
||||
FetchTimeoutError,
|
||||
fetchWithTimeout,
|
||||
getConfiguredTimeout,
|
||||
} from "../../../src/shared/utils/fetchTimeout.ts";
|
||||
import { sanitizeErrorMessage } from "../../utils/error.ts";
|
||||
|
||||
type MediaKind = "video" | "music";
|
||||
|
||||
type FalBody = Record<string, unknown>;
|
||||
|
||||
type FalCredentials = {
|
||||
apiKey?: unknown;
|
||||
accessToken?: unknown;
|
||||
};
|
||||
|
||||
type FalProviderConfig = {
|
||||
baseUrl: string;
|
||||
};
|
||||
|
||||
type FalLog = {
|
||||
info?: (scope: string, message: string, meta?: unknown) => void;
|
||||
error?: (scope: string, message: string) => void;
|
||||
};
|
||||
|
||||
function stringValue(value: unknown): string | undefined {
|
||||
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||
}
|
||||
|
||||
function stringArray(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map(stringValue).filter((value): value is string => Boolean(value));
|
||||
}
|
||||
|
||||
function numberValue(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function falDuration(value: unknown, fallback: string): string {
|
||||
if (typeof value === "string" && /^(4|6|8)s$/.test(value)) return value;
|
||||
const numeric = numberValue(value);
|
||||
return numeric && [4, 6, 8].includes(numeric) ? `${numeric}s` : fallback;
|
||||
}
|
||||
|
||||
function grokDuration(value: unknown, fallback = 6): number {
|
||||
const numeric = numberValue(value);
|
||||
if (numeric !== undefined) return Math.round(numeric);
|
||||
if (typeof value === "string") {
|
||||
const match = value.trim().match(/^(\d+)s$/);
|
||||
if (match) return Number(match[1]);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function geminiDuration(value: unknown, fallback = 8): number {
|
||||
const numeric = numberValue(value);
|
||||
const parsed =
|
||||
numeric ??
|
||||
(typeof value === "string" && /^\d+(?:\.\d+)?s$/.test(value.trim())
|
||||
? Number(value.trim().slice(0, -1))
|
||||
: undefined);
|
||||
return parsed === undefined ? fallback : Math.min(10, Math.max(3, Math.round(parsed)));
|
||||
}
|
||||
|
||||
export function buildFalVideoRequestBody(body: FalBody, model = ""): FalBody {
|
||||
if (model.startsWith("google/gemini-omni-flash")) {
|
||||
const request: FalBody = {
|
||||
prompt: stringValue(body.prompt) || "",
|
||||
aspect_ratio: stringValue(body.aspect_ratio) || "16:9",
|
||||
duration: geminiDuration(body.duration),
|
||||
};
|
||||
|
||||
const imageUrl = stringValue(body.image_url) || stringArray(body.image_urls)[0];
|
||||
if (imageUrl) request.image_url = imageUrl;
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
if (model.startsWith("xai/grok-imagine-video/")) {
|
||||
const request: FalBody = {
|
||||
prompt: stringValue(body.prompt) || "",
|
||||
aspect_ratio: stringValue(body.aspect_ratio) || "16:9",
|
||||
duration: grokDuration(body.duration),
|
||||
resolution: stringValue(body.resolution) || "720p",
|
||||
};
|
||||
|
||||
const imageUrls = stringArray(body.image_urls);
|
||||
if (imageUrls.length === 1) {
|
||||
request.image_url = imageUrls[0];
|
||||
} else if (imageUrls.length > 1) {
|
||||
request.reference_image_urls = imageUrls;
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
const request: FalBody = {
|
||||
prompt: stringValue(body.prompt) || "",
|
||||
aspect_ratio: stringValue(body.aspect_ratio) || "16:9",
|
||||
duration: falDuration(body.duration, "8s"),
|
||||
resolution: stringValue(body.resolution) || (body.quality === "hd" ? "1080p" : "720p"),
|
||||
generate_audio: typeof body.generate_audio === "boolean" ? body.generate_audio : true,
|
||||
};
|
||||
|
||||
const optionalStringFields = ["negative_prompt", "safety_tolerance"];
|
||||
for (const field of optionalStringFields) {
|
||||
const value = stringValue(body[field]);
|
||||
if (value) request[field] = value;
|
||||
}
|
||||
|
||||
const seed = numberValue(body.seed);
|
||||
if (seed !== undefined) request.seed = seed;
|
||||
if (typeof body.auto_fix === "boolean") request.auto_fix = body.auto_fix;
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
function resolveFalModel(model: string, body: FalBody, kind: MediaKind): string {
|
||||
if (kind !== "video") return model;
|
||||
|
||||
if (model.startsWith("google/gemini-omni-flash") && !model.endsWith("/image-to-video")) {
|
||||
const hasImage = typeof body.image_url === "string" || stringArray(body.image_urls).length > 0;
|
||||
return hasImage ? "google/gemini-omni-flash/image-to-video" : model;
|
||||
}
|
||||
|
||||
if (!model.startsWith("xai/grok-imagine-video/")) return model;
|
||||
|
||||
const suffix = Array.isArray(body.reference_image_urls)
|
||||
? "reference-to-video"
|
||||
: typeof body.image_url === "string"
|
||||
? "image-to-video"
|
||||
: "text-to-video";
|
||||
return `xai/grok-imagine-video/${suffix}`;
|
||||
}
|
||||
|
||||
export function buildFalMusicRequestBody(body: FalBody): FalBody {
|
||||
const request: FalBody = {
|
||||
tags: stringValue(body.tags) || stringValue(body.prompt) || "",
|
||||
};
|
||||
|
||||
const lyrics = stringValue(body.lyrics);
|
||||
if (lyrics) request.lyrics = lyrics;
|
||||
|
||||
const duration = numberValue(body.duration);
|
||||
if (duration !== undefined) request.duration = Math.min(240, Math.max(5, duration));
|
||||
|
||||
const seed = numberValue(body.seed);
|
||||
if (seed !== undefined) request.seed = seed;
|
||||
|
||||
const optionalNumberFields = [
|
||||
"number_of_steps",
|
||||
"granularity_scale",
|
||||
"guidance_interval",
|
||||
"guidance_interval_decay",
|
||||
"tag_guidance_scale",
|
||||
"lyric_guidance_scale",
|
||||
"minimum_guidance_scale",
|
||||
"guidance_scale",
|
||||
];
|
||||
for (const field of optionalNumberFields) {
|
||||
const value = numberValue(body[field]);
|
||||
if (value !== undefined) request[field] = value;
|
||||
}
|
||||
|
||||
const scheduler = stringValue(body.scheduler);
|
||||
if (scheduler === "euler" || scheduler === "heun") request.scheduler = scheduler;
|
||||
|
||||
const guidanceType = stringValue(body.guidance_type);
|
||||
if (guidanceType === "cfg" || guidanceType === "apg" || guidanceType === "cfg_star") {
|
||||
request.guidance_type = guidanceType;
|
||||
}
|
||||
|
||||
return request;
|
||||
}
|
||||
|
||||
function extensionFromMedia(item: Record<string, unknown>, kind: MediaKind): string {
|
||||
const contentType = stringValue(item.content_type);
|
||||
if (contentType?.includes("/")) return contentType.split("/", 2)[1];
|
||||
|
||||
const fileName = stringValue(item.file_name);
|
||||
const url = stringValue(item.url);
|
||||
const candidate = fileName || url || "";
|
||||
const extension = candidate.match(/\.([a-z0-9]+)(?:\?|$)/i)?.[1]?.toLowerCase();
|
||||
return extension || (kind === "video" ? "mp4" : "wav");
|
||||
}
|
||||
|
||||
export function normalizeFalMediaResult(payload: unknown, kind: MediaKind) {
|
||||
const record = payload && typeof payload === "object" ? (payload as FalBody) : {};
|
||||
const media = record[kind === "video" ? "video" : "audio"];
|
||||
const item = media && typeof media === "object" ? (media as Record<string, unknown>) : null;
|
||||
const url = stringValue(item?.url);
|
||||
|
||||
if (!url) {
|
||||
return {
|
||||
success: false as const,
|
||||
status: 502,
|
||||
error: `Fal ${kind} generation returned no media URL`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true as const,
|
||||
data: {
|
||||
created: numberValue(record.created) || 0,
|
||||
data: [{ url, format: extensionFromMedia(item, kind) }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function absoluteFalUrl(value: unknown, baseUrl: string): string | undefined {
|
||||
const url = stringValue(value);
|
||||
if (!url) return undefined;
|
||||
return url.startsWith("http://") || url.startsWith("https://")
|
||||
? url
|
||||
: `${baseUrl.replace(/\/$/, "")}/${url.replace(/^\//, "")}`;
|
||||
}
|
||||
|
||||
function getToken(credentials: FalCredentials | null | undefined): string {
|
||||
return String(credentials?.apiKey || credentials?.accessToken || "");
|
||||
}
|
||||
|
||||
async function wait(ms: number) {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function runFalQueue({
|
||||
model,
|
||||
body,
|
||||
kind,
|
||||
provider,
|
||||
providerConfig,
|
||||
credentials,
|
||||
log,
|
||||
}: {
|
||||
model: string;
|
||||
body: FalBody;
|
||||
kind: MediaKind;
|
||||
provider: string;
|
||||
providerConfig: FalProviderConfig;
|
||||
credentials: FalCredentials | null | undefined;
|
||||
log?: FalLog | null;
|
||||
}) {
|
||||
const startTime = Date.now();
|
||||
const baseUrl = providerConfig.baseUrl.replace(/\/$/, "");
|
||||
const token = getToken(credentials);
|
||||
const headers = {
|
||||
Authorization: `Key ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
const timeoutMs = getConfiguredTimeout();
|
||||
const deadline = startTime + timeoutMs;
|
||||
const resolvedModel = resolveFalModel(model, body, kind);
|
||||
const falModel =
|
||||
resolvedModel.startsWith("fal-ai/") ||
|
||||
resolvedModel.startsWith("xai/") ||
|
||||
resolvedModel.startsWith("google/")
|
||||
? resolvedModel
|
||||
: `fal-ai/${resolvedModel}`;
|
||||
const queueUrl = `${baseUrl}/${falModel}`;
|
||||
|
||||
try {
|
||||
const createResponse = await fetchWithTimeout(queueUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
timeoutMs,
|
||||
});
|
||||
const createPayload = await createResponse.json().catch(() => ({}));
|
||||
|
||||
if (!createResponse.ok) {
|
||||
const error = JSON.stringify(createPayload).slice(0, 500);
|
||||
log?.error?.(
|
||||
"MEDIA",
|
||||
`${provider} ${kind} create failed (${createResponse.status}): ${error}`
|
||||
);
|
||||
saveCallLog({
|
||||
method: "POST",
|
||||
path: `/v1/${kind === "video" ? "videos" : "music"}/generations`,
|
||||
status: createResponse.status,
|
||||
model: `${provider}/${model}`,
|
||||
provider,
|
||||
duration: Date.now() - startTime,
|
||||
error,
|
||||
}).catch(() => {});
|
||||
return { success: false, status: createResponse.status, error };
|
||||
}
|
||||
|
||||
const requestId = stringValue(createPayload?.request_id);
|
||||
if (!requestId) {
|
||||
const normalized = normalizeFalMediaResult(createPayload, kind);
|
||||
if (!normalized.success) return normalized;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
const statusUrl =
|
||||
absoluteFalUrl(createPayload.status_url, baseUrl) ||
|
||||
`${queueUrl}/requests/${requestId}/status`;
|
||||
const responseUrl =
|
||||
absoluteFalUrl(createPayload.response_url, baseUrl) || `${queueUrl}/requests/${requestId}`;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const statusResponse = await fetchWithTimeout(statusUrl, {
|
||||
headers: { Authorization: `Key ${token}` },
|
||||
timeoutMs: Math.min(getConfiguredTimeout(), Math.max(1000, deadline - Date.now())),
|
||||
});
|
||||
const statusPayload = await statusResponse.json().catch(() => ({}));
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
const error = JSON.stringify(statusPayload).slice(0, 500);
|
||||
return { success: false, status: statusResponse.status, error };
|
||||
}
|
||||
|
||||
const status = stringValue(statusPayload?.status);
|
||||
if (status === "COMPLETED") {
|
||||
const resultResponse = await fetchWithTimeout(responseUrl, {
|
||||
headers: { Authorization: `Key ${token}` },
|
||||
timeoutMs: Math.min(getConfiguredTimeout(), Math.max(1000, deadline - Date.now())),
|
||||
});
|
||||
const resultPayload = await resultResponse.json().catch(() => ({}));
|
||||
if (!resultResponse.ok) {
|
||||
return {
|
||||
success: false,
|
||||
status: resultResponse.status,
|
||||
error: JSON.stringify(resultPayload).slice(0, 500),
|
||||
};
|
||||
}
|
||||
|
||||
const normalized = normalizeFalMediaResult(resultPayload, kind);
|
||||
saveCallLog({
|
||||
method: "POST",
|
||||
path: `/v1/${kind === "video" ? "videos" : "music"}/generations`,
|
||||
status: normalized.success ? 200 : normalized.status,
|
||||
model: `${provider}/${model}`,
|
||||
provider,
|
||||
duration: Date.now() - startTime,
|
||||
...(normalized.success ? {} : { error: normalized.error }),
|
||||
}).catch(() => {});
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (status && !["IN_QUEUE", "IN_PROGRESS"].includes(status)) {
|
||||
return {
|
||||
success: false,
|
||||
status: 502,
|
||||
error: `Fal ${kind} generation ended with status ${status}`,
|
||||
};
|
||||
}
|
||||
|
||||
await wait(Math.min(1000, Math.max(100, deadline - Date.now())));
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
status: 504,
|
||||
error: `Fal ${kind} generation timed out after ${timeoutMs}ms`,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
const isTimeout =
|
||||
error instanceof FetchTimeoutError || (error as { name?: string })?.name === "AbortError";
|
||||
const status = isTimeout ? 504 : 502;
|
||||
log?.error?.("MEDIA", `${provider} ${kind} request failed: ${sanitizeErrorMessage(message)}`);
|
||||
return {
|
||||
success: false,
|
||||
status,
|
||||
error: `Fal ${kind} provider error: ${sanitizeErrorMessage(message)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function handleFalVideoGeneration(args: {
|
||||
model: string;
|
||||
provider: string;
|
||||
providerConfig: FalProviderConfig;
|
||||
body: FalBody;
|
||||
credentials: FalCredentials | null | undefined;
|
||||
log?: FalLog | null;
|
||||
}) {
|
||||
return runFalQueue({
|
||||
...args,
|
||||
body: buildFalVideoRequestBody(args.body, args.model),
|
||||
kind: "video",
|
||||
});
|
||||
}
|
||||
|
||||
export function handleFalMusicGeneration(args: {
|
||||
model: string;
|
||||
provider: string;
|
||||
providerConfig: FalProviderConfig;
|
||||
body: FalBody;
|
||||
credentials: FalCredentials | null | undefined;
|
||||
log?: FalLog | null;
|
||||
}) {
|
||||
return runFalQueue({ ...args, body: buildFalMusicRequestBody(args.body), kind: "music" });
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import {
|
||||
parseKieResultJson,
|
||||
} from "../utils/kieTask.ts";
|
||||
import { sanitizeErrorMessage } from "../utils/error.ts";
|
||||
import { handleFalMusicGeneration } from "./mediaGeneration/fal.ts";
|
||||
|
||||
function normalizeKieSunoModel(model: string): string {
|
||||
const map: Record<string, string> = {
|
||||
@@ -124,6 +125,10 @@ export async function handleMusicGeneration({ body, credentials, log }) {
|
||||
}
|
||||
}
|
||||
|
||||
if (providerConfig.format === "fal-ai-music") {
|
||||
return handleFalMusicGeneration({ model, provider, providerConfig, body, credentials, log });
|
||||
}
|
||||
|
||||
if (providerConfig.format === "comfyui") {
|
||||
return handleComfyUIMusicGeneration({
|
||||
model,
|
||||
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
fetchWithTimeout,
|
||||
getConfiguredTimeout,
|
||||
} from "@/shared/utils/fetchTimeout";
|
||||
import { handleFalVideoGeneration } from "./mediaGeneration/fal.ts";
|
||||
|
||||
/**
|
||||
* Resolve the base URL for OpenAI-compatible video generation endpoints.
|
||||
|
||||
@@ -82,24 +82,32 @@ test("cleanup: scheduler is wired into server-init.ts", () => {
|
||||
|
||||
test("cleanup: mcp_tool_audit uses correct table name (not 'mcp_audit_log')", () => {
|
||||
assert.ok(
|
||||
source.includes("DELETE FROM mcp_tool_audit WHERE"),
|
||||
"must use correct table name mcp_tool_audit"
|
||||
source.includes("DELETE FROM mcp_tool_audit WHERE created_at < ?"),
|
||||
"mcp_tool_audit cleanup must use its created_at column"
|
||||
);
|
||||
assert.ok(
|
||||
!source.includes("DELETE FROM mcp_audit_log WHERE"),
|
||||
"must NOT use non-existent table name mcp_audit_log"
|
||||
);
|
||||
assert.ok(
|
||||
!source.includes("DELETE FROM mcp_tool_audit WHERE timestamp"),
|
||||
"must NOT use timestamp for mcp_tool_audit"
|
||||
);
|
||||
});
|
||||
|
||||
test("cleanup: a2a_task_events uses correct table name (not 'a2a_events')", () => {
|
||||
assert.ok(
|
||||
source.includes("DELETE FROM a2a_task_events WHERE"),
|
||||
"must use correct table name a2a_task_events"
|
||||
source.includes("DELETE FROM a2a_task_events WHERE created_at < ?"),
|
||||
"a2a_task_events cleanup must use its created_at column"
|
||||
);
|
||||
assert.ok(
|
||||
!source.includes("DELETE FROM a2a_events WHERE"),
|
||||
"must NOT use non-existent table name a2a_events"
|
||||
);
|
||||
assert.ok(
|
||||
!source.includes("DELETE FROM a2a_task_events WHERE timestamp"),
|
||||
"must NOT use timestamp for a2a_task_events"
|
||||
);
|
||||
});
|
||||
|
||||
test("cleanup: memories uses correct table name (not 'memory_entries')", () => {
|
||||
|
||||
Reference in New Issue
Block a user