diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f80a616e5..e666de56fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ _In development — bullets added per PR; finalized at release._ - **kiro**: inline `` stream splitter — when `enabled` is present, `assistantResponseEvent` content is now split into separate `delta.content` / `delta.reasoning_content` SSE chunks (new `open-sse/executors/kiroThinking.ts` module wired into `KiroExecutor.transformEventStreamToSSE`). - **feat(cursor):** parse Cursor Composer DeepSeek-style inline tool calls — Composer `cu/composer-2.5*` models embed tool invocations in their visible text using `<|tool▁calls▁begin|>…<|tool▁calls▁end|>` markers instead of structured protobuf frames; a new streaming parser (`composerToolCalls.ts`) intercepts these in both streaming and non-streaming paths, suppresses the markers from the client-visible content, and emits proper OpenAI `tool_calls` deltas so downstream clients handle them natively. (thanks @noestelar) - **feat(proxy):** support auth-less `host:port` batch import and surface proxy-test failures. (thanks @dimaslanjaka) +- **feat(video): Alibaba DashScope video provider (`wan2.7-t2v`)** — adds the `alibaba` video provider (DashScope async task → poll → MP4) wired through the standard apikey credential path, so text-to-video requests can route to Alibaba's `wan2.7-t2v` model. (thanks @josevictorferreira) ### 🔧 Bug Fixes diff --git a/open-sse/config/videoRegistry.ts b/open-sse/config/videoRegistry.ts index af1f3652b4..442699b047 100644 --- a/open-sse/config/videoRegistry.ts +++ b/open-sse/config/videoRegistry.ts @@ -192,6 +192,19 @@ export const VIDEO_PROVIDERS: Record = { format: "runwayml", models: RUNWAYML_SUPPORTED_VIDEO_MODELS, }, + + alibaba: { + id: "alibaba", + alias: "ali", + // DashScope (Alibaba Cloud Model Studio) async video-synthesis API. Reuses + // the stored alibaba provider Bearer apiKey — no separate credential flow. + baseUrl: "https://dashscope-intl.aliyuncs.com/api/v1", + statusUrl: "https://dashscope-intl.aliyuncs.com/api/v1/tasks", + authType: "apikey", + authHeader: "bearer", + format: "dashscope-video", + models: [{ id: "wan2.7-t2v", name: "Wan 2.7 T2V" }], + }, }; /** diff --git a/open-sse/handlers/videoGeneration.ts b/open-sse/handlers/videoGeneration.ts index d3708fa4be..16c9c27d41 100644 --- a/open-sse/handlers/videoGeneration.ts +++ b/open-sse/handlers/videoGeneration.ts @@ -101,6 +101,17 @@ export async function handleVideoGeneration({ body, credentials, log }) { }); } + if (providerConfig.format === "dashscope-video") { + return handleDashscopeVideoGeneration({ + model, + provider, + providerConfig, + body, + credentials, + log, + }); + } + return { success: false, status: 400, @@ -108,6 +119,177 @@ export async function handleVideoGeneration({ body, credentials, log }) { }; } +/** + * Alibaba (DashScope) Wan video generation: create async task → poll → MP4. + * Targets wan2.7-t2v on the DashScope intl region. Reuses the stored alibaba + * provider Bearer apiKey — no separate credential flow. + */ +async function handleDashscopeVideoGeneration({ + model, + provider, + providerConfig, + body, + credentials, + log, +}: { + model: string; + provider: string; + providerConfig: { baseUrl: string; statusUrl?: string }; + body: Record & { + prompt?: unknown; + negative_prompt?: unknown; + size?: unknown; + aspect_ratio?: unknown; + duration?: unknown; + timeout_ms?: unknown; + poll_interval_ms?: unknown; + }; + credentials?: { apiKey?: string; accessToken?: string } | null; + log?: { + info: (scope: string, message: string) => void; + error: (scope: string, message: string) => void; + } | null; +}) { + const startTime = Date.now(); + const timeoutMs = Number(body.timeout_ms) > 0 ? Number(body.timeout_ms) : 300000; + const pollIntervalMs = Number(body.poll_interval_ms) > 0 ? Number(body.poll_interval_ms) : 2500; + const token = credentials?.apiKey || credentials?.accessToken; + const baseUrl = providerConfig.baseUrl.replace(/\/$/, ""); + const statusUrl = (providerConfig.statusUrl || `${baseUrl}/tasks`).replace(/\/$/, ""); + const prompt = typeof body.prompt === "string" ? body.prompt : String(body.prompt ?? ""); + + if (!token) { + return { success: false, status: 401, error: "Alibaba DashScope API key is required" }; + } + + const sizeParam = normalizeDashscopeSize(body.size, body.aspect_ratio); + const parameters: Record = {}; + if (sizeParam) parameters.size = sizeParam; + if (body.duration != null) parameters.duration = Number(body.duration); + + const payload = { + model, + input: { + prompt, + ...(typeof body.negative_prompt === "string" + ? { negative_prompt: body.negative_prompt } + : {}), + }, + parameters, + }; + + if (log) { + log.info( + "VIDEO", + `${provider}/${model} (dashscope-video) | prompt: "${prompt.slice(0, 60)}..."` + ); + } + + try { + // Step 1: create async task (X-DashScope-Async: enable) + const createRes = await fetch(`${baseUrl}/services/aigc/video-generation/video-synthesis`, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-DashScope-Async": "enable", + }, + body: JSON.stringify(payload), + }); + const createData = await createRes.json().catch(() => ({})); + const taskId = createData?.output?.task_id; + if (!taskId) { + const errorMessage = + createData?.message || + createData?.errors?.[0]?.message || + "DashScope video generation did not return task_id"; + if (log) { + log.error("VIDEO", `DashScope createTask failed: ${JSON.stringify(createData)}`); + } + return { success: false, status: 502, error: String(errorMessage) }; + } + + // Step 2: poll statusUrl/{task_id} until terminal + const deadline = startTime + timeoutMs; + let lastStatus = "PENDING"; + while (Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + const pollRes = await fetch(`${statusUrl}/${taskId}`, { + headers: { Authorization: `Bearer ${token}` }, + }); + const pollData = await pollRes.json().catch(() => ({})); + lastStatus = pollData?.output?.task_status || "PENDING"; + + if (lastStatus === "SUCCEEDED") { + const videoUrl = pollData?.output?.video_url; + if (!videoUrl) { + return { + success: false, + status: 502, + error: "DashScope task SUCCEEDED but no video_url", + }; + } + saveCallLog({ + method: "POST", + path: "/v1/videos/generations", + status: 200, + model: `${provider}/${model}`, + provider, + duration: Date.now() - startTime, + responseBody: { videos_count: 1 }, + }).catch(() => {}); + return { + success: true, + data: { + created: Math.floor(Date.now() / 1000), + data: [{ url: videoUrl, format: "mp4" }], + }, + }; + } + + if (lastStatus === "FAILED" || lastStatus === "UNKNOWN_ERROR") { + const errorMessage = + pollData?.output?.message || + pollData?.output?.errors?.[0]?.message || + "DashScope video task FAILED"; + return { success: false, status: 502, error: String(errorMessage) }; + } + // PENDING / RUNNING → keep polling + } + + return { + success: false, + status: 504, + error: `DashScope task ${taskId} timed out (status: ${lastStatus})`, + }; + } catch (err: unknown) { + return { + success: false, + status: isJsonObject(err) && Number.isFinite(Number(err.status)) ? Number(err.status) : 502, + error: sanitizeErrorMessage(err) || "Video provider error", + }; + } +} + +// Map OmniRoute size/aspect_ratio → Alibaba DashScope "WxH" (1280*720). +// Accepts "1280*720", "1280x720", or a ratio "16:9". Returns undefined if unparseable +// (then omitted from the payload so DashScope applies its own default). +function normalizeDashscopeSize(size: unknown, aspectRatio: unknown): string | undefined { + if (typeof size === "string") { + if (/^\d+\*\d+$/.test(size)) return size; + if (/^\d+x\d+$/.test(size)) return size.replace("x", "*"); + } + if (typeof aspectRatio === "string") { + const ratioMap: Record = { + "16:9": "1280*720", + "9:16": "720*1280", + "1:1": "960*960", + }; + return ratioMap[aspectRatio]; + } + return undefined; +} + /** * Veo video generation via Vertex AI (predictLongRunning → poll → MP4). * Uses the Vertex chat credentials (Service Account JSON or Express key). diff --git a/tests/unit/video-dashscope.test.ts b/tests/unit/video-dashscope.test.ts new file mode 100644 index 0000000000..907be50c21 --- /dev/null +++ b/tests/unit/video-dashscope.test.ts @@ -0,0 +1,216 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-video-dashscope-")); + +const { handleVideoGeneration } = await import("../../open-sse/handlers/videoGeneration.ts"); +const { VIDEO_PROVIDERS } = await import("../../open-sse/config/videoRegistry.ts"); + +// Makes poll-interval waits resolve instantly so tests don't sleep. +function immediateTimeout(callback, _ms, ...args) { + if (typeof callback === "function") callback(...args); + return 0; +} + +const CREATE_URL = + "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis"; +const POLL_URL_PREFIX = "https://dashscope-intl.aliyuncs.com/api/v1/tasks/"; + +function jsonResponse(body, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +test("VIDEO_PROVIDERS exposes the alibaba dashscope-video entry", () => { + assert.ok(VIDEO_PROVIDERS.alibaba, "alibaba video provider is registered"); + assert.equal(VIDEO_PROVIDERS.alibaba.format, "dashscope-video"); + assert.ok( + VIDEO_PROVIDERS.alibaba.models.some((m) => m.id === "wan2.7-t2v"), + "wan2.7-t2v is listed" + ); +}); + +test("handleVideoGeneration creates + polls a DashScope task and returns mp4 URL", async () => { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.setTimeout; + let createRequest; + + globalThis.setTimeout = immediateTimeout; + globalThis.fetch = async (url, options = {}) => { + const stringUrl = String(url); + + if (stringUrl === CREATE_URL) { + createRequest = { + url: stringUrl, + headers: options.headers, + body: JSON.parse(String(options.body || "{}")), + }; + return jsonResponse({ + output: { task_id: "ds-task-1", task_status: "PENDING" }, + request_id: "req-1", + }); + } + + if (stringUrl.startsWith(POLL_URL_PREFIX)) { + return jsonResponse({ + output: { + task_status: "SUCCEEDED", + video_url: "https://dashscope-cdn.example.com/wan-out.mp4", + }, + request_id: "req-2", + usage: { video_count: 1 }, + }); + } + + throw new Error(`Unexpected URL: ${stringUrl}`); + }; + + try { + const result = await handleVideoGeneration({ + body: { + model: "alibaba/wan2.7-t2v", + prompt: "a neon city in the rain", + negative_prompt: "blurry", + aspect_ratio: "16:9", + duration: 5, + }, + credentials: { apiKey: "dashscope-key" }, + log: null, + }); + + // Create request shape + assert.equal(createRequest.headers["X-DashScope-Async"], "enable"); + assert.equal(createRequest.headers["Authorization"], "Bearer dashscope-key"); + assert.equal(createRequest.body.model, "wan2.7-t2v"); + assert.equal(createRequest.body.input.prompt, "a neon city in the rain"); + assert.equal(createRequest.body.input.negative_prompt, "blurry"); + // aspect_ratio "16:9" → DashScope size "1280*720" + assert.equal(createRequest.body.parameters.size, "1280*720"); + assert.equal(createRequest.body.parameters.duration, 5); + + // Response shape + assert.equal(result.success, true); + assert.equal(result.data.data[0].url, "https://dashscope-cdn.example.com/wan-out.mp4"); + assert.equal(result.data.data[0].format, "mp4"); + } finally { + globalThis.fetch = originalFetch; + globalThis.setTimeout = originalSetTimeout; + } +}); + +test("handleVideoGeneration rejects DashScope requests without credentials", async () => { + const result = await handleVideoGeneration({ + body: { model: "alibaba/wan2.7-t2v", prompt: "x" }, + credentials: null, + log: null, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.match(result.error, /DashScope API key is required/); +}); + +test("handleVideoGeneration surfaces a 502 when DashScope returns no task_id", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => jsonResponse({ message: "Invalid API key", request_id: "x" }, 401); + + try { + const result = await handleVideoGeneration({ + body: { model: "alibaba/wan2.7-t2v", prompt: "x" }, + credentials: { apiKey: "bad-key" }, + log: null, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 502); + assert.equal(result.error, "Invalid API key"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleVideoGeneration returns 502 when the DashScope task FAILED", async () => { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.setTimeout; + globalThis.setTimeout = immediateTimeout; + + globalThis.fetch = async (url) => { + const stringUrl = String(url); + if (stringUrl === CREATE_URL) { + return jsonResponse({ output: { task_id: "ds-fail", task_status: "PENDING" } }); + } + if (stringUrl.startsWith(POLL_URL_PREFIX)) { + return jsonResponse({ + output: { task_status: "FAILED", message: "content policy violation" }, + }); + } + throw new Error(`Unexpected URL: ${stringUrl}`); + }; + + try { + const result = await handleVideoGeneration({ + body: { model: "alibaba/wan2.7-t2v", prompt: "x" }, + credentials: { apiKey: "dashscope-key" }, + log: null, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 502); + assert.equal(result.error, "content policy violation"); + } finally { + globalThis.fetch = originalFetch; + globalThis.setTimeout = originalSetTimeout; + } +}); + +test("handleVideoGeneration returns 504 when the DashScope task never completes", async () => { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.setTimeout; + const originalNow = Date.now; + globalThis.setTimeout = immediateTimeout; + + // Deterministic clock: start at 1000, allow exactly one poll iteration, then + // jump past the deadline so the while-loop exits on the next check. + let nowCalls = 0; + Date.now = () => { + nowCalls += 1; + return nowCalls === 1 ? 1000 : nowCalls === 2 ? 2000 : 1_000_000; + }; + + globalThis.fetch = async (url) => { + const stringUrl = String(url); + if (stringUrl === CREATE_URL) { + return jsonResponse({ output: { task_id: "ds-stuck", task_status: "PENDING" } }); + } + if (stringUrl.startsWith(POLL_URL_PREFIX)) { + return jsonResponse({ output: { task_status: "RUNNING" } }); + } + throw new Error(`Unexpected URL: ${stringUrl}`); + }; + + try { + const result = await handleVideoGeneration({ + body: { + model: "alibaba/wan2.7-t2v", + prompt: "x", + timeout_ms: 5000, + poll_interval_ms: 100, + }, + credentials: { apiKey: "dashscope-key" }, + log: null, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 504); + assert.match(result.error, /timed out/); + } finally { + globalThis.fetch = originalFetch; + globalThis.setTimeout = originalSetTimeout; + Date.now = originalNow; + } +});