diff --git a/.env.example b/.env.example index 739e696f84..3d00ae7360 100644 --- a/.env.example +++ b/.env.example @@ -1802,6 +1802,12 @@ APP_LOG_TO_FILE=true # Used by: open-sse/executors/cloudflare-ai.ts # CLOUDFLARE_ACCOUNT_ID= +# ── Cloudflare AI Playground ── +# Full desktop Chrome binary path, used when Playwright's bundled Chromium is +# blocked by the headless fingerprint check. +# Used by: open-sse/executors/cloudflare-playground.ts +# CLOUDFLARE_PLAYGROUND_CHROME_PATH= + # ── Deno Deploy proxy relay (#4643 / 9router#1437) ── # Override the Deno Deploy REST API base used by the proxy-pool relay deployer. # Default: https://api.deno.com/v2 (omit unless mocking). diff --git a/changelog.d/features/10389-cloudflare-playground.md b/changelog.d/features/10389-cloudflare-playground.md new file mode 100644 index 0000000000..fb6bd80c0a --- /dev/null +++ b/changelog.d/features/10389-cloudflare-playground.md @@ -0,0 +1 @@ +- feat(providers): add **Cloudflare AI Playground** as a No Auth provider (`cloudflare-playground`, alias `cfp`) — free anonymous chat over the reverse-engineered `cf_agent` WebSocket protocol (PartySocket transport, no account/API key/cookies) with GLM 5.2, Kimi K2.7 Code, DeepSeek V4 Pro, gpt-oss-120B, Llama 3.3 70B, Qwen2.5 Coder 32B and 14 more curated models. The executor drives a headless Chromium via Playwright (the WS upgrade is TLS-fingerprint-gated), translates the `cf_agent` frame stream into OpenAI SSE, and surfaces upstream rate limits (3021) as HTTP 429. Fixes #10389 diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index ed07710a17..60163731a9 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -968,6 +968,7 @@ Chrome-driven session refresh (ARP) for the Adobe Firefly web provider (`open-ss | `AWS_REGION` | _(unset)_ | `src/lib/providers/validation.ts`, `open-sse/handlers/audioSpeech.ts` | Region used to construct AWS Bedrock endpoints (Kiro, audio). | | `AWS_DEFAULT_REGION` | _(unset)_ | `src/lib/providers/validation.ts`, `open-sse/handlers/audioSpeech.ts` | Fallback when `AWS_REGION` is not set. | | `CLOUDFLARE_ACCOUNT_ID` | _(unset)_ | `open-sse/executors/cloudflare-ai.ts` | Account ID for Cloudflare Workers AI. | +| `CLOUDFLARE_PLAYGROUND_CHROME_PATH` | _(unset)_ | `open-sse/executors/cloudflare-playground.ts` | Full desktop Chrome binary path for the Cloudflare AI Playground executor, used when the headless fingerprint check blocks Playwright's bundled Chromium. | | `CLOUDFLARE_API_BASE` | `https://api.cloudflare.com/client/v4` | `src/app/api/settings/proxy/cloudflare-deploy/route.ts` | Override the Cloudflare REST API base used by the proxy-pool Workers relay deployer (#4640 / 9router#1360). | | `NEXT_PUBLIC_CLOUDFLARE_RELAY_DEFAULT_PROJECT` | `omniroute-relay` | `src/app/(dashboard)/dashboard/settings/components/proxy/CloudflareRelayModal.tsx` | Default worker project name suggested in the proxy-pool "Deploy Relay" modal. | | `NEXT_PUBLIC_CLOUDFLARE_RELAY_ENABLED` | `true` | `src/app/(dashboard)/dashboard/settings/components/proxy/ProxyPoolTab.tsx` | Set to `false` to hide the Cloudflare Workers relay option from the Proxy Pool tab. | diff --git a/open-sse/config/imageRegistry.ts b/open-sse/config/imageRegistry.ts index d7b0150849..3e24139e85 100644 --- a/open-sse/config/imageRegistry.ts +++ b/open-sse/config/imageRegistry.ts @@ -248,6 +248,26 @@ export const IMAGE_PROVIDERS: Record = { supportedSizes: ["1024x1024", "1024x1536", "1536x1024"], }, + // #10466: Gemini Web session image generation (Nano Banana). Same + // web-cookie transport as the gemini-web chat provider — the handler + // drives the session executor in image mode and extracts the generated + // asset URLs from the StreamGenerate frames. + "gemini-web": { + id: "gemini-web", + alias: "gweb", + baseUrl: "https://gemini.google.com/app", + authType: "apikey", + authHeader: "cookie", + format: "gemini-web", + // `-web` suffix on purpose: the bare `nano-banana` id is owned by + // adobe-firefly (operator decision 2026-07-31, pinned by the + // cheaperinference-image-models guard). parseImageModel's bare-model scan + // walks providers in insertion order, so a bare `nano-banana` here would + // steal that resolution. Keep this id distinct. + models: [{ id: "nano-banana-web", name: "Nano Banana (Gemini Web Image)" }], + supportedSizes: ["1024x1024", "1024x1536", "1536x1024"], + }, + "microsoft-designer-web": { id: "microsoft-designer-web", alias: "msdesigner", diff --git a/open-sse/config/providers/index.ts b/open-sse/config/providers/index.ts index d38c6e6de2..01395e7dd8 100644 --- a/open-sse/config/providers/index.ts +++ b/open-sse/config/providers/index.ts @@ -31,6 +31,7 @@ import { difyProvider } from "./registry/dify/index.ts"; import { ovhcloudProvider } from "./registry/ovhcloud/index.ts"; import { claudeProvider } from "./registry/claude/index.ts"; import { claude_webProvider } from "./registry/claude/web/index.ts"; +import { cloudflarePlaygroundProvider } from "./registry/cloudflare-playground/index.ts"; import { bedrockProvider } from "./registry/bedrock/index.ts"; import { inner_aiProvider } from "./registry/inner-ai/index.ts"; import { qoderProvider } from "./registry/qoder/index.ts"; @@ -290,6 +291,7 @@ export const REGISTRY: Record = { ovhcloud: ovhcloudProvider, claude: claudeProvider, "claude-web": claude_webProvider, + "cloudflare-playground": cloudflarePlaygroundProvider, bedrock: bedrockProvider, "inner-ai": inner_aiProvider, qoder: qoderProvider, diff --git a/open-sse/config/providers/registry/cloudflare-playground/index.ts b/open-sse/config/providers/registry/cloudflare-playground/index.ts new file mode 100644 index 0000000000..1c369d6979 --- /dev/null +++ b/open-sse/config/providers/registry/cloudflare-playground/index.ts @@ -0,0 +1,57 @@ +/** + * Cloudflare AI Playground — No Auth provider registry entry. + * + * Free, anonymous access to the Cloudflare AI Playground + * (https://playground.ai.cloudflare.com) — no account, no API key, no cookies. + * Chat runs over a PartySocket WebSocket speaking Cloudflare's `cf_agent` + * protocol; the only gate is a browser-grade TLS fingerprint on the WS upgrade, + * which the `cloudflare-playground` executor satisfies by driving a headless + * Chromium via Playwright (see executors/cloudflare-playground.ts). + * + * Model catalog captured from the playground's live `getModels` RPC + * (2026-08-15, 63 models total; the 20 chat/text-generation entries are listed + * here). Model IDs use the playground's `org/model` slug form — the executor + * prefixes them with `@cf/` when talking to the upstream. + */ +import type { RegistryEntry } from "../../shared.ts"; + +export const cloudflarePlaygroundProvider: RegistryEntry = { + id: "cloudflare-playground", + alias: "cfp", + format: "openai", + executor: "cloudflare-playground", + baseUrl: "https://playground.ai.cloudflare.com", + authType: "none", + authHeader: "none", + models: [ + // Frontier/open-weight flagships first. + { id: "zai-org/glm-5.2", name: "GLM 5.2 (Z.ai)", supportsReasoning: true }, + { id: "moonshotai/kimi-k2.7-code", name: "Kimi K2.7 Code (Moonshot)", supportsReasoning: true }, + { id: "moonshotai/kimi-k2.6", name: "Kimi K2.6 (Moonshot)", supportsReasoning: true }, + { + id: "deepseek-ai/deepseek-v4-pro-0813", + name: "DeepSeek V4 Pro (DeepSeek)", + supportsReasoning: true, + }, + { id: "deepseek-ai/deepseek-v4-flash-0731", name: "DeepSeek V4 Flash (DeepSeek)" }, + { id: "zai-org/glm-4.7-flash", name: "GLM 4.7 Flash (Z.ai)", supportsReasoning: true }, + { id: "openai/gpt-oss-120b", name: "GPT-OSS 120B (OpenAI)" }, + { id: "openai/gpt-oss-20b", name: "GPT-OSS 20B (OpenAI)" }, + { id: "meta-llama/llama-3.3-70b-instruct-fp8-fast", name: "Llama 3.3 70B Instruct (Meta)" }, + { id: "meta/llama-3.1-8b-instruct-fp8", name: "Llama 3.1 8B Instruct (Meta)" }, + { id: "meta/llama-4-scout-17b-16e-instruct", name: "Llama 4 Scout 17B (Meta)" }, + { id: "nvidia/nemotron-3-120b-a12b", name: "Nemotron 3 120B (NVIDIA)" }, + { id: "qwen/qwen2.5-coder-32b-instruct", name: "Qwen2.5 Coder 32B (Qwen)" }, + { id: "qwen/qwen3-30b-a3b-fp8", name: "Qwen3 30B A3B (Qwen)" }, + { id: "qwen/qwq-32b", name: "QwQ 32B (Qwen)", supportsReasoning: true }, + { + id: "deepseek-ai/deepseek-r1-distill-qwen-32b", + name: "DeepSeek R1 Distill Qwen 32B", + supportsReasoning: true, + }, + { id: "google/gemma-4-26b-a4b-it", name: "Gemma 4 26B A4B (Google)" }, + { id: "mistralai/mistral-small-3.1-24b-instruct", name: "Mistral Small 3.1 24B" }, + { id: "ibm-granite/granite-4.0-h-micro", name: "Granite 4.0 H Micro (IBM)" }, + { id: "aisingapore/gemma-sea-lion-v4-27b-it", name: "Gemma SEA-LION V4 27B (AI Singapore)" }, + ], +}; diff --git a/open-sse/executors/cloudflare-playground.ts b/open-sse/executors/cloudflare-playground.ts new file mode 100644 index 0000000000..ba309f1eed --- /dev/null +++ b/open-sse/executors/cloudflare-playground.ts @@ -0,0 +1,591 @@ +/** + * CloudflarePlaygroundExecutor — Cloudflare AI Playground (No Auth) provider + * + * Reverse-engineered access to the free, anonymous Cloudflare AI Playground + * (https://playground.ai.cloudflare.com). No account, no API key, no cookies: + * chat runs over a PartySocket WebSocket speaking Cloudflare's `cf_agent` RPC + * protocol, and the only gate is a browser-grade TLS fingerprint on the WS + * upgrade. This executor therefore drives a headless Chromium via Playwright, + * opens the WebSocket *inside the page context* (only a real browser TLS stack + * passes the upgrade), and translates the `cf_agent` frame stream into + * OpenAI-format chat completion chunks. + * + * Protocol (captured live 2026-08-15): + * - Transport: wss://playground.ai.cloudflare.com/agents/playground/?_pk= + * - Resume: {"type":"cf_agent_stream_resume_request"} + * - Config: {"type":"rpc","method":"setConfig","args":[{model,temperature,stream}]} + * - Chat: {"id":,"init":{"method":"POST","body":{messages,trigger}},"type":"cf_agent_use_chat_request"} + * - Stream: start → start-step → (reasoning-start/delta/end)* → text-start → + * text-delta* → finish-step → finish{messageMetadata.finishReason} → {done:true} + * - Errors: {"error":true,"body":"{message,details}","id":} — e.g. + * "3021: rate limiting: inference request per min rate reached" + * + * Notes: + * - The playground's system prompt is server-side (set via setConfig by the + * app itself); client `system` messages are dropped. Tool calls are not + * implemented (v1) — text-only chat. + * - Upstream rate limits arrive in-band as `error:true` frames. Non-streaming + * requests surface them as HTTP 429/502; streaming requests emit an SSE + * error chunk before `[DONE]` (the response status is already committed). + * A server-side chat timeout follows the same rule: streaming requests + * emit a `timeout_error` chunk before `[DONE]` instead of silently + * completing (#10494). + * - Set CLOUDFLARE_PLAYGROUND_CHROME_PATH to point at a full desktop Chrome + * binary when Playwright's bundled Chromium gets fingerprint-blocked. + */ +import { randomUUID } from "crypto"; +import { BaseExecutor, type ExecuteInput } from "./base.ts"; +import { makeExecutorErrorResult as makeErrorResult } from "../utils/error.ts"; +import type { Browser, Page } from "playwright"; + +export const PLAYGROUND_URL = "https://playground.ai.cloudflare.com/"; +const PLAYGROUND_WS_BASE = "wss://playground.ai.cloudflare.com/agents/playground/"; +const PLAYGROUND_UA = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"; +const BROWSER_ARGS = [ + "--disable-blink-features=AutomationControlled", + "--no-first-run", + "--no-default-browser-check", +]; +const MODEL_PREFIX = "@cf/"; +const DEFAULT_MODEL = "zai-org/glm-4.7-flash"; +const DEFAULT_TEMPERATURE = 0.7; +const NAV_TIMEOUT_MS = 45_000; +const CHAT_TIMEOUT_MS = 120_000; +const BLOCKED_MESSAGE = + "Cloudflare Playground blocked the headless browser (fingerprint check). Set CLOUDFLARE_PLAYGROUND_CHROME_PATH to a full desktop Chrome binary and retry."; + +// ── Frame parsing & translation (pure — unit-tested against live captures) ── + +export interface CfChatFrame { + id?: string; + type?: string; + error?: boolean; + done?: boolean; + body?: unknown; +} + +/** Parse a raw WS frame. Returns null for non-JSON / unrelated frames. */ +export function parseCfFrame(raw: string): CfChatFrame | null { + try { + const msg = JSON.parse(raw) as CfChatFrame; + if (msg && typeof msg === "object" && typeof msg.type === "string") return msg; + } catch { + /* non-JSON — ignore */ + } + return null; +} + +export interface CfStreamEvent { + type: "role" | "content" | "reasoning" | "finish"; + value?: string; +} + +/** + * Translates `cf_agent_use_chat_response` frames for one chat id into + * OpenAI-format stream events. Frames for other ids (RPC responses such as + * `setConfig` also carry `done:true`!) and non-chat frame types + * (`cf_agent_identity`, `cf_agent_state`, ...) are ignored. + */ +export class CfStreamParser { + readonly chatId: string; + done = false; + text = ""; + reasoningText = ""; + finishReason: string | null = null; + error: { status: number; message: string } | null = null; + private seenStart = false; + + constructor(chatId: string) { + this.chatId = chatId; + } + + /** Returns the SSE-relevant event, or null when the frame is ignorable. */ + push(raw: string): CfStreamEvent | null { + const msg = parseCfFrame(raw); + if (!msg || msg.type !== "cf_agent_use_chat_response" || msg.id !== this.chatId) return null; + + if (msg.error) { + this.error = classifyError(msg.body); + return null; + } + if (msg.done) { + this.done = true; + return null; + } + + let body: Record; + try { + body = + typeof msg.body === "string" + ? (JSON.parse(msg.body) as Record) + : (msg.body as Record); + } catch { + return null; + } + if (!body || typeof body.type !== "string") return null; + + switch (body.type) { + case "start": + if (this.seenStart) return null; + this.seenStart = true; + return { type: "role" }; + case "reasoning-delta": { + const delta = typeof body.delta === "string" ? body.delta : ""; + if (!delta) return null; + this.reasoningText += delta; + return { type: "reasoning", value: delta }; + } + case "text-delta": { + const delta = typeof body.delta === "string" ? body.delta : ""; + if (!delta) return null; + this.text += delta; + return { type: "content", value: delta }; + } + case "finish": { + const meta = (body.messageMetadata ?? {}) as Record; + const reason = typeof meta.finishReason === "string" ? meta.finishReason : "stop"; + this.finishReason = reason; + return { type: "finish", value: reason }; + } + default: + // reasoning-start/end, start-step, finish-step, text-start/end, heartbeat — ignored. + return null; + } + } +} + +/** Map an in-band upstream error frame to an HTTP-ish status + clean message. */ +function classifyError(body: unknown): { status: number; message: string } { + let detail = ""; + if (typeof body === "string") { + try { + const parsed = JSON.parse(body) as Record; + detail = String(parsed.details || parsed.message || ""); + } catch { + detail = body; + } + } else if (body && typeof body === "object") { + const parsed = body as Record; + detail = String(parsed.details || parsed.message || ""); + } + const status = /rate|limit|quota|throttl/i.test(detail) ? 429 : 502; + return { status, message: detail || "Cloudflare Playground upstream error" }; +} + +// ── Message conversion ─────────────────────────────────────────────────────── + +export interface CfChatMessage { + role: "user" | "assistant"; + parts: Array<{ type: "text"; text: string }>; + id: string; +} + +/** + * Convert OpenAI-format messages to the playground's chat body shape. + * `system` messages are dropped (the playground's persona is server-side) and + * tool/image parts are flattened to text — v1 is text-only chat. + */ +export function toCfMessages( + messages: Array<{ role?: string; content?: unknown }> +): CfChatMessage[] { + const out: CfChatMessage[] = []; + for (const message of messages ?? []) { + if (message.role !== "user" && message.role !== "assistant") continue; + let text = ""; + if (typeof message.content === "string") { + text = message.content; + } else if (Array.isArray(message.content)) { + text = message.content + .map((part) => + typeof part === "string" ? part : ((part as { text?: string })?.text ?? "") + ) + .filter(Boolean) + .join("\n"); + } + if (!text) continue; + out.push({ role: message.role, parts: [{ type: "text", text }], id: `m${out.length + 1}` }); + } + return out; +} + +// ── Transport ──────────────────────────────────────────────────────────────── + +export interface CfTransportConfig { + model: string; + messages: CfChatMessage[]; + temperature: number; + signal?: AbortSignal | null; +} + +export interface CfTransport { + start( + config: CfTransportConfig + ): Promise<{ ok: true } | { ok: false; status: number; message: string }>; + frames(): AsyncGenerator; + close(): Promise; +} + +/** Open the anonymous playground session inside the browser page context. */ +function openPlaygroundSession(args: { + chatId: string; + model: string; + messages: CfChatMessage[]; + temperature: number; + wsBase: string; +}): void { + const { chatId, model, messages, temperature, wsBase } = args; + const pk = crypto.randomUUID(); + const room = "playground-" + crypto.randomUUID().replace(/-/g, "").slice(0, 25); + const socket = new WebSocket(wsBase + room + "?_pk=" + pk); + const push = (raw: string) => { + try { + (window as unknown as { __cfpPush: (raw: string) => void }).__cfpPush(raw); + } catch { + /* page torn down */ + } + }; + socket.onopen = () => { + socket.send(JSON.stringify({ type: "cf_agent_stream_resume_request" })); + socket.send( + JSON.stringify({ + type: "rpc", + id: "cfp-config", + method: "setConfig", + args: [{ model, temperature, stream: true }], + }) + ); + socket.send( + JSON.stringify({ + id: chatId, + init: { method: "POST", body: JSON.stringify({ messages, trigger: "submit-message" }) }, + type: "cf_agent_use_chat_request", + }) + ); + }; + socket.onmessage = (event: MessageEvent) => push(String(event.data)); + socket.onerror = () => + push( + JSON.stringify({ + id: chatId, + type: "cf_agent_use_chat_response", + error: true, + body: JSON.stringify({ + message: "Playground WebSocket error", + details: "ws transport failed", + }), + }) + ); +} + +export class PlaywrightCfTransport implements CfTransport { + private browser: Browser | null = null; + private page: Page | null = null; + private pending: string[] = []; + private waiters: Array<(frame: string | null) => void> = []; + private closed = false; + private abortSignal: AbortSignal | null = null; + private abortListener: (() => void) | null = null; + + constructor( + private chatId: string, + private chromeExecutablePath?: string + ) {} + + async start( + config: CfTransportConfig + ): Promise<{ ok: true } | { ok: false; status: number; message: string }> { + try { + const playwright = await importPlaywright(); + const executablePath = + this.chromeExecutablePath ?? process.env.CLOUDFLARE_PLAYGROUND_CHROME_PATH; + this.browser = await playwright.chromium.launch({ + ...(executablePath ? { executablePath } : {}), + headless: true, + args: BROWSER_ARGS, + }); + const context = await this.browser.newContext({ userAgent: PLAYGROUND_UA }); + const page = await context.newPage(); + this.page = page; + await page.goto(PLAYGROUND_URL, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }); + const title = await page.title().catch(() => ""); + if (title.includes("Attention Required")) { + // #10494: this branch used to return without closing the browser it + // just launched, leaking a Chromium process for every blocked + // request. Close it on every non-success start path, same as the + // catch block below. + await this.close().catch(() => {}); + return { ok: false, status: 502, message: BLOCKED_MESSAGE }; + } + await page.exposeFunction("__cfpPush", (raw: string) => { + this.push(raw); + }); + // Bundlers (esbuild/webpack keepNames) inject a `__name` helper call into + // serialized function bodies; define it in the page context so + // page.evaluate(openPlaygroundSession) doesn't throw ReferenceError. + await page.evaluate(() => { + (window as unknown as { __name?: unknown }).__name = (fn: unknown) => fn; + }); + await page.evaluate(openPlaygroundSession, { + ...config, + chatId: this.chatId, + wsBase: PLAYGROUND_WS_BASE, + }); + if (config.signal) { + this.abortSignal = config.signal; + this.abortListener = () => { + void this.close(); + }; + config.signal.addEventListener("abort", this.abortListener, { once: true }); + } + return { ok: true }; + } catch (error) { + await this.close().catch(() => {}); + return { + ok: false, + status: 502, + message: `Cloudflare Playground browser session failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + + push(raw: string): void { + const waiter = this.waiters.shift(); + if (waiter) waiter(raw); + else this.pending.push(raw); + } + + async *frames(): AsyncGenerator { + while (this.pending.length > 0 || !this.closed) { + if (this.pending.length > 0) { + yield this.pending.shift()!; + continue; + } + const frame = await new Promise((resolve) => this.waiters.push(resolve)); + if (frame === null) return; + yield frame; + } + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + if (this.abortSignal && this.abortListener) { + this.abortSignal.removeEventListener("abort", this.abortListener); + } + this.abortSignal = null; + this.abortListener = null; + for (const waiter of this.waiters.splice(0)) waiter(null); + const browser = this.browser; + this.browser = null; + if (browser) await browser.close().catch(() => {}); + } +} + +async function importPlaywright(): Promise { + try { + return await import("playwright"); + } catch { + throw new Error( + "Playwright is not available. Install it (npm i playwright && npx playwright install chromium) or set CLOUDFLARE_PLAYGROUND_CHROME_PATH to a Chrome binary." + ); + } +} + +// ── Executor ───────────────────────────────────────────────────────────────── + +function sseChunk( + cid: string, + created: number, + model: string, + payload: { delta?: Record; finish_reason?: string | null; error?: unknown } +): string { + const base = { id: cid, object: "chat.completion.chunk", created, model }; + if (payload.error) { + return `data: ${JSON.stringify({ ...base, error: payload.error })}\n\n`; + } + return `data: ${JSON.stringify({ + ...base, + choices: [ + { index: 0, delta: payload.delta ?? {}, finish_reason: payload.finish_reason ?? null }, + ], + })}\n\n`; +} + +export class CloudflarePlaygroundExecutor extends BaseExecutor { + constructor( + private transportFactory: (chatId: string) => CfTransport = (chatId) => + new PlaywrightCfTransport(chatId), + // Injectable so tests can force the timeout branch without waiting + // CHAT_TIMEOUT_MS (120s) for a real timer to fire. + private chatTimeoutMs: number = CHAT_TIMEOUT_MS + ) { + super("cloudflare-playground", { id: "cloudflare-playground", baseUrl: PLAYGROUND_URL }); + } + + async execute(input: ExecuteInput) { + const { body, signal, stream: wantStream } = input; + const bodyObj = (body || {}) as Record; + const rawModel = (bodyObj.model as string) || DEFAULT_MODEL; + const model = rawModel.startsWith(MODEL_PREFIX) ? rawModel : MODEL_PREFIX + rawModel; + const temperature = + typeof bodyObj.temperature === "number" ? bodyObj.temperature : DEFAULT_TEMPERATURE; + const chatId = `chatcmpl-cfp-${randomUUID().slice(0, 12)}`; + const created = Math.floor(Date.now() / 1000); + + const transport = this.transportFactory(chatId); + const started = await transport.start({ + model, + messages: toCfMessages( + (bodyObj.messages as Array<{ role?: string; content?: unknown }>) || [] + ), + temperature, + signal, + }); + if (started.ok !== true) { + return makeErrorResult(started.status, started.message, body, PLAYGROUND_URL); + } + + const timedOut = { current: false }; + const timer = setTimeout(() => { + timedOut.current = true; + void transport.close(); + }, this.chatTimeoutMs); + + try { + if (!wantStream) { + const parser = new CfStreamParser(chatId); + for await (const raw of transport.frames()) { + parser.push(raw); + if (parser.error || parser.done) break; + } + if (parser.error) { + return makeErrorResult(parser.error.status, parser.error.message, body, PLAYGROUND_URL); + } + if (timedOut.current && !parser.text) { + return makeErrorResult(504, "Cloudflare Playground timed out", body, PLAYGROUND_URL); + } + const text = parser.text; + const messagePayload: Record = { role: "assistant", content: text }; + if (parser.reasoningText) { + messagePayload.reasoning_content = parser.reasoningText; + } + return { + response: new Response( + JSON.stringify({ + id: chatId, + object: "chat.completion", + created, + model: rawModel, + choices: [ + { + index: 0, + message: messagePayload, + finish_reason: parser.finishReason ?? "stop", + }, + ], + usage: { + prompt_tokens: 0, + completion_tokens: Math.ceil((text.length + parser.reasoningText.length) / 4), + total_tokens: 0, + }, + }), + { headers: { "Content-Type": "application/json" } } + ), + url: PLAYGROUND_URL, + headers: {}, + transformedBody: body, + }; + } + + // Streaming: translate cf_agent frames → OpenAI SSE chunks. + const encoder = new TextEncoder(); + const responseStream = new ReadableStream({ + async start(controller) { + const parser = new CfStreamParser(chatId); + let roleSent = false; + const enqueue = (payload: { + delta?: Record; + finish_reason?: string | null; + error?: unknown; + }) => { + controller.enqueue(encoder.encode(sseChunk(chatId, created, rawModel, payload))); + }; + try { + for await (const raw of transport.frames()) { + if (signal?.aborted) break; + const event = parser.push(raw); + if (event) { + if (event.type === "role" && !roleSent) { + enqueue({ delta: { role: "assistant" }, finish_reason: null }); + roleSent = true; + } else if (event.type === "reasoning") { + enqueue({ delta: { reasoning_content: event.value }, finish_reason: null }); + } else if (event.type === "content") { + enqueue({ delta: { content: event.value }, finish_reason: null }); + } else if (event.type === "finish") { + enqueue({ delta: {}, finish_reason: event.value ?? "stop" }); + } + } + if (parser.error) { + enqueue({ + error: { + message: parser.error.message, + type: "upstream_error", + code: `HTTP_${parser.error.status}`, + }, + }); + break; + } + if (parser.done || timedOut.current) break; + } + } catch (error) { + if (!signal?.aborted) controller.error(error); + } finally { + clearTimeout(timer); + await transport.close().catch(() => {}); + // #10494: a timeout used to fall straight through to a bare + // [DONE], so a client receiving an empty or partial stream saw + // an ordinary successful completion. Emit an explicit error + // chunk first (same shape as the parser.error branch above) so + // the client can distinguish a timed-out/partial answer from a + // real completion. + if (timedOut.current) { + try { + enqueue({ + error: { + message: "Cloudflare Playground timed out", + type: "timeout_error", + code: "HTTP_504", + }, + }); + } catch { + /* stream already torn down */ + } + } + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + controller.close(); + } + }, + }); + + return { + response: new Response(responseStream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }), + url: PLAYGROUND_URL, + headers: {}, + transformedBody: body, + }; + } finally { + if (!wantStream) { + clearTimeout(timer); + await transport.close().catch(() => {}); + } + } + } +} diff --git a/open-sse/executors/gemini-web.ts b/open-sse/executors/gemini-web.ts index 8810b43cc3..3ae6df79cd 100644 --- a/open-sse/executors/gemini-web.ts +++ b/open-sse/executors/gemini-web.ts @@ -260,6 +260,70 @@ export function parseStreamResponse(raw: string): string { return lastText; } +/** + * Extract generated-image URLs from a Gemini StreamGenerate response (#10466). + * + * When the web UI generates images (Nano Banana), the model's answer frames + * carry the assets in the candidate's extension block, NOT in the text: + * + * inner[4][0][12][7][0] → array of generated-image entries + * entry[0][3][3] → the image URL — either a plain string or a + * list of strings (take the first http(s) one) + * + * This path is corroborated by the two maintained reverse-engineered clients + * (gpt4free's Gemini provider and HanaokaYuzu/Gemini-API's _parse_candidate). + * Deliberately NOT collected: `inner[4][0][12][1]` — those are web-search + * result thumbnails, not generated content; mixing them in would serve + * scraped images as "generated" (#10466 acceptance criteria). + * + * Frames are cumulative snapshots, so later frames repeat earlier images; + * we dedupe while preserving first-seen order. A `=s2048` size suffix is + * appended (gpt4free's proven heuristic) so callers get full-resolution + * assets instead of UI thumbnails. + */ +export function parseStreamResponseImages(raw: string): string[] { + const urls: string[] = []; + const seen = new Set(); + const lines = raw.split("\n"); + + for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line || line === ")]}'" || /^\d+$/.test(line)) continue; + if (!line.includes("wrb.fr")) continue; + try { + const arr = JSON.parse(line); + if (!Array.isArray(arr) || !Array.isArray(arr[0]) || arr[0][0] !== "wrb.fr") continue; + const payload = arr[0]?.[2]; + if (typeof payload !== "string") continue; + const inner = JSON.parse(payload); + const imageEntries = inner?.[4]?.[0]?.[12]?.[7]?.[0]; + if (!Array.isArray(imageEntries)) continue; + for (const entry of imageEntries) { + const urlField = entry?.[0]?.[3]?.[3]; + let url = ""; + if (typeof urlField === "string") { + url = urlField; + } else if (Array.isArray(urlField)) { + const firstHttp = urlField.find( + (u: unknown) => typeof u === "string" && /^https?:\/\//.test(u) + ); + url = typeof firstHttp === "string" ? firstHttp : ""; + } + if (!url || !/^https?:\/\//.test(url)) continue; + // Upgrade to full resolution unless a size directive is already present + // (googleusercontent size syntax: trailing `=s2048`, `=w1024-h512`, ...). + if (!/=[swh]\d+/.test(url)) url += "=s2048"; + if (seen.has(url)) continue; + seen.add(url); + urls.push(url); + } + } catch { + // Skip unparseable lines + } + } + return urls; +} + function readCredentialString(value: unknown): string { if (typeof value !== "string") return ""; const trimmed = value.trim(); @@ -365,9 +429,7 @@ export class GeminiWebExecutor extends BaseExecutor { _signal?: AbortSignal ): Promise { try { - const cookie = resolveGeminiWebCookie( - credentials as unknown as ExecuteInput["credentials"] - ); + const cookie = resolveGeminiWebCookie(credentials as unknown as ExecuteInput["credentials"]); if (!cookie) return false; const pairs = parseCookies(cookie); return pairs.some((p) => p.value.length > 0); @@ -506,20 +568,52 @@ export class GeminiWebExecutor extends BaseExecutor { const page = await context.newPage(); + // #10466: image mode — the /v1/images/generations handler sets + // x_gemini_web_image_mode. Generated images arrive in the candidate's + // extension block ([12][7][0]) of the StreamGenerate frames, sometimes + // only in a LATER frame of the stream (or a follow-up StreamGenerate + // call), so image mode captures every StreamGenerate response, merges + // image URLs across frames, and resolves as soon as one is found. + // Chat mode keeps the original first-response-only behavior. + const imageMode = (body as Record)?.x_gemini_web_image_mode === true; + // Capture first StreamGenerate response let responseText = ""; + const responseImages: string[] = []; let captured = false; const responsePromise = new Promise((resolve) => { page.on("response", async (resp: any) => { - if (captured || !resp.url().includes("StreamGenerate")) return; - captured = true; - try { - const raw = await resp.text(); - responseText = parseStreamResponse(raw); - } catch { - /* ignore */ + if (!resp.url().includes("StreamGenerate")) return; + if (!imageMode && captured) return; + if (imageMode) { + // Image mode: merge text + image URLs across every frame and + // resolve as soon as an image appears (images can land in a + // later frame than the text). + try { + const raw = await resp.text(); + const text = parseStreamResponse(raw); + if (text) responseText = text; + for (const url of parseStreamResponseImages(raw)) { + if (!responseImages.includes(url)) responseImages.push(url); + } + } catch { + /* ignore unreadable frames */ + } + if (responseImages.length > 0) resolve(); + } else { + // Chat mode: byte-for-byte the original first-response capture — + // resolve even if reading the body throws, so the flow falls + // through to the "No response from Gemini" 502 instead of + // burning the full wait window. + captured = true; + try { + const raw = await resp.text(); + responseText = parseStreamResponse(raw); + } catch { + /* ignore */ + } + resolve(); } - resolve(); }); }); @@ -538,12 +632,36 @@ export class GeminiWebExecutor extends BaseExecutor { await page.waitForTimeout(300); await page.keyboard.press("Enter"); - // Wait for response or timeout - await Promise.race([responsePromise, page.waitForTimeout(30000)]); + // Wait for response or timeout. Image generation (Nano Banana) is + // noticeably slower than text — the UI renders the asset only after + // the full generation completes — so image mode gets a wider window. + await Promise.race([responsePromise, page.waitForTimeout(imageMode ? 90000 : 30000)]); if (signal?.aborted) { throw signal.reason instanceof Error ? signal.reason : new Error("Request aborted"); } + // #10466 image mode: return the captured image URLs to the image + // handler via a custom field (same precedent as chatgpt-web's + // x_image_resolution_failed). An image-only answer can carry little or + // no text, so the empty-text 502 below must not fire when images + // were captured. + if (imageMode) { + await this.persistRotatedCookies(context, cookie, credentials, onCredentialsRefreshed, log); + const modelId = model || "gemini-2.5-pro"; + return { + response: new Response( + JSON.stringify({ + ...formatChatCompletion(responseText, modelId), + x_gemini_web_image_urls: responseImages, + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ), + url: GEMINI_URL, + headers: {}, + transformedBody: body, + }; + } + if (!responseText) { return { response: new Response(JSON.stringify({ error: "No response from Gemini" }), { diff --git a/open-sse/executors/index.ts b/open-sse/executors/index.ts index faba4f2937..a780e1c8b5 100644 --- a/open-sse/executors/index.ts +++ b/open-sse/executors/index.ts @@ -71,6 +71,7 @@ import { LMArenaExecutor } from "./lmarena.ts"; import { GrokCliExecutor } from "./grok-cli.ts"; import { CodeBuddyCnExecutor } from "./codebuddy-cn.ts"; import { ZenmuxFreeExecutor } from "./zenmux-free.ts"; +import { CloudflarePlaygroundExecutor } from "./cloudflare-playground.ts"; import { TinyCmsExecutor } from "./tinycms.ts"; import { HyperAgentExecutor } from "./hyperagent.ts"; import { XaiExecutor } from "./xai.ts"; @@ -204,6 +205,8 @@ const executors = { "codebuddy-cn": new CodeBuddyCnExecutor(), cbcn: new CodeBuddyCnExecutor(), // Alias for codebuddy-cn "zenmux-free": new ZenmuxFreeExecutor(), + "cloudflare-playground": new CloudflarePlaygroundExecutor(), + cfp: new CloudflarePlaygroundExecutor(), // Alias for cloudflare-playground "tinycms-web": new TinyCmsExecutor(), tcw: new TinyCmsExecutor(), // Alias hyperagent: new HyperAgentExecutor(), @@ -323,6 +326,7 @@ export { LMArenaExecutor } from "./lmarena.ts"; export { GrokCliExecutor } from "./grok-cli.ts"; export { CodeBuddyCnExecutor } from "./codebuddy-cn.ts"; export { ZenmuxFreeExecutor } from "./zenmux-free.ts"; +export { CloudflarePlaygroundExecutor } from "./cloudflare-playground.ts"; export { TinyCmsExecutor } from "./tinycms.ts"; export { HyperAgentExecutor } from "./hyperagent.ts"; export { XaiExecutor } from "./xai.ts"; diff --git a/open-sse/handlers/imageGeneration.ts b/open-sse/handlers/imageGeneration.ts index 436871ae92..5d76987286 100644 --- a/open-sse/handlers/imageGeneration.ts +++ b/open-sse/handlers/imageGeneration.ts @@ -51,6 +51,7 @@ import { extractMarkdownImageUrls, CHATGPT_WEB_IMAGE_ID_RE, } from "./imageGeneration/providers/chatgptWeb.ts"; +import { handleGeminiWebImageGeneration } from "./imageGeneration/providers/geminiWeb.ts"; import { handleNvidiaNimImageGeneration } from "./imageGeneration/providers/nvidiaNim.ts"; import { handleSegmindImageGeneration } from "./imageGeneration/providers/segmind.ts"; import { handleDesignerWebImageGeneration } from "./imageGeneration/providers/designerWeb.ts"; @@ -493,6 +494,19 @@ export async function handleImageGeneration({ }); } + // #10466: Gemini Web session image generation (Nano Banana) + if (providerConfig.format === "gemini-web") { + return handleGeminiWebImageGeneration({ + model, + provider, + body, + credentials, + log, + signal, + clientHeaders, + }); + } + if (providerConfig.format === "designer-web") { return handleDesignerWebImageGeneration({ model, @@ -2683,6 +2697,22 @@ export function saveImageErrorResult({ error, requestBody = null, path = "/v1/images/generations", + // #10494: opt-in signal for executeImageWithCredentialFallback — set by a + // provider handler when the failure is account/session-specific (expired + // or blocked credentials) rather than a generic request/provider error, so + // the retry loop tries the next eligible account even when the upstream + // status isn't a plain 401. Defaults to unset (existing 401-only behavior + // for every other provider is unchanged). + retryable = undefined, +}: { + provider: string; + model: string; + status: number; + startTime: number; + error: unknown; + requestBody?: unknown; + path?: string; + retryable?: boolean; }) { saveCallLog({ method: "POST", @@ -2699,6 +2729,7 @@ export function saveImageErrorResult({ success: false, status, error, + ...(retryable !== undefined ? { retryable } : {}), }; } diff --git a/open-sse/handlers/imageGeneration/providers/geminiWeb.ts b/open-sse/handlers/imageGeneration/providers/geminiWeb.ts new file mode 100644 index 0000000000..43c91de8fc --- /dev/null +++ b/open-sse/handlers/imageGeneration/providers/geminiWeb.ts @@ -0,0 +1,228 @@ +// Gemini Web image generation handler (#10466). +// +// Exposes the gemini-web session provider through POST /v1/images/generations. +// Follows the chatgpt-web precedent (./chatgptWeb.ts): the web-session chat +// executor is driven with an image-generation prompt, and the generated +// assets are extracted from the response. +// +// Transport: GeminiWebExecutor in image mode (x_gemini_web_image_mode). The +// executor types the prompt into gemini.google.com, captures every +// StreamGenerate frame, and returns generated-image URLs in the custom +// `x_gemini_web_image_urls` field. URLs point at lh3.googleusercontent.com +// with a `=s2048` full-resolution size directive; they are public (no +// cookies needed to fetch them). +// +// Prompting: the web UI only GENERATES images when the prompt uses a +// generation verb ("generate"/"create"/"draw"); otherwise it answers with +// web-search thumbnails. The prompt builder therefore always leads with an +// explicit generation directive (corroborated by gemini-webapi's docs). + +import { GeminiWebExecutor } from "../../../executors/gemini-web.ts"; +import { fetchRemoteImage } from "@/shared/network/remoteImageFetch"; +import { saveImageErrorResult, saveImageSuccessResult } from "../../imageGeneration.ts"; + +/** Each image is one gemini.google.com turn (~30-60s). Cap like chatgpt-web. */ +const GEMINI_WEB_IMAGE_N_MAX = 4; + +export function buildGeminiWebImagePrompt(body: Record): string { + const prompt = String(body.prompt || "").trim(); + const details: string[] = [ + `Generate an image for this prompt: ${prompt}`, + "Use the image generation model. Do not search the web for existing images.", + ]; + if (typeof body.size === "string" && body.size.trim()) { + details.push(`Requested aspect/size: ${body.size.trim()}.`); + } + if (typeof body.style === "string" && body.style.trim()) { + details.push(`Requested style: ${body.style.trim()}.`); + } + return details.join("\n"); +} + +/** + * #10494: the underlying GeminiWebExecutor's browser-automation catch paths + * classify an expired/blocked Gemini Web session as HTTP 400 ("the session + * is so expired it lands on a different page" — see gemini-web.ts's + * Playwright selector/click-timeout branch, #9407) or HTTP 500 (its generic + * automation-failure catch-all, which covers a blocked/CAPTCHA/login page + * this handler has no further way to inspect). Both statuses previously + * passed straight through to executeImageWithCredentialFallback, which only + * advances to another account on a plain 401 — so an expired/blocked + * session never triggered account fallback, contrary to #10466's + * acceptance criteria ("Expired or blocked sessions ... can fall back + * normally inside an image Combo"). HTTP 503 (missing Playwright browser — + * a host/config problem, not a per-account issue) is intentionally excluded, + * as is the local 401 this handler already returns before any account is + * selected (missing session cookie — handled by the 401 path already). + */ +export function isExpiredOrBlockedGeminiWebSession(status: number): boolean { + return status === 400 || status === 500; +} + +export async function handleGeminiWebImageGeneration({ + model, + provider, + body, + credentials, + log, + signal, + clientHeaders, + // Injectable so unit tests can drive the handler without a live Gemini + // session; production uses the real executor. + executorFactory = () => new GeminiWebExecutor(), + // Injectable for tests; production fetches the public googleusercontent URL. + imageFetcher = fetchRemoteImage, +}: { + model: string; + provider: string; + body: Record; + credentials: Record | null | undefined; + log: { + info: (scope: string, message: string) => void; + warn: (scope: string, message: string) => void; + error: (scope: string, message: string) => void; + } | null; + signal?: AbortSignal | null; + clientHeaders?: Record | null; + executorFactory?: () => { + execute: (input: Record) => Promise<{ response: Response }>; + }; + imageFetcher?: (url: string) => Promise<{ buffer: Buffer; contentType: string }>; +}) { + const startTime = Date.now(); + const prompt = typeof body.prompt === "string" ? body.prompt.trim() : ""; + if (!prompt) { + return saveImageErrorResult({ + provider, + model, + status: 400, + startTime, + error: "Prompt is required for Gemini Web image generation", + }); + } + + if (!credentials?.apiKey) { + return saveImageErrorResult({ + provider, + model, + status: 401, + startTime, + error: "Gemini Web credentials missing session cookie", + }); + } + + const rawCount = Number.isInteger(body.n) && (body.n as number) > 0 ? (body.n as number) : 1; + if (rawCount > GEMINI_WEB_IMAGE_N_MAX) { + return saveImageErrorResult({ + provider, + model, + status: 400, + startTime, + error: `Gemini Web image generation supports n=1..${GEMINI_WEB_IMAGE_N_MAX} (got ${rawCount}); each n is a separate ~30-60s web turn.`, + }); + } + const requestedCount = rawCount; + if (log && requestedCount > 1) { + log.warn( + "IMAGE", + `Gemini Web returns image(s) per chat turn; requested n=${requestedCount} will run sequentially` + ); + } + + const wantsBase64 = body.response_format === "b64_json"; + const images: Array<{ url?: string; b64_json?: string }> = []; + const requestBody = { + model, + prompt: prompt.slice(0, 500), + size: body.size || undefined, + n: requestedCount, + }; + + for (let i = 0; i < requestedCount; i++) { + const executor = executorFactory(); + const result = await executor.execute({ + model, + body: { + messages: [{ role: "user", content: buildGeminiWebImagePrompt(body) }], + x_gemini_web_image_mode: true, + }, + stream: false, + credentials, + signal, + log, + clientHeaders, + }); + + const responseText = await result.response.text(); + if (result.response.status >= 400) { + return saveImageErrorResult({ + provider, + model, + status: result.response.status, + startTime, + error: responseText, + requestBody, + retryable: isExpiredOrBlockedGeminiWebSession(result.response.status), + }); + } + + let content = ""; + let urls: string[] = []; + try { + const json = JSON.parse(responseText); + content = String(json?.choices?.[0]?.message?.content || ""); + urls = Array.isArray(json?.x_gemini_web_image_urls) + ? (json.x_gemini_web_image_urls as unknown[]).filter( + (u): u is string => typeof u === "string" && /^https?:\/\//.test(u) + ) + : []; + } catch { + content = responseText; + } + + if (urls.length === 0) { + // Distinguish "refused / no image produced" from a transport failure: + // the executor returns 200 with an empty URL list when the model + // answered with text only (e.g. a policy refusal or a web-search + // answer instead of generation). Surface the assistant text so the + // caller can see WHY nothing was generated. + return saveImageErrorResult({ + provider, + model, + status: 502, + startTime, + error: `Gemini Web completed without generating an image. Assistant text: ${content.slice(0, 300) || "(empty)"}`, + requestBody, + }); + } + + for (const url of urls) { + if (!wantsBase64) { + images.push({ url }); + continue; + } + try { + const fetched = await imageFetcher(url); + images.push({ b64_json: fetched.buffer.toString("base64") }); + } catch (err) { + return saveImageErrorResult({ + provider, + model, + status: 502, + startTime, + error: `Gemini Web generated an image but OmniRoute could not download it for b64_json conversion: ${err instanceof Error ? err.message : String(err)}`, + requestBody, + }); + } + } + } + + return saveImageSuccessResult({ + provider, + model, + startTime, + requestBody, + responseBody: { images_count: images.length }, + images, + }); +} diff --git a/src/shared/constants/providers/noauth.ts b/src/shared/constants/providers/noauth.ts index 2039a50a9f..f7a0f1082a 100644 --- a/src/shared/constants/providers/noauth.ts +++ b/src/shared/constants/providers/noauth.ts @@ -55,6 +55,25 @@ export const NOAUTH_PROVIDERS = { // #7286: tools[] is prompt-emulated via webTools.ts (parseToolCallsFromText). toolCalling: "emulated", }, + "cloudflare-playground": { + id: "cloudflare-playground", + alias: "cfp", + name: "Cloudflare AI Playground", + icon: "cloud", + color: "#F38020", + textIcon: "CF", + website: "https://playground.ai.cloudflare.com", + noAuth: true, + hasFree: true, + serviceKinds: ["llm"], + freeNote: + "Free — Cloudflare's AI Playground: GLM 5.2, Kimi K2.7 Code, DeepSeek V4 Pro, gpt-oss-120B and 16 more. No account, no API key.", + authHint: + "No credentials required — anonymous browser sessions over a reverse-engineered cf_agent WebSocket protocol (Playwright transport).", + notice: { + text: "Cloudflare AI Playground uses a reverse-engineered anonymous WebSocket protocol (no official API). Requires Playwright with a Chromium browser on first request. Rate limits apply per IP (error 3021).", + }, + }, "felo-web": { id: "felo-web", alias: "felo", diff --git a/src/sse/services/imageCredentialRetry.ts b/src/sse/services/imageCredentialRetry.ts index c43d033a02..5758aed108 100644 --- a/src/sse/services/imageCredentialRetry.ts +++ b/src/sse/services/imageCredentialRetry.ts @@ -9,6 +9,14 @@ interface ImageGenerationResult { status?: number; error?: unknown; data?: unknown; + // #10494: opt-in signal a provider handler can set (via + // saveImageErrorResult's `retryable` option) when a non-401 failure is + // still account/session-specific — e.g. an expired or blocked Gemini Web + // session, which the underlying browser-automation executor surfaces as + // 400/500 rather than 401. Only honored together with a connectionId, same + // as the existing 401 path, so providers that never set it keep the + // original 401-only fallback behavior unchanged. + retryable?: boolean; } interface ImageCredentialRetryOptions { @@ -16,6 +24,14 @@ interface ImageCredentialRetryOptions { requestedModel: string | null; credentials: any; execute: (credentials: any) => Promise; + // Injectable so unit tests can drive multi-account fallback deterministically + // without a live DB-backed credential store; production always uses the real + // getProviderCredentialsWithQuotaPreflight-backed selectNextCredentials below. + selectNextCredentials?: ( + provider: string, + requestedModel: string | null, + excludedConnectionIds: Set + ) => Promise; } interface ImageCredentialRetryResult { @@ -34,7 +50,7 @@ function isCredentialSentinel(credentials: any): boolean { return Boolean(credentials?.allRateLimited || credentials?.allExpired); } -async function selectNextCredentials( +async function defaultSelectNextCredentials( provider: string, requestedModel: string | null, excludedConnectionIds: Set @@ -56,6 +72,7 @@ export async function executeImageWithCredentialFallback({ requestedModel, credentials, execute, + selectNextCredentials = defaultSelectNextCredentials, }: ImageCredentialRetryOptions): Promise { // Local/no-auth image providers intentionally have no credential row. They // still need one direct attempt, but there is no account identity to refresh @@ -93,7 +110,8 @@ export async function executeImageWithCredentialFallback({ lastCredentials = currentCredentials; lastResult = await execute(currentCredentials); - if (lastResult.success || Number(lastResult.status) !== 401 || !connectionId) { + const isAuthFailure = Number(lastResult.status) === 401 || lastResult.retryable === true; + if (lastResult.success || !isAuthFailure || !connectionId) { return { credentials: lastCredentials, result: lastResult }; } diff --git a/tests/snapshots/provider/translate-path.json b/tests/snapshots/provider/translate-path.json index 97e7b70709..917bc7d086 100644 --- a/tests/snapshots/provider/translate-path.json +++ b/tests/snapshots/provider/translate-path.json @@ -1196,6 +1196,29 @@ "stream": "https://api.cloudflare.com/client/v4/accounts" } }, + "cloudflare-playground": { + "format": "openai", + "headers": { + "apiKey": { + "Accept": "text/event-stream", + "Authorization": "Bearer ", + "Content-Type": "application/json" + }, + "nonStream": { + "Authorization": "Bearer ", + "Content-Type": "application/json" + }, + "oauth": { + "Accept": "text/event-stream", + "Authorization": "Bearer ", + "Content-Type": "application/json" + } + }, + "url": { + "nonStream": "https://playground.ai.cloudflare.com", + "stream": "https://playground.ai.cloudflare.com" + } + }, "clova-studio": { "format": "openai", "headers": { diff --git a/tests/unit/cloudflare-playground-provider.test.ts b/tests/unit/cloudflare-playground-provider.test.ts new file mode 100644 index 0000000000..7031d1e71f --- /dev/null +++ b/tests/unit/cloudflare-playground-provider.test.ts @@ -0,0 +1,526 @@ +/** + * Tests for the Cloudflare AI Playground (No Auth) provider. + * + * Validates: + * - NOAUTH_PROVIDERS contains the cloudflare-playground entry (noAuth category) + * - Registry entry has correct shape (authType none), curated 20-model catalog + * - Executor resolves for both the primary id and the alias (cfp) + * - cf_agent frame → OpenAI SSE translation, exercised with REAL frames captured + * from the playground on 2026-08-15 (including decoy RPC `done:true` frames + * that must NOT terminate the chat stream, and a real 3021 rate-limit error) + * - Streaming + non-streaming responses, clean upstream errors (no stack traces) + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { NOAUTH_PROVIDERS } from "../../src/shared/constants/providers/noauth.ts"; +import { REGISTRY } from "../../open-sse/config/providers/index.ts"; +import { getExecutor } from "../../open-sse/executors/index.ts"; +import { + CloudflarePlaygroundExecutor, + CfStreamParser, + PlaywrightCfTransport, + toCfMessages, + type CfTransport, +} from "../../open-sse/executors/cloudflare-playground.ts"; + +const CHAT_ID = "chatcmpl-cfp-test123"; + +// ── Fixtures: REAL frames captured from the playground (2026-08-15) ───────── + +const identityFrame = JSON.stringify({ + name: "playground-8d57d26b34b144108fd1f49d2", + agent: "playground", + type: "cf_agent_identity", +}); +const stateFrame = JSON.stringify({ + state: { + model: "@cf/zai-org/glm-4.7-flash", + temperature: 1, + stream: true, + system: "You are a helpful assistant.", + }, + type: "cf_agent_state", +}); +/** Decoy: the setConfig RPC response also carries `done:true` — must be ignored. */ +const decoyRpcDone = JSON.stringify({ + id: "cfp-config", + done: true, + type: "cf_agent_rpc_response", +}); + +const cfFrame = (chatId: string, body: unknown) => + JSON.stringify({ id: chatId, type: "cf_agent_use_chat_response", body: JSON.stringify(body) }); + +/** Full success stream built for a given chat id. */ +const buildSuccessFrames = (chatId: string) => [ + identityFrame, + stateFrame, + decoyRpcDone, + cfFrame(chatId, { type: "start" }), + cfFrame(chatId, { type: "start-step" }), + cfFrame(chatId, { type: "reasoning-start", id: "r1" }), + cfFrame(chatId, { type: "reasoning-delta", delta: "thinking about it...", id: "r1" }), + cfFrame(chatId, { type: "reasoning-end", id: "r1" }), + cfFrame(chatId, { type: "text-start", id: "t1" }), + cfFrame(chatId, { type: "text-delta", delta: "Hello ", id: "t1" }), + cfFrame(chatId, { type: "text-delta", delta: "world!", id: "t1" }), + cfFrame(chatId, { type: "finish-step" }), + cfFrame(chatId, { type: "finish", messageMetadata: { finishReason: "stop" } }), + JSON.stringify({ id: chatId, type: "cf_agent_use_chat_response", done: true }), +]; + +/** Real rate-limit error frame (kimi-k2.6, captured live), built for a chat id. */ +const buildRateLimitFrame = (chatId: string) => + JSON.stringify({ + error: true, + body: JSON.stringify({ + message: "The model is currently rate limited. Please wait a moment and try again.", + details: "3021: rate limiting: inference request per min rate reached", + }), + done: false, + id: chatId, + type: "cf_agent_use_chat_response", + }); + +class FakeTransport implements CfTransport { + constructor( + private framesList: string[], + private fail: { status: number; message: string } | null = null + ) {} + + async start(): Promise<{ ok: true } | { ok: false; status: number; message: string }> { + return this.fail ? { ok: false, ...this.fail } : { ok: true }; + } + + async *frames(): AsyncGenerator { + for (const frame of this.framesList) yield frame; + } + + async close(): Promise {} +} + +function makeExecutor( + buildFrames: (chatId: string) => string[], + fail?: { status: number; message: string } +) { + return new CloudflarePlaygroundExecutor((chatId) => new FakeTransport(buildFrames(chatId), fail)); +} + +const executeArgs = (body: Record, stream: boolean) => + ({ body, credentials: {}, signal: null, stream }) as unknown as Parameters< + CloudflarePlaygroundExecutor["execute"] + >[0]; + +// ── Catalog / NOAUTH_PROVIDERS ─────────────────────────────────────────────── + +test("cloudflare-playground is present in NOAUTH_PROVIDERS (noAuth category)", () => { + const p = (NOAUTH_PROVIDERS as Record)["cloudflare-playground"] as Record< + string, + unknown + >; + assert.ok(p, "NOAUTH_PROVIDERS['cloudflare-playground'] must exist"); + assert.equal(p.id, "cloudflare-playground"); + assert.equal(p.alias, "cfp"); + assert.equal((p.name as string).includes("Cloudflare"), true); + assert.equal(p.noAuth, true); + assert.equal(p.hasFree, true); + assert.ok(typeof p.freeNote === "string" && (p.freeNote as string).length > 0); + assert.ok(typeof p.authHint === "string" && (p.authHint as string).length > 0); + assert.ok(typeof p.website === "string" && (p.website as string).includes("cloudflare.com")); +}); + +test("cloudflare-playground registry entry has no-auth shape and curated models", () => { + const entry = REGISTRY["cloudflare-playground"]; + assert.ok(entry, "REGISTRY['cloudflare-playground'] must exist"); + assert.equal(entry.alias, "cfp"); + assert.equal(entry.format, "openai"); + assert.equal(entry.executor, "cloudflare-playground"); + assert.equal(entry.authType, "none"); + assert.equal(entry.authHeader, "none"); + assert.equal(entry.baseUrl, "https://playground.ai.cloudflare.com"); + + assert.ok( + entry.models.length >= 15, + `expected a curated catalog, got ${entry.models.length} models` + ); + // No model id carries the upstream @cf/ prefix (executor adds it). + for (const model of entry.models) { + assert.ok(!model.id.startsWith("@cf/"), `model id must be prefix-free: ${model.id}`); + } + // Flagships present. + const ids = new Set(entry.models.map((m) => m.id)); + for (const expected of [ + "zai-org/glm-5.2", + "moonshotai/kimi-k2.6", + "deepseek-ai/deepseek-v4-flash-0731", + "openai/gpt-oss-120b", + "qwen/qwen2.5-coder-32b-instruct", + ]) { + assert.ok(ids.has(expected), `expected model ${expected} in catalog`); + } + // Reasoning flags on the known thinking models. + const glm = entry.models.find((m) => m.id === "zai-org/glm-5.2"); + assert.equal(glm?.supportsReasoning, true); + const llama = entry.models.find((m) => m.id === "meta-llama/llama-3.3-70b-instruct-fp8-fast"); + assert.equal(llama?.supportsReasoning, undefined); +}); + +test("executor resolves for both the id and the cfp alias", () => { + const byId = getExecutor("cloudflare-playground"); + const byAlias = getExecutor("cfp"); + assert.ok(byId instanceof CloudflarePlaygroundExecutor); + assert.ok(byAlias instanceof CloudflarePlaygroundExecutor); +}); + +// ── Frame → SSE translation (real captured traffic) ───────────────────────── + +test("CfStreamParser translates a real captured stream (decoys ignored)", () => { + const parser = new CfStreamParser(CHAT_ID); + let events = 0; + for (const frame of buildSuccessFrames(CHAT_ID)) { + const event = parser.push(frame); + if (event) events += 1; + } + assert.equal(parser.text, "Hello world!"); + assert.equal(parser.reasoningText, "thinking about it..."); + assert.equal(parser.finishReason, "stop"); + assert.equal(parser.done, true); + assert.equal(parser.error, null); + // role + 1 reasoning + 2 content + 1 finish + assert.equal(events, 5); +}); + +test("CfStreamParser ignores done:true frames that belong to other ids/RPCs", () => { + const parser = new CfStreamParser(CHAT_ID); + // Decoy RPC response with done:true + parser.push(decoyRpcDone); + assert.equal(parser.done, false, "RPC done:true must not end the chat stream"); + // A chat-response frame for a DIFFERENT chat id + parser.push( + JSON.stringify({ id: "chatcmpl-OTHER", type: "cf_agent_use_chat_response", done: true }) + ); + assert.equal(parser.done, false, "foreign chat id must not end the stream"); + // The real one + parser.push(JSON.stringify({ id: CHAT_ID, type: "cf_agent_use_chat_response", done: true })); + assert.equal(parser.done, true); +}); + +test("CfStreamParser maps the real 3021 rate-limit frame to HTTP 429", () => { + const parser = new CfStreamParser(CHAT_ID); + parser.push(buildRateLimitFrame(CHAT_ID)); + assert.ok(parser.error, "rate-limit frame must surface as an error"); + assert.equal(parser.error?.status, 429); + assert.ok((parser.error?.message ?? "").includes("rate limiting")); + assert.equal(parser.done, false); +}); + +test("toCfMessages drops system/tool, flattens parts, keeps user/assistant", () => { + const out = toCfMessages([ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "hi" }, + { role: "assistant", content: [{ type: "text", text: "hello" }] }, + { + role: "user", + content: [ + { type: "text", text: "a" }, + { type: "text", text: "b" }, + ], + }, + { role: "tool", content: "tool result" }, + { role: "user", content: "" }, + ]); + assert.equal(out.length, 3); + assert.deepEqual(out[0].parts, [{ type: "text", text: "hi" }]); + assert.equal(out[1].parts[0].text, "hello"); + assert.equal(out[2].parts[0].text, "a\nb"); + assert.equal(out[0].role, "user"); + assert.equal(out[1].role, "assistant"); +}); + +// ── Executor behavior (fake transport, real frames) ───────────────────────── + +test("executor streams OpenAI SSE chunks from captured frames", async () => { + const executor = makeExecutor(buildSuccessFrames); + const result = await executor.execute( + executeArgs( + { model: "zai-org/glm-4.7-flash", messages: [{ role: "user", content: "hi" }] }, + true + ) + ); + const response = result.response; + assert.equal(response.status, 200); + assert.match(response.headers.get("content-type") ?? "", /text\/event-stream/); + + const raw = await response.text(); + assert.ok(raw.endsWith("data: [DONE]\n\n"), "stream must end with [DONE]"); + + const chunks = raw + .split("\n") + .filter((line) => line.startsWith("data: ") && line !== "data: [DONE]") + .map((line) => JSON.parse(line.slice(6))); + assert.ok(chunks.length >= 5, `expected several chunks, got ${chunks.length}`); + + const first = chunks[0]; + assert.equal(first.choices[0].delta.role, "assistant"); + assert.equal(first.choices[0].finish_reason, null); + + const reasoningChunk = chunks.find((c) => c.choices?.[0]?.delta?.reasoning_content); + assert.equal(reasoningChunk?.choices?.[0]?.delta?.reasoning_content, "thinking about it..."); + + const content = chunks + .filter((c) => c.choices?.[0]?.delta?.content) + .map((c) => c.choices[0].delta.content) + .join(""); + assert.equal(content, "Hello world!"); + + const last = chunks[chunks.length - 1]; + assert.equal(last.choices[0].finish_reason, "stop"); + assert.equal(last.model, "zai-org/glm-4.7-flash"); +}); + +test("executor returns JSON for non-streaming requests", async () => { + const executor = makeExecutor(buildSuccessFrames); + const result = await executor.execute( + executeArgs( + { model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "hi" }] }, + false + ) + ); + const response = result.response; + assert.equal(response.status, 200); + assert.match(response.headers.get("content-type") ?? "", /application\/json/); + + const parsed = JSON.parse(await response.text()) as { + choices: Array<{ + message: { content: string; reasoning_content?: string }; + finish_reason: string; + }>; + model: string; + }; + assert.equal(parsed.choices[0].message.content, "Hello world!"); + assert.equal(parsed.choices[0].message.reasoning_content, "thinking about it..."); + assert.equal(parsed.choices[0].finish_reason, "stop"); + assert.equal(parsed.model, "moonshotai/kimi-k2.6"); +}); + +test("executor surfaces the 3021 rate limit as a clean 429 (no stack traces)", async () => { + const executor = makeExecutor((chatId) => [buildRateLimitFrame(chatId)]); + const result = await executor.execute( + executeArgs( + { model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "hi" }] }, + false + ) + ); + assert.equal(result.response.status, 429); + const parsed = JSON.parse(await result.response.text()) as { + error: { message: string; type: string }; + }; + assert.ok(parsed.error.message.includes("rate limiting")); + assert.equal(parsed.error.type, "upstream_error"); + assert.ok(!parsed.error.message.includes(" at "), "no stack-trace leak"); +}); + +test("executor returns a clean 502 when the browser session cannot start", async () => { + const executor = makeExecutor(buildSuccessFrames, { + status: 502, + message: "Cloudflare Playground browser session failed: boom", + }); + const result = await executor.execute( + executeArgs( + { model: "zai-org/glm-4.7-flash", messages: [{ role: "user", content: "hi" }] }, + true + ) + ); + assert.equal(result.response.status, 502); + const parsed = JSON.parse(await result.response.text()) as { error: { message: string } }; + assert.ok(parsed.error.message.includes("browser session failed")); + assert.ok(!parsed.error.message.includes(" at "), "no stack-trace leak"); +}); + +test("executor prefixes bare model ids with @cf/ (upstream convention)", async () => { + const seen: string[] = []; + class CapturingTransport extends FakeTransport { + async start(config: Parameters[0]) { + seen.push(config.model); + return { ok: true } as const; + } + } + const executor = new CloudflarePlaygroundExecutor( + (chatId) => new CapturingTransport(buildSuccessFrames(chatId)) + ); + await executor.execute( + executeArgs( + { model: "zai-org/glm-4.7-flash", messages: [{ role: "user", content: "hi" }] }, + false + ) + ); + assert.equal(seen.length, 1); + assert.equal(seen[0], "@cf/zai-org/glm-4.7-flash"); +}); + +// ── #10494: browser/transport resource leak on blocked-request paths ─────── + +test("PlaywrightCfTransport.start() closes the browser when Cloudflare Attention Required is detected", async () => { + const playwright = await import("playwright"); + const originalLaunch = playwright.chromium.launch; + let closeCalls = 0; + + playwright.chromium.launch = (async () => + ({ + newContext: async () => ({ + newPage: async () => ({ + goto: async () => {}, + title: async () => "Attention Required! | Cloudflare", + exposeFunction: async () => {}, + evaluate: async () => {}, + }), + }), + close: async () => { + closeCalls += 1; + }, + }) as unknown as ReturnType) as typeof playwright.chromium.launch; + + try { + const transport = new PlaywrightCfTransport("chat-attention-required"); + const started = await transport.start({ + model: "@cf/test-model", + messages: [], + temperature: 0.7, + }); + assert.equal(started.ok, false); + if (started.ok === false) { + assert.equal(started.status, 502); + } + assert.equal(closeCalls, 1, "browser launched for the challenge check must be closed"); + } finally { + playwright.chromium.launch = originalLaunch; + } +}); + +// ── #10494: streaming timeout must not be misreported as a clean [DONE] ──── + +/** + * A transport whose frames() hangs (never yields) once its initial queue is + * drained, mirroring PlaywrightCfTransport's real behavior: frames() only + * resolves again once close() is called (real close() unblocks pending + * waiters with null, ending the generator). This lets tests force the + * executor's internal chat-timeout branch deterministically instead of + * waiting for CHAT_TIMEOUT_MS. + */ +class HangingTransport implements CfTransport { + closeCalls = 0; + private closed = false; + private queue: string[]; + private waiters: Array<(frame: string | null) => void> = []; + + constructor(initialFrames: string[] = []) { + this.queue = [...initialFrames]; + } + + async start(): Promise<{ ok: true } | { ok: false; status: number; message: string }> { + return { ok: true }; + } + + async *frames(): AsyncGenerator { + while (true) { + if (this.queue.length > 0) { + yield this.queue.shift()!; + continue; + } + const frame = await new Promise((resolve) => this.waiters.push(resolve)); + if (frame === null) return; + yield frame; + } + } + + // Idempotent, mirroring PlaywrightCfTransport.close(): the timer callback + // and the streaming finally block both call close() on the timeout path. + async close(): Promise { + if (this.closed) return; + this.closed = true; + this.closeCalls += 1; + for (const waiter of this.waiters.splice(0)) waiter(null); + } +} + +function parseSseChunks(raw: string) { + return raw + .split("\n\n") + .filter((chunk) => chunk.startsWith("data: ") && chunk !== "data: [DONE]") + .map((chunk) => JSON.parse(chunk.slice(6))); +} + +test("streaming: an empty timeout (no frames at all) emits an explicit error chunk, not a bare [DONE]", async () => { + const transport = new HangingTransport([]); + const executor = new CloudflarePlaygroundExecutor(() => transport, 20); + const result = await executor.execute( + executeArgs( + { model: "zai-org/glm-4.7-flash", messages: [{ role: "user", content: "hi" }] }, + true + ) + ); + const raw = await result.response.text(); + assert.ok(raw.endsWith("data: [DONE]\n\n"), "stream must still end with [DONE]"); + assert.equal(transport.closeCalls, 1, "timed-out transport must be closed"); + + const chunks = parseSseChunks(raw); + assert.ok(chunks.length >= 1, "an error chunk must be emitted before [DONE]"); + const errorChunk = chunks.find((c) => c.error); + assert.ok(errorChunk, "expected an explicit error chunk on timeout"); + assert.equal(errorChunk.error.type, "timeout_error"); + assert.equal(errorChunk.error.code, "HTTP_504"); + assert.ok(!errorChunk.error.message.includes(" at "), "no stack-trace leak"); +}); + +test("streaming: a partial answer followed by a timeout emits content THEN an explicit error chunk", async () => { + // The executor mints its own random chat id (chatcmpl-cfp-) and only + // the transportFactory receives it — frames must reference that same id or + // CfStreamParser silently ignores them (see `msg.id !== this.chatId` + // above). Build the partial frames from the factory callback, exactly like + // buildSuccessFrames()/makeExecutor() do above. + let transport!: HangingTransport; + const executor = new CloudflarePlaygroundExecutor((chatId) => { + const partialFrames = [ + JSON.stringify({ + id: chatId, + type: "cf_agent_use_chat_response", + body: JSON.stringify({ type: "start" }), + }), + JSON.stringify({ + id: chatId, + type: "cf_agent_use_chat_response", + body: JSON.stringify({ type: "text-delta", delta: "Hello", id: "t1" }), + }), + ]; + transport = new HangingTransport(partialFrames); + return transport; + }, 20); + const result = await executor.execute( + executeArgs( + { model: "zai-org/glm-4.7-flash", messages: [{ role: "user", content: "hi" }] }, + true + ) + ); + const raw = await result.response.text(); + assert.ok(raw.endsWith("data: [DONE]\n\n")); + assert.equal(transport.closeCalls, 1); + + const chunks = parseSseChunks(raw); + const content = chunks + .filter((c) => c.choices?.[0]?.delta?.content) + .map((c) => c.choices[0].delta.content) + .join(""); + assert.equal(content, "Hello", "the partial content already streamed must not be dropped"); + + const errorChunk = chunks.find((c) => c.error); + assert.ok(errorChunk, "a partial-then-timeout stream must still surface an explicit error"); + assert.equal(errorChunk.error.type, "timeout_error"); + + // The error chunk must come after the content, so a client processing the + // stream in order sees the partial answer followed by a clear failure — + // never a silent, successful-looking [DONE] right after partial content. + const errorIndex = chunks.indexOf(errorChunk); + const lastContentIndex = chunks.findLastIndex((c) => c.choices?.[0]?.delta?.content); + assert.ok(errorIndex > lastContentIndex, "error chunk must follow the streamed content"); +}); diff --git a/tests/unit/gemini-web-image-account-fallback.test.ts b/tests/unit/gemini-web-image-account-fallback.test.ts new file mode 100644 index 0000000000..3be797f6b6 --- /dev/null +++ b/tests/unit/gemini-web-image-account-fallback.test.ts @@ -0,0 +1,174 @@ +// #10494: Gemini Web image-generation account fallback gap. +// +// #10466's acceptance criteria require that "expired or blocked sessions +// return a clear session/provider error and can fall back normally inside an +// image Combo." The gemini-web image handler passed the executor's raw HTTP +// status straight through to executeImageWithCredentialFallback, whose retry +// loop only advances to the next account on a plain HTTP 401 — but the +// underlying GeminiWebExecutor's browser-automation catch paths surface an +// expired/blocked session as 400 (Playwright selector/click timeout — "the +// session is so expired it lands on a different page", #9407) or 500 (the +// generic automation-failure catch-all), never 401. So expired/blocked +// Gemini Web sessions never triggered account fallback. +// +// Covers: +// - isExpiredOrBlockedGeminiWebSession() classification (unit). +// - A multi-account regression: first account fails with a classified +// status, the retry loop advances to a second account, which succeeds. +// - An invalid-session test that drives the REAL GeminiWebExecutor (Playwright +// launch mocked, same technique as tests/unit/gemini-web.test.ts) so the +// classified status is the executor's actual status code, not a synthetic +// one, and confirms the handler marks it retryable end to end. +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const TEST_DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-geminiweb-image-fallback-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const { isExpiredOrBlockedGeminiWebSession, handleGeminiWebImageGeneration } = await import( + "../../open-sse/handlers/imageGeneration/providers/geminiWeb.ts" +); +const { executeImageWithCredentialFallback } = await import( + "../../src/sse/services/imageCredentialRetry.ts" +); +const { GeminiWebExecutor } = await import("../../open-sse/executors/gemini-web.ts"); +const core = await import("../../src/lib/db/core.ts"); + +test.after(() => { + core.resetDbInstance(); + rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +// ── Classification (unit) ─────────────────────────────────────────────────── + +test("isExpiredOrBlockedGeminiWebSession classifies 400/500 as retryable, everything else as not", () => { + assert.equal(isExpiredOrBlockedGeminiWebSession(400), true); + assert.equal(isExpiredOrBlockedGeminiWebSession(500), true); + assert.equal(isExpiredOrBlockedGeminiWebSession(401), false, "handled by the plain 401 path"); + assert.equal( + isExpiredOrBlockedGeminiWebSession(503), + false, + "missing-Playwright-browser is a host/config problem, not a per-account issue" + ); + assert.equal(isExpiredOrBlockedGeminiWebSession(502), false); + assert.equal(isExpiredOrBlockedGeminiWebSession(200), false); +}); + +// ── Multi-account regression: 2 accounts, first classified-fails, second succeeds ── + +test("executeImageWithCredentialFallback: expired/blocked (400) on account 1 falls back to account 2", async () => { + const attempts: string[] = []; + const accountA = { connectionId: "conn-a", apiKey: "cookie-a" }; + const accountB = { connectionId: "conn-b", apiKey: "cookie-b" }; + + const execution = await executeImageWithCredentialFallback({ + provider: "gemini-web", + requestedModel: "gemini-2.5-pro", + credentials: accountA, + // Simulates the real handler path: geminiWeb.ts sets retryable via + // saveImageErrorResult when the executor status is classified as an + // expired/blocked session (400/500), not just a plain 401. + execute: async (creds) => { + attempts.push(creds.connectionId); + if (creds.connectionId === "conn-a") { + return { success: false, status: 400, error: "session expired", retryable: true }; + } + return { success: true, data: { created: 1, data: [{ url: "https://example/img.png" }] } }; + }, + selectNextCredentials: async () => accountB, + }); + + assert.deepEqual(attempts, ["conn-a", "conn-b"], "must try both accounts in order"); + assert.equal(execution.result.success, true); + assert.equal(execution.credentials.connectionId, "conn-b"); +}); + +test("executeImageWithCredentialFallback: a non-retryable 400 (e.g. bad prompt) does NOT burn a second account", async () => { + const attempts: string[] = []; + const accountA = { connectionId: "conn-a", apiKey: "cookie-a" }; + + const execution = await executeImageWithCredentialFallback({ + provider: "gemini-web", + requestedModel: "gemini-2.5-pro", + credentials: accountA, + execute: async (creds) => { + attempts.push(creds.connectionId); + return { success: false, status: 400, error: "Prompt is required" }; // retryable unset + }, + selectNextCredentials: async () => { + throw new Error("must not be called for a non-retryable failure"); + }, + }); + + assert.deepEqual(attempts, ["conn-a"]); + assert.equal(execution.result.success, false); + assert.equal(execution.result.status, 400); +}); + +// ── Invalid-session test against the REAL executor's actual status code ──── + +test("handler classifies the REAL GeminiWebExecutor's session-expired 400 as retryable", async () => { + const playwright = await import("playwright"); + const originalLaunch = playwright.chromium.launch; + + // Mirrors tests/unit/gemini-web.test.ts's pattern for a fake page whose + // waitForSelector() times out — the exact path (#9407) that makes the + // real executor return a 400 tagged "the session is so expired it lands + // on a different page". + playwright.chromium.launch = (async () => + ({ + newContext: async () => ({ + addCookies: async () => {}, + newPage: async () => ({ + on: () => {}, + goto: async () => {}, + waitForTimeout: async () => {}, + waitForSelector: async () => { + const err = new Error("Timeout 10000ms exceeded while waiting for selector"); + err.name = "TimeoutError"; + throw err; + }, + }), + }), + close: async () => {}, + }) as unknown as ReturnType) as typeof playwright.chromium.launch; + + try { + const executor = new GeminiWebExecutor(); + const direct = await executor.execute({ + model: "gemini-2.5-pro", + body: { messages: [{ role: "user", content: "hi" }], x_gemini_web_image_mode: true }, + stream: false, + credentials: { apiKey: "expired-session-cookie" }, + signal: AbortSignal.timeout(10000), + log: null, + }); + // Confirm the REAL executor really does surface this as 400 (not a + // synthetic status invented by the test). + assert.equal(direct.response.status, 400, "sanity: executor's real session-expired status"); + + const res = await handleGeminiWebImageGeneration({ + model: "gemini-2.5-pro", + provider: "gemini-web", + body: { prompt: "a kitten" }, + credentials: { apiKey: "expired-session-cookie", connectionId: "conn-real" }, + log: null, + signal: null, + clientHeaders: {}, + executorFactory: () => new GeminiWebExecutor(), + }); + + assert.equal(res.success, false); + assert.equal(res.status, 400); + assert.equal( + (res as { retryable?: boolean }).retryable, + true, + "the handler must mark the real executor's session-expired status as retryable" + ); + } finally { + playwright.chromium.launch = originalLaunch; + } +}); diff --git a/tests/unit/gemini-web-image-generation-10466.test.ts b/tests/unit/gemini-web-image-generation-10466.test.ts new file mode 100644 index 0000000000..252329e8dd --- /dev/null +++ b/tests/unit/gemini-web-image-generation-10466.test.ts @@ -0,0 +1,320 @@ +// Tests for gemini-web image generation (#10466). +// +// Fixtures are built from the documented StreamGenerate frame layout for +// generated images (corroborated by gpt4free's Gemini provider and +// HanaokaYuzu/Gemini-API's _parse_candidate): +// +// wrb.fr line → JSON [ "wrb.fr", null, "" ] +// payload → JSON [ ..., [4] = [ candidate ] ] +// candidate[1] = [ "answer text" ] +// candidate[12][1] = web-search images (must NOT be collected) +// candidate[12][7][0] = generated-image entries +// entry[0][3][3] = image URL (string OR list of strings) +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-gweb-image-")); + +const { parseStreamResponse, parseStreamResponseImages } = + await import("../../open-sse/executors/gemini-web.ts"); +const { handleGeminiWebImageGeneration, buildGeminiWebImagePrompt } = + await import("../../open-sse/handlers/imageGeneration/providers/geminiWeb.ts"); +const { parseImageModel, getImageProvider } = + await import("../../open-sse/config/imageRegistry.ts"); + +// ─── Fixture builders ─────────────────────────────────────────────────────── + +/** Build one wrb.fr StreamGenerate line with the given candidate. */ +function frameLine(candidate: unknown): string { + const payload = JSON.stringify([null, [], null, null, [candidate]]); + return JSON.stringify([["wrb.fr", null, payload]]); +} + +/** Candidate carrying answer text and/or generated images. */ +function candidate({ + text = "", + generatedUrls = [], + webImageUrls = [], +}: { + text?: string; + generatedUrls?: Array; + webImageUrls?: string[]; +} = {}): unknown[] { + const cand: unknown[] = []; + cand[1] = [text]; + if (webImageUrls.length > 0 || generatedUrls.length > 0) { + const ext: unknown[] = []; + if (webImageUrls.length > 0) { + // [12][1]: web-search result thumbnails — [[ [url, ...], ... ]] + ext[1] = webImageUrls.map((u) => [[[u]]]); + } + if (generatedUrls.length > 0) { + // [12][7][0]: generated-image entries; parser reads entry[0][3][3] = url + ext[7] = [generatedUrls.map((u) => [[null, null, null, [null, null, null, u]]])]; + } + cand[12] = ext; + } + return cand; +} + +function streamResponse(lines: string[]): string { + return [")]}'", ...lines.map((l) => `${l.length}\n${l}`)].join("\n"); +} + +const IMG_URL = "https://lh3.googleusercontent.com/gg-dl/generated-abc123"; +const IMG_URL_2 = "https://lh3.googleusercontent.com/gg-dl/generated-def456"; +const WEB_URL = "https://example.com/web-search-thumb.jpg"; + +// ─── parseStreamResponseImages ────────────────────────────────────────────── + +test("extracts generated-image URL from a realistic frame (string form)", () => { + const raw = streamResponse([ + frameLine(candidate({ text: "Here you go!", generatedUrls: [IMG_URL] })), + ]); + assert.deepEqual(parseStreamResponseImages(raw), [`${IMG_URL}=s2048`]); +}); + +test("handles list-form URL field (takes first http entry)", () => { + const raw = streamResponse([ + frameLine(candidate({ generatedUrls: [["not-a-url", IMG_URL, IMG_URL_2]] })), + ]); + assert.deepEqual(parseStreamResponseImages(raw), [`${IMG_URL}=s2048`]); +}); + +test("dedupes across cumulative frames, preserving first-seen order", () => { + // Frames are cumulative snapshots: frame 2 repeats image 1 and adds image 2. + const raw = streamResponse([ + frameLine(candidate({ text: "partial", generatedUrls: [IMG_URL] })), + frameLine(candidate({ text: "full answer", generatedUrls: [IMG_URL, IMG_URL_2] })), + ]); + assert.deepEqual(parseStreamResponseImages(raw), [`${IMG_URL}=s2048`, `${IMG_URL_2}=s2048`]); +}); + +test("does NOT collect web-search images at [12][1]", () => { + const raw = streamResponse([ + frameLine(candidate({ text: "found these", webImageUrls: [WEB_URL] })), + ]); + assert.deepEqual(parseStreamResponseImages(raw), []); +}); + +test("does not double-append size directive when one is present", () => { + const sized = `${IMG_URL}=w1024-h512`; + const raw = streamResponse([frameLine(candidate({ generatedUrls: [sized] }))]); + assert.deepEqual(parseStreamResponseImages(raw), [sized]); +}); + +test("returns [] for text-only frames (chat responses unaffected)", () => { + const raw = streamResponse([frameLine(candidate({ text: "just text, no images" }))]); + assert.deepEqual(parseStreamResponseImages(raw), []); +}); + +test("skips malformed lines without throwing", () => { + const raw = [ + ")]}'", + "garbage not json", + JSON.stringify([["wrb.fr", null, "{broken json"]]), + frameLine(candidate({ generatedUrls: [IMG_URL] })), + ].join("\n"); + assert.deepEqual(parseStreamResponseImages(raw), [`${IMG_URL}=s2048`]); +}); + +test("text parser still extracts text from image-bearing frames", () => { + const raw = streamResponse([ + frameLine(candidate({ text: "Here is your image!", generatedUrls: [IMG_URL] })), + ]); + assert.equal(parseStreamResponse(raw), "Here is your image!"); +}); + +// ─── buildGeminiWebImagePrompt ────────────────────────────────────────────── + +test("prompt leads with an explicit generation directive", () => { + const prompt = buildGeminiWebImagePrompt({ prompt: "a red panda", size: "1024x1536" }); + assert.match(prompt, /^Generate an image for this prompt: a red panda/); + assert.match(prompt, /Do not search the web/); + assert.match(prompt, /1024x1536/); +}); + +// ─── handleGeminiWebImageGeneration ───────────────────────────────────────── + +function fakeExecutor(jsonBody: object, status = 200) { + return { + execute: async () => ({ + response: new Response(JSON.stringify(jsonBody), { + status, + headers: { "Content-Type": "application/json" }, + }), + }), + }; +} + +const baseArgs = { + model: "nano-banana-web", + provider: "gemini-web", + body: { prompt: "a red panda eating bamboo" }, + credentials: { apiKey: "***" }, + log: null, + signal: null, + clientHeaders: {}, +}; + +test("success: returns image URLs in OpenAI image response shape", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "Here you go!" } }], + x_gemini_web_image_urls: [IMG_URL], + }), + }); + assert.equal(res.success, true); + assert.equal(res.data.data.length, 1); + assert.equal(res.data.data[0].url, IMG_URL); + assert.ok(res.data.created > 0); +}); + +test("success: b64_json downloads the image via injected fetcher", async () => { + const bytes = Buffer.from("fake-png-bytes"); + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + body: { prompt: "a red panda", response_format: "b64_json" }, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "" } }], + x_gemini_web_image_urls: [IMG_URL], + }), + imageFetcher: async (url: string) => { + assert.equal(url, IMG_URL); + return { buffer: bytes, contentType: "image/png" }; + }, + }); + assert.equal(res.success, true); + assert.equal(res.data.data[0].b64_json, bytes.toString("base64")); + assert.equal(res.data.data[0].url, undefined); +}); + +test("b64_json download failure surfaces a specific 502", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + body: { prompt: "a red panda", response_format: "b64_json" }, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "" } }], + x_gemini_web_image_urls: [IMG_URL], + }), + imageFetcher: async () => { + throw new Error("Remote image fetch error 403"); + }, + }); + assert.equal(res.success, false); + assert.equal(res.status, 502); + assert.match(res.error, /generated an image but OmniRoute could not download it/); +}); + +test("no images generated: 502 includes assistant text (refusal visibility)", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "I can't generate that image." } }], + x_gemini_web_image_urls: [], + }), + }); + assert.equal(res.success, false); + assert.equal(res.status, 502); + assert.match(res.error, /without generating an image/); + assert.match(res.error, /I can't generate that image/); +}); + +test("missing prompt → 400", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + body: { prompt: " " }, + }); + assert.equal(res.success, false); + assert.equal(res.status, 400); +}); + +test("missing cookie → 401", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + credentials: {}, + }); + assert.equal(res.success, false); + assert.equal(res.status, 401); +}); + +test("n above the cap → 400 with the cap named", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + body: { prompt: "a red panda", n: 5 }, + }); + assert.equal(res.success, false); + assert.equal(res.status, 400); + assert.match(res.error, /n=1\.\.4/); +}); + +test("executor error status passes through", async () => { + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + executorFactory: () => fakeExecutor({ error: "Missing Gemini cookies" }, 401), + }); + assert.equal(res.success, false); + assert.equal(res.status, 401); +}); + +test("n=2 runs sequentially and collects both turns' images", async () => { + let calls = 0; + const res = await handleGeminiWebImageGeneration({ + ...baseArgs, + body: { prompt: "a red panda", n: 2 }, + executorFactory: () => ({ + execute: async () => { + calls++; + const url = calls === 1 ? IMG_URL : IMG_URL_2; + return { + response: new Response( + JSON.stringify({ + choices: [{ message: { role: "assistant", content: "" } }], + x_gemini_web_image_urls: [url], + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ), + }; + }, + }), + }); + assert.equal(calls, 2); + assert.equal(res.success, true); + assert.deepEqual( + res.data.data.map((d: { url?: string }) => d.url), + [IMG_URL, IMG_URL_2] + ); +}); + +// ─── Registry wiring ──────────────────────────────────────────────────────── + +test("registry: gemini-web/nano-banana resolves to the gemini-web provider", () => { + const parsed = parseImageModel("gemini-web/nano-banana-web"); + assert.equal(parsed.provider, "gemini-web"); + assert.equal(parsed.model, "nano-banana-web"); + const config = getImageProvider("gemini-web"); + assert.ok(config); + assert.equal(config.format, "gemini-web"); + assert.equal(config.authHeader, "cookie"); +}); + +test("registry: alias gweb/nano-banana resolves too", () => { + const parsed = parseImageModel("gweb/nano-banana-web"); + assert.equal(parsed.provider, "gemini-web"); + assert.equal(parsed.model, "nano-banana-web"); +}); + +test("registry regression: bare nano-banana still routes to adobe-firefly", () => { + // adobe-firefly owns the bare nano-banana ids (operator decision 2026-07-31); + // the new gemini-web entry must not steal that resolution. + const parsed = parseImageModel("nano-banana"); + assert.equal(parsed.provider, "adobe-firefly"); +});