diff --git a/.env.example b/.env.example index 4e6438b593..81b03d5b11 100644 --- a/.env.example +++ b/.env.example @@ -1524,6 +1524,15 @@ CURSOR_USER_AGENT="Cursor/3.4" # request into the browser-backed path. # OMNIROUTE_BROWSER_POOL=on # WEB_COOKIE_USE_BROWSER=0 +# Obscura (https://github.com/h4ckf0r0day/obscura) is the primary headless +# engine: a lightweight CDP server the pool and cloudflare-playground connect +# to before falling back to Chromium. Unset OBSCURA_BIN to auto-detect from +# PATH; set OBSCURA_CDP_ENDPOINT to reuse an already-running Obscura instead +# of spawning one; set OBSCURA_PORT to pin the spawned serve port. +# Used by: open-sse/services/obscura.ts +# OBSCURA_BIN= +# OBSCURA_CDP_ENDPOINT= +# OBSCURA_PORT= # ── Kimi Web (international kimi.ai Connect-RPC) ── # Used by: open-sse/executors/kimi-web.ts. Override the base/chat URLs only if diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index ace16cf6c9..9dd2fff9f0 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -776,6 +776,9 @@ REQUEST_TIMEOUT_MS (global override) | `OMNIROUTE_NOTION_TLS_TIMEOUT_MS` | `30000` | Native wreq-js request timeout (`notionTlsClient.ts`); `notion-web` raises it per request to `180000` for long generations. | | `OMNIROUTE_NOTION_TLS_GRACE_MS` | `10000` | Absolute JS hard-deadline grace added on top of the native timeout. | | `OMNIROUTE_BROWSER_POOL` | `on` | Shared Playwright browser pool for browser-backed web-cookie chat (`browserPool.ts`); set `off` to disable. | +| `OBSCURA_BIN` | `auto-detect` | Path to the `obscura` binary used as the primary engine by the browser pool and Cloudflare Playground executor (`open-sse/services/obscura.ts`); auto-detected from the system PATH when unset. | +| `OBSCURA_CDP_ENDPOINT` | _(unset)_ | Point at an already-running Obscura (`http://host:port`) instead of spawning one; the module does not own that process (`open-sse/services/obscura.ts`). | +| `OBSCURA_PORT` | `random free port` | Explicit port for the spawned `obscura serve`; a free port is chosen automatically when unset (`open-sse/services/obscura.ts`). | | `WEB_COOKIE_USE_BROWSER` | `0` | Opt a web-cookie chat request into the browser-backed path (`browserBackedChat.ts`); `1` to enable. | | `KIMI_WEB_BASE_URL` | `https://www.kimi.ai` | Base URL for the Kimi Web (international kimi.ai Connect-RPC) executor (`kimi-web.ts`); override only for mirror/proxy endpoints. | | `KIMI_WEB_CHAT_URL` | `/apiv2/kimi.gateway.chat.v1.ChatService/Chat` | Full chat endpoint for the Kimi Web executor (`kimi-web.ts`). | diff --git a/open-sse/executors/cloudflare-playground.ts b/open-sse/executors/cloudflare-playground.ts index ba309f1eed..249e069da3 100644 --- a/open-sse/executors/cloudflare-playground.ts +++ b/open-sse/executors/cloudflare-playground.ts @@ -36,6 +36,7 @@ import { randomUUID } from "crypto"; import { BaseExecutor, type ExecuteInput } from "./base.ts"; import { makeExecutorErrorResult as makeErrorResult } from "../utils/error.ts"; +import { connectObscuraBrowser } from "../services/obscura.ts"; import type { Browser, Page } from "playwright"; export const PLAYGROUND_URL = "https://playground.ai.cloudflare.com/"; @@ -296,14 +297,22 @@ export class PlaywrightCfTransport implements CfTransport { config: CfTransportConfig ): Promise<{ ok: true } | { ok: false; status: number; message: string }> { try { + // #12274: prefer the shared Obscura browser (browser-grade TLS fingerprint + // on the WS upgrade, ~30MB) over a full Chromium per request; fall back to + // a direct Chromium launch when Obscura is unavailable. + const obscura = await connectObscuraBrowser(); const playwright = await importPlaywright(); - const executablePath = - this.chromeExecutablePath ?? process.env.CLOUDFLARE_PLAYGROUND_CHROME_PATH; - this.browser = await playwright.chromium.launch({ - ...(executablePath ? { executablePath } : {}), - headless: true, - args: BROWSER_ARGS, - }); + if (obscura) { + this.browser = obscura.browser; + } else { + const executablePath = + this.chromeExecutablePath ?? process.env.CLOUDFLARE_PLAYGROUND_CHROME_PATH; + this.browser = await playwright.chromium.launch({ + ...(executablePath ? { executablePath } : {}), + headless: true, + args: BROWSER_ARGS, + }); + } const context = await this.browser.newContext({ userAgent: PLAYGROUND_UA }); const page = await context.newPage(); this.page = page; diff --git a/open-sse/services/browserPool.ts b/open-sse/services/browserPool.ts index bcab2dc174..51a1abb44b 100644 --- a/open-sse/services/browserPool.ts +++ b/open-sse/services/browserPool.ts @@ -26,6 +26,8 @@ import { Buffer } from "node:buffer"; +import { connectObscuraBrowser } from "./obscura.ts"; + type Browser = import("playwright").Browser; type BrowserContext = import("playwright").BrowserContext; type Page = import("playwright").Page; @@ -86,8 +88,12 @@ function createBrowserPoolMetrics(): BrowserPoolMetrics { }; } +type PoolEngine = "obscura" | "cloakbrowser" | "chromium"; + interface PoolState { browser: Browser | null; + /** Engine backing the headless browser, for metrics and stealth detection. */ + engine: PoolEngine | null; headedBrowser: Browser | null; contexts: Map; pendingContexts: Map>; @@ -110,6 +116,7 @@ const DEFAULT_USER_AGENT = const state: PoolState = { browser: null, + engine: null, headedBrowser: null, contexts: new Map(), pendingContexts: new Map(), @@ -288,13 +295,26 @@ async function launchBrowserInstance( options: BrowserPoolContextOptions, headless: boolean ): Promise { + // A headed browser must be a real windowed Chromium, so the engine + // preference below applies to the headless path only. if (!headless) { const { chromium } = await import("playwright"); return chromium.launch(resolvePlainBrowserLaunchOptions(options)); } + // #12274: prefer Obscura (lightweight, browser-grade CDP) over a full + // Chromium; fall back to cloakbrowser, then plain Chromium. Obscura's + // lifecycle (one shared `obscura serve` per process) lives in ./obscura.ts, + // so executors like cloudflare-playground reuse the same server. + const obscura = await connectObscuraBrowser(); + if (obscura) { + state.engine = "obscura"; + return obscura.browser; + } + const cloakLaunch = await resolveCloakLaunch(); if (cloakLaunch) { + state.engine = "cloakbrowser"; return cloakLaunch({ headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage"], @@ -303,6 +323,7 @@ async function launchBrowserInstance( // Fallback: plain Playwright. Works for Claude web (cookie-only auth) but // DDG's VQD challenge will detect this Chromium build. + state.engine = "chromium"; const { chromium } = await import("playwright"); return chromium.launch(resolvePlainBrowserLaunchOptions(options)); } @@ -471,7 +492,7 @@ export async function acquireBrowserContext( launchBrowser(options), resolveBrowserContextProxy(key, options), ]); - const isStealth = headless && state.cloakLaunch !== null; + const isStealth = headless && (state.engine === "obscura" || state.cloakLaunch !== null); const context = await browser.newContext({ userAgent: options.userAgent || DEFAULT_USER_AGENT, locale: options.locale || "en-US", @@ -580,6 +601,10 @@ export async function shutdownPool(reason: string): Promise { } state.launching = null; state.headedLaunching = null; + // #12274: the shared Obscura server is owned by ./obscura.ts and reused by + // executors (cloudflare-playground), so closing the pool's CDP connection is + // enough — never kill the server here. + state.engine = null; state.lastActivity = Date.now(); // Avoid unused-parameter lint: log reason via debug if anyone hooks // process.on('exit') and prints state. @@ -590,6 +615,7 @@ export function getBrowserPoolStatus(): { enabled: boolean; contexts: number; browserRunning: boolean; + engine: PoolEngine | null; stealthAvailable: boolean; lastActivityAgoMs: number; } { @@ -597,7 +623,8 @@ export function getBrowserPoolStatus(): { enabled: isPoolEnabled(), contexts: state.contexts.size, browserRunning: state.browser !== null || state.headedBrowser !== null, - stealthAvailable: state.cloakLaunch !== null, + engine: state.engine, + stealthAvailable: state.engine === "obscura" || state.cloakLaunch !== null, lastActivityAgoMs: state.lastActivity === 0 ? -1 : Date.now() - state.lastActivity, }; } diff --git a/open-sse/services/obscura.ts b/open-sse/services/obscura.ts new file mode 100644 index 0000000000..23c4917214 --- /dev/null +++ b/open-sse/services/obscura.ts @@ -0,0 +1,163 @@ +/** + * obscura.ts — Shared Obscura browser engine (#12274). + * + * Obscura (https://github.com/h4ckf0r0day/obscura) is a lightweight Rust + * headless browser (~30MB resident) that speaks the Chrome DevTools Protocol. + * Playwright's `chromium.connectOverCDP` drives it like a real Chrome, so the + * browser pool and the cloudflare-playground executor can both use it without + * holding a 150-400MB Chromium process. + * + * Lifecycle: one Obscura `serve` process is spawned lazily on first use and + * shared for the server's lifetime. Callers receive a fresh CDP connection on + * demand; closing the connection does not stop the shared server. Set + * OBSCURA_CDP_ENDPOINT to point at an already-running Obscura instead of + * spawning one here (the process is then not owned by this module). The + * module is also disabled entirely when OMNIROUTE_BROWSER_POOL=off. + */ + +import { spawn, type ChildProcess } from "node:child_process"; +import { createServer } from "node:net"; + +export interface ObscuraConnection { + /** Playwright Browser connected over CDP to the shared Obscura server. */ + browser: import("playwright").Browser; + /** The spawned `obscura serve` process, or null when an external endpoint is used. */ + child: ChildProcess | null; +} + +let shared: { child: ChildProcess | null; endpoint: string } | null = null; +let starting: Promise<{ child: ChildProcess | null; endpoint: string } | null> | null = null; + +export function isObscuraUsable(): boolean { + const flag = process.env.OMNIROUTE_BROWSER_POOL; + if (flag === undefined) return true; + return flag !== "off" && flag !== "0" && flag !== "false"; +} + +function findFreePort(): Promise { + return new Promise((resolve, reject) => { + const srv = createServer(); + srv.once("error", reject); + srv.listen(0, "127.0.0.1", () => { + const address = srv.address(); + srv.close(() => { + if (address && typeof address === "object") resolve(address.port); + else reject(new Error("obscura: could not allocate a free port")); + }); + }); + }); +} + +async function obscuraBinaryPath(): Promise { + const bin = process.env.OBSCURA_BIN; + if (bin) return bin; + const { resolve } = await import("node:path"); + const { existsSync, accessSync, constants } = await import("node:fs"); + const dirs = (process.env.PATH || "").split(":"); + for (const dir of dirs) { + const candidate = resolve(dir, "obscura"); + try { + accessSync(candidate, constants.X_OK); + if (existsSync(candidate)) return candidate; + } catch { + /* not executable here — keep looking */ + } + } + return null; +} + +async function waitForCdpEndpoint(endpoint: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 1500); + // Probe /json/version, not the base URL: Obscura's HTTP server answers + // the CDP info route, while a bare GET to "/" never completes a response. + const probe = endpoint.replace(/^ws/, "http").replace(/\/$/, "") + "/json/version"; + const res = await fetch(probe, { signal: controller.signal }); + clearTimeout(timer); + if (res.ok) return true; + } catch { + /* not up yet */ + } + await new Promise((r) => setTimeout(r, 250)); + } + return false; +} + +/** Ensure the shared Obscura server is up; returns its endpoint or null. */ +export async function ensureObscuraServer(): Promise<{ + child: ChildProcess | null; + endpoint: string; +} | null> { + if (!isObscuraUsable()) return null; + if (shared) return shared; + if (starting) return starting; + starting = (async () => { + const endpoint = process.env.OBSCURA_CDP_ENDPOINT; + if (endpoint) { + shared = { child: null, endpoint }; + return shared; + } + const bin = await obscuraBinaryPath(); + if (!bin) return null; + const port = Number(process.env.OBSCURA_PORT) || (await findFreePort()); + const child = spawn(bin, ["serve", "--port", String(port), "--host", "127.0.0.1"], { + stdio: ["ignore", "ignore", "pipe"], + }); + child.stderr?.on("data", () => {}); // obscura logs verbosely — swallow + const endpointForServer = `http://127.0.0.1:${port}`; + // A bad binary path (or a binary that cannot serve) must not hold the + // readiness wait for the full timeout: bail as soon as the child exits + // (or fails to spawn at all — 'exit' alone misses an ENOENT 'error'). + const died = new Promise((resolve) => { + child.once("exit", () => resolve(true)); + child.once("error", () => resolve(true)); + }); + const ready = await Promise.race([ + waitForCdpEndpoint(endpointForServer, 30_000), + died.then(() => false as const), + ]); + if (ready !== true) { + child.kill("SIGKILL"); + return null; + } + shared = { child, endpoint: endpointForServer }; + return shared; + })(); + try { + return await starting; + } finally { + starting = null; + } +} + +/** + * Connect Playwright to the shared Obscura server. Returns null when Obscura + * is disabled, not installed, or the server could not start (callers fall + * back to their previous Chromium strategy). + */ +export async function connectObscuraBrowser(): Promise { + const server = await ensureObscuraServer(); + if (!server) return null; + try { + const { chromium } = await import("playwright"); + const browser = await chromium.connectOverCDP(server.endpoint); + return { browser, child: server.child }; + } catch { + return null; + } +} + +/** Caution: this terminates the shared `obscura serve` process (process-lifetime anyway). */ +export function killSharedObscuraServer(): void { + if (shared?.child) { + try { + shared.child.kill("SIGKILL"); + } catch { + /* ignore */ + } + } + shared = null; +} diff --git a/tests/unit/obscura-integration.test.ts b/tests/unit/obscura-integration.test.ts new file mode 100644 index 0000000000..67a3006d79 --- /dev/null +++ b/tests/unit/obscura-integration.test.ts @@ -0,0 +1,155 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { createServer } from "node:net"; + +import { + connectObscuraBrowser, + ensureObscuraServer, + killSharedObscuraServer, + isObscuraUsable, +} from "../../open-sse/services/obscura.ts"; + +// #12274 — Obscura-first browser engine. The shared server is process-lifetime; +// each test resets it so suites run independently. When `obscura` is not +// installed the live tests skip; the null-return path is still covered. + +const BIN_RESULT = spawnSync("which", ["obscura"], { encoding: "utf8" }); +const HAS_OBSCURA = BIN_RESULT.status === 0 && BIN_RESULT.stdout.trim().length > 0; + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const srv = createServer(); + srv.once("error", reject); + srv.listen(0, "127.0.0.1", () => { + const address = srv.address(); + srv.close(() => { + if (address && typeof address === "object") resolve(address.port); + else reject(new Error("no free port")); + }); + }); + }); +} + +async function waitForCdp(endpoint: string, timeoutMs = 30_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 1500); + // Probe /json/version (Obscura's bare "/" never completes a response). + const probe = endpoint.replace(/^ws/, "http").replace(/\/$/, "") + "/json/version"; + const res = await fetch(probe, { signal: controller.signal }); + clearTimeout(timer); + if (res.ok) return true; + } catch { + /* not up yet */ + } + await new Promise((r) => setTimeout(r, 250)); + } + return false; +} + +describe("obscura engine", () => { + it("respects OMNIROUTE_BROWSER_POOL=off", () => { + const original = process.env.OMNIROUTE_BROWSER_POOL; + process.env.OMNIROUTE_BROWSER_POOL = "off"; + try { + assert.equal(isObscuraUsable(), false); + } finally { + if (original === undefined) delete process.env.OMNIROUTE_BROWSER_POOL; + else process.env.OMNIROUTE_BROWSER_POOL = original; + } + }); + + it("is enabled by default (no env var)", () => { + const original = process.env.OMNIROUTE_BROWSER_POOL; + delete process.env.OMNIROUTE_BROWSER_POOL; + try { + assert.equal(isObscuraUsable(), true); + } finally { + if (original !== undefined) process.env.OMNIROUTE_BROWSER_POOL = original; + } + }); + + it("returns null when the binary is absent or cannot start", async () => { + killSharedObscuraServer(); + const originalBin = process.env.OBSCURA_BIN; + const originalEndpoint = process.env.OBSCURA_CDP_ENDPOINT; + process.env.OBSCURA_BIN = "/nonexistent/obscura"; + delete process.env.OBSCURA_CDP_ENDPOINT; + try { + const server = await ensureObscuraServer(); + assert.equal(server, null); + } finally { + if (originalBin === undefined) delete process.env.OBSCURA_BIN; + else process.env.OBSCURA_BIN = originalBin; + if (originalEndpoint === undefined) delete process.env.OBSCURA_CDP_ENDPOINT; + else process.env.OBSCURA_CDP_ENDPOINT = originalEndpoint; + killSharedObscuraServer(); + } + }); + + it("round-trips a page through Obscura when installed", async (t) => { + killSharedObscuraServer(); + if (!HAS_OBSCURA) { + t.skip("obscura binary not installed"); + return; + } + try { + const connection = await connectObscuraBrowser(); + assert.ok(connection, "expected a live Obscura connection"); + const { browser } = connection; + const context = await browser.newContext({ userAgent: "obscura-integration-test" }); + const page = await context.newPage(); + await page.goto("https://example.com", { waitUntil: "domcontentloaded", timeout: 30000 }); + const title = await page.title(); + assert.equal(title, "Example Domain"); + await context.close(); + await browser.close(); + } finally { + killSharedObscuraServer(); + } + }); + + it("connects to an external endpoint without owning its process", async (t) => { + void t; + killSharedObscuraServer(); + if (!HAS_OBSCURA) { + t.skip("obscura binary not installed"); + return; + } + // Standalone server we own outside the module, referenced as "external". + const port = await freePort(); + const child = spawn( + process.env.OBSCURA_BIN ?? "obscura", + ["serve", "--port", String(port), "--host", "127.0.0.1"], + { stdio: ["ignore", "ignore", "pipe"] } + ); + const endpoint = `http://127.0.0.1:${port}`; + const ready = await waitForCdp(endpoint); + if (!ready) { + child.kill("SIGKILL"); + t.skip("external obscura server did not come up"); + return; + } + const originalBin = process.env.OBSCURA_BIN; + const originalEndpoint = process.env.OBSCURA_CDP_ENDPOINT; + process.env.OBSCURA_CDP_ENDPOINT = endpoint; + process.env.OBSCURA_BIN = "/nonexistent/obscura"; // force the external path + try { + const connection = await connectObscuraBrowser(); + assert.ok(connection); + assert.equal(connection.child, null, "external endpoint must not own a child process"); + assert.ok(connection.browser.version().length > 0); + await connection.browser.close(); + } finally { + if (originalBin === undefined) delete process.env.OBSCURA_BIN; + else process.env.OBSCURA_BIN = originalBin; + if (originalEndpoint === undefined) delete process.env.OBSCURA_CDP_ENDPOINT; + else process.env.OBSCURA_CDP_ENDPOINT = originalEndpoint; + child.kill("SIGKILL"); + killSharedObscuraServer(); + } + }); +}); diff --git a/tests/unit/webpack-create-require-warning.test.ts b/tests/unit/webpack-create-require-warning.test.ts index 7f454c614a..9c375c52bd 100644 --- a/tests/unit/webpack-create-require-warning.test.ts +++ b/tests/unit/webpack-create-require-warning.test.ts @@ -75,6 +75,10 @@ async function compileRuntimeRequireModules(): Promise { "sqlite-vec", "playwright", "wreq-js", + // browserPool.ts imports `./obscura.ts`. The isolated webpack compile + // has no repo tree, so treat the sibling as external instead of + // erroring "Can't resolve './obscura.ts'". + "./obscura.ts", ], externalsPresets: { node: true }, mode: "development",