From babfa7b8febf5248b21c6dffec799d04778dad16 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:59:50 -0300 Subject: [PATCH] feat(webfetch): support self-hosted FireCrawl instances (#5793) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.44 — self-hosted FireCrawl support (FIRECRAWL_BASE_URL/FIRECRAWL_TIMEOUT_MS). Re-cut clean onto the release tip (branch was fossilized from a pre-v3.8.40 snapshot). Validated: 4 firecrawl tests green, env-doc-sync + docs-sync pass. UNSTABLE red is the inherited environmental setup-claude base-red. --- .env.example | 6 + docs/reference/ENVIRONMENT.md | 2 + open-sse/executors/firecrawl-fetch.ts | 49 ++++-- tests/unit/executors/firecrawl-fetch.test.ts | 151 +++++++++++++++++++ 4 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tests/unit/executors/firecrawl-fetch.test.ts diff --git a/.env.example b/.env.example index 5a1fd9d9c1..4086714a6f 100644 --- a/.env.example +++ b/.env.example @@ -1039,6 +1039,12 @@ CURSOR_USER_AGENT="Cursor/3.4" # fallback when FETCH_TIMEOUT_MS is unset. Default: 120000 (2 min). # OMNIROUTE_DEFAULT_FETCH_TIMEOUT_MS=120000 +# ── Firecrawl web-fetch executor ── +# Point at a self-hosted Firecrawl instance (defaults to the public cloud API). +# When set to a non-cloud base URL, the API key becomes optional. +# FIRECRAWL_BASE_URL=https://api.firecrawl.dev +# FIRECRAWL_TIMEOUT_MS=30000 # Per-request timeout (default: 30000 = 30s) + # ── ChatGPT TLS sidecar (Firefox-fingerprinted client) ── # Used by: open-sse/services/chatgptTlsClient.ts — wire-level timeout for # the bogdanfinn/tls-client koffi binding and the JS-side grace window diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 9404f33517..ff634a7f6e 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -619,6 +619,8 @@ REQUEST_TIMEOUT_MS (global override) | `FETCH_KEEPALIVE_TIMEOUT_MS` | `4000` | Keep-alive socket idle timeout. | | `TLS_CLIENT_TIMEOUT_MS` | = `FETCH_TIMEOUT_MS` | TLS fingerprint proxy (wreq-js) timeout. | | `API_BRIDGE_PROXY_TIMEOUT_MS` | `30000` | Proxy hop timeout for `/v1` bridge requests. | +| `FIRECRAWL_BASE_URL` | `https://api.firecrawl.dev` | Point the Firecrawl web-fetch executor at a self-hosted instance (API key optional off-cloud). | +| `FIRECRAWL_TIMEOUT_MS` | `30000` | Per-request timeout for the Firecrawl web-fetch executor. | | `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS` | `300000` | Overall server request timeout for the bridge. | | `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS` | `60000` | Time to send response headers via the bridge. | | `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS` | `5000` | Bridge keep-alive idle timeout. | diff --git a/open-sse/executors/firecrawl-fetch.ts b/open-sse/executors/firecrawl-fetch.ts index c8ae752466..b2c59386ec 100644 --- a/open-sse/executors/firecrawl-fetch.ts +++ b/open-sse/executors/firecrawl-fetch.ts @@ -6,13 +6,37 @@ * * Free tier: 500 fetches/month, no credit card required. * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape + * + * Self-hosted: set FIRECRAWL_BASE_URL to point at a self-hosted Firecrawl + * instance (e.g. http://127.0.0.1:3002). The API key is only required against + * the default cloud base URL — self-hosted instances typically run with no + * auth in front of them, so credentials.apiKey becomes optional in that case. */ import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts"; import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts"; -const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1"; -const FIRECRAWL_TIMEOUT_MS = 30_000; +const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev"; +const FIRECRAWL_DEFAULT_TIMEOUT_MS = 30_000; + +/** Resolve the configured Firecrawl base URL, falling back to the public cloud API. */ +function getFirecrawlBaseUrl(): string { + const envBase = process.env.FIRECRAWL_BASE_URL?.trim(); + return envBase ? envBase.replace(/\/+$/, "") : FIRECRAWL_DEFAULT_BASE_URL; +} + +/** Whether the given base URL is the default Firecrawl cloud endpoint. */ +function isDefaultFirecrawlBaseUrl(baseUrl: string): boolean { + return baseUrl === FIRECRAWL_DEFAULT_BASE_URL; +} + +/** Resolve the configured request timeout, falling back to a sane default. */ +function getFirecrawlTimeoutMs(): number { + const raw = process.env.FIRECRAWL_TIMEOUT_MS; + if (!raw) return FIRECRAWL_DEFAULT_TIMEOUT_MS; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : FIRECRAWL_DEFAULT_TIMEOUT_MS; +} function mapFormat(format: WebFetchFormat): string { switch (format) { @@ -43,7 +67,12 @@ interface FirecrawlScrapeOptions { export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise { const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts; - if (!credentials.apiKey) { + const baseUrl = getFirecrawlBaseUrl(); + const isDefaultBaseUrl = isDefaultFirecrawlBaseUrl(baseUrl); + + // The API key is mandatory for the public Firecrawl cloud API, but optional + // once a custom (self-hosted) base URL is configured. + if (isDefaultBaseUrl && !credentials.apiKey) { const body = buildErrorBody(401, "Firecrawl API key required"); return { success: false, status: 401, error: body.error.message }; } @@ -71,15 +100,17 @@ export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise controller.abort(), FIRECRAWL_TIMEOUT_MS); + const timeoutId = setTimeout(() => controller.abort(), getFirecrawlTimeoutMs()); try { - const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, { + const headers: Record = { "Content-Type": "application/json" }; + if (credentials.apiKey) { + headers.Authorization = `Bearer ${credentials.apiKey}`; + } + + const response = await fetch(`${baseUrl}/v1/scrape`, { method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${credentials.apiKey}`, - }, + headers, body: JSON.stringify(requestBody), signal: controller.signal, }); diff --git a/tests/unit/executors/firecrawl-fetch.test.ts b/tests/unit/executors/firecrawl-fetch.test.ts new file mode 100644 index 0000000000..912cbc0eb9 --- /dev/null +++ b/tests/unit/executors/firecrawl-fetch.test.ts @@ -0,0 +1,151 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { firecrawlFetch } = await import("../../../open-sse/executors/firecrawl-fetch.ts"); + +// ── #2253 (decolua/9router): self-hosted Firecrawl support ──────────────────── +// FIRECRAWL_BASE_URL lets operators point the executor at a self-hosted instance. +// The API key stays required for the default cloud endpoint, but becomes optional +// once a custom base URL is configured (self-hosted instances usually run with no +// auth in front of them). + +function withEnv(vars: Record, fn: () => Promise) { + const original: Record = {}; + for (const key of Object.keys(vars)) { + original[key] = process.env[key]; + } + for (const [key, value] of Object.entries(vars)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + return fn().finally(() => { + for (const [key, value] of Object.entries(original)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); +} + +test("firecrawlFetch routes to FIRECRAWL_BASE_URL when set", async () => { + await withEnv({ FIRECRAWL_BASE_URL: "http://127.0.0.1:3002" }, async () => { + const originalFetch = globalThis.fetch; + let capturedUrl = ""; + + globalThis.fetch = async (url) => { + capturedUrl = String(url); + return new Response(JSON.stringify({ data: { markdown: "# Self-hosted" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: { apiKey: "any-key" }, + }); + + assert.equal(result.success, true); + assert.equal(capturedUrl, "http://127.0.0.1:3002/v1/scrape"); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); + +test("firecrawlFetch allows missing apiKey when FIRECRAWL_BASE_URL is custom", async () => { + await withEnv({ FIRECRAWL_BASE_URL: "http://127.0.0.1:3002" }, async () => { + const originalFetch = globalThis.fetch; + let capturedHeaders: Record = {}; + + globalThis.fetch = async (_url, init = {}) => { + capturedHeaders = (init as RequestInit).headers as Record; + return new Response(JSON.stringify({ data: { markdown: "# Self-hosted" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, true, "custom base URL must not require an API key"); + assert.equal( + capturedHeaders["Authorization"], + undefined, + "no Authorization header should be sent without an apiKey" + ); + } finally { + globalThis.fetch = originalFetch; + } + }); +}); + +test("firecrawlFetch still requires apiKey against the default cloud base URL", async () => { + await withEnv({ FIRECRAWL_BASE_URL: undefined }, async () => { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace"); + }); +}); + +test("firecrawlFetch honors FIRECRAWL_TIMEOUT_MS override", async () => { + await withEnv( + { FIRECRAWL_BASE_URL: "http://127.0.0.1:3002", FIRECRAWL_TIMEOUT_MS: "5" }, + async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async (_url, init = {}) => { + const signal = (init as RequestInit).signal; + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => + resolve( + new Response(JSON.stringify({ data: { markdown: "late" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }) + ), + 50 + ); + signal?.addEventListener("abort", () => { + clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }); + }); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 504); + } finally { + globalThis.fetch = originalFetch; + } + } + ); +});