feat(webfetch): support self-hosted FireCrawl instances (#5793)

Integrated into release/v3.8.44 — self-hosted FireCrawl support (FIRECRAWL_BASE_URL/FIRECRAWL_TIMEOUT_MS). Re-cut clean onto the release tip (branch was fossilized from a pre-v3.8.40 snapshot). Validated: 4 firecrawl tests green, env-doc-sync + docs-sync pass. UNSTABLE red is the inherited environmental setup-claude base-red.

.env.example

@@ -1039,6 +1039,12 @@ CURSOR_USER_AGENT="Cursor/3.4"
 # fallback when FETCH_TIMEOUT_MS is unset. Default: 120000 (2 min).
 # OMNIROUTE_DEFAULT_FETCH_TIMEOUT_MS=120000
 
+# ── Firecrawl web-fetch executor ──
+# Point at a self-hosted Firecrawl instance (defaults to the public cloud API).
+# When set to a non-cloud base URL, the API key becomes optional.
+# FIRECRAWL_BASE_URL=https://api.firecrawl.dev
+# FIRECRAWL_TIMEOUT_MS=30000            # Per-request timeout (default: 30000 = 30s)
+
 # ── ChatGPT TLS sidecar (Firefox-fingerprinted client) ──
 # Used by: open-sse/services/chatgptTlsClient.ts — wire-level timeout for
 # the bogdanfinn/tls-client koffi binding and the JS-side grace window

docs/reference/ENVIRONMENT.md

@@ -619,6 +619,8 @@ REQUEST_TIMEOUT_MS (global override)
 | `FETCH_KEEPALIVE_TIMEOUT_MS`              | `4000`               | Keep-alive socket idle timeout.                                                                                                                                 |
 | `TLS_CLIENT_TIMEOUT_MS`                   | = `FETCH_TIMEOUT_MS` | TLS fingerprint proxy (wreq-js) timeout.                                                                                                                        |
 | `API_BRIDGE_PROXY_TIMEOUT_MS`             | `30000`              | Proxy hop timeout for `/v1` bridge requests.                                                                                                                    |
+| `FIRECRAWL_BASE_URL`                      | `https://api.firecrawl.dev` | Point the Firecrawl web-fetch executor at a self-hosted instance (API key optional off-cloud).                                                          |
+| `FIRECRAWL_TIMEOUT_MS`                    | `30000`              | Per-request timeout for the Firecrawl web-fetch executor.                                                                                                       |
 | `API_BRIDGE_SERVER_REQUEST_TIMEOUT_MS`    | `300000`             | Overall server request timeout for the bridge.                                                                                                                  |
 | `API_BRIDGE_SERVER_HEADERS_TIMEOUT_MS`    | `60000`              | Time to send response headers via the bridge.                                                                                                                   |
 | `API_BRIDGE_SERVER_KEEPALIVE_TIMEOUT_MS`  | `5000`               | Bridge keep-alive idle timeout.                                                                                                                                 |

open-sse/executors/firecrawl-fetch.ts

@@ -6,13 +6,37 @@
  *
  * Free tier: 500 fetches/month, no credit card required.
  * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape
+ *
+ * Self-hosted: set FIRECRAWL_BASE_URL to point at a self-hosted Firecrawl
+ * instance (e.g. http://127.0.0.1:3002). The API key is only required against
+ * the default cloud base URL — self-hosted instances typically run with no
+ * auth in front of them, so credentials.apiKey becomes optional in that case.
  */
 
 import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
 import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
 
-const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1";
-const FIRECRAWL_TIMEOUT_MS = 30_000;
+const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev";
+const FIRECRAWL_DEFAULT_TIMEOUT_MS = 30_000;
+
+/** Resolve the configured Firecrawl base URL, falling back to the public cloud API. */
+function getFirecrawlBaseUrl(): string {
+  const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
+  return envBase ? envBase.replace(/\/+$/, "") : FIRECRAWL_DEFAULT_BASE_URL;
+}
+
+/** Whether the given base URL is the default Firecrawl cloud endpoint. */
+function isDefaultFirecrawlBaseUrl(baseUrl: string): boolean {
+  return baseUrl === FIRECRAWL_DEFAULT_BASE_URL;
+}
+
+/** Resolve the configured request timeout, falling back to a sane default. */
+function getFirecrawlTimeoutMs(): number {
+  const raw = process.env.FIRECRAWL_TIMEOUT_MS;
+  if (!raw) return FIRECRAWL_DEFAULT_TIMEOUT_MS;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : FIRECRAWL_DEFAULT_TIMEOUT_MS;
+}
 
 function mapFormat(format: WebFetchFormat): string {
   switch (format) {
@@ -43,7 +67,12 @@ interface FirecrawlScrapeOptions {
 export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebFetchResult> {
   const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts;
 
-  if (!credentials.apiKey) {
+  const baseUrl = getFirecrawlBaseUrl();
+  const isDefaultBaseUrl = isDefaultFirecrawlBaseUrl(baseUrl);
+
+  // The API key is mandatory for the public Firecrawl cloud API, but optional
+  // once a custom (self-hosted) base URL is configured.
+  if (isDefaultBaseUrl && !credentials.apiKey) {
     const body = buildErrorBody(401, "Firecrawl API key required");
     return { success: false, status: 401, error: body.error.message };
   }
@@ -71,15 +100,17 @@ export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebF
   }
 
   const controller = new AbortController();
-  const timeoutId = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS);
+  const timeoutId = setTimeout(() => controller.abort(), getFirecrawlTimeoutMs());
 
   try {
-    const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, {
+    const headers: Record<string, string> = { "Content-Type": "application/json" };
+    if (credentials.apiKey) {
+      headers.Authorization = `Bearer ${credentials.apiKey}`;
+    }
+
+    const response = await fetch(`${baseUrl}/v1/scrape`, {
       method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${credentials.apiKey}`,
-      },
+      headers,
       body: JSON.stringify(requestBody),
       signal: controller.signal,
     });

tests/unit/executors/firecrawl-fetch.test.ts

@@ -0,0 +1,151 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { firecrawlFetch } = await import("../../../open-sse/executors/firecrawl-fetch.ts");
+
+// ── #2253 (decolua/9router): self-hosted Firecrawl support ────────────────────
+// FIRECRAWL_BASE_URL lets operators point the executor at a self-hosted instance.
+// The API key stays required for the default cloud endpoint, but becomes optional
+// once a custom base URL is configured (self-hosted instances usually run with no
+// auth in front of them).
+
+function withEnv(vars: Record<string, string | undefined>, fn: () => Promise<void>) {
+  const original: Record<string, string | undefined> = {};
+  for (const key of Object.keys(vars)) {
+    original[key] = process.env[key];
+  }
+  for (const [key, value] of Object.entries(vars)) {
+    if (value === undefined) delete process.env[key];
+    else process.env[key] = value;
+  }
+  return fn().finally(() => {
+    for (const [key, value] of Object.entries(original)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+  });
+}
+
+test("firecrawlFetch routes to FIRECRAWL_BASE_URL when set", async () => {
+  await withEnv({ FIRECRAWL_BASE_URL: "http://127.0.0.1:3002" }, async () => {
+    const originalFetch = globalThis.fetch;
+    let capturedUrl = "";
+
+    globalThis.fetch = async (url) => {
+      capturedUrl = String(url);
+      return new Response(JSON.stringify({ data: { markdown: "# Self-hosted" } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    };
+
+    try {
+      const result = await firecrawlFetch({
+        url: "https://example.com",
+        format: "markdown",
+        depth: 0,
+        includeMetadata: false,
+        credentials: { apiKey: "any-key" },
+      });
+
+      assert.equal(result.success, true);
+      assert.equal(capturedUrl, "http://127.0.0.1:3002/v1/scrape");
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+test("firecrawlFetch allows missing apiKey when FIRECRAWL_BASE_URL is custom", async () => {
+  await withEnv({ FIRECRAWL_BASE_URL: "http://127.0.0.1:3002" }, async () => {
+    const originalFetch = globalThis.fetch;
+    let capturedHeaders: Record<string, string> = {};
+
+    globalThis.fetch = async (_url, init = {}) => {
+      capturedHeaders = (init as RequestInit).headers as Record<string, string>;
+      return new Response(JSON.stringify({ data: { markdown: "# Self-hosted" } }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    };
+
+    try {
+      const result = await firecrawlFetch({
+        url: "https://example.com",
+        format: "markdown",
+        depth: 0,
+        includeMetadata: false,
+        credentials: {},
+      });
+
+      assert.equal(result.success, true, "custom base URL must not require an API key");
+      assert.equal(
+        capturedHeaders["Authorization"],
+        undefined,
+        "no Authorization header should be sent without an apiKey"
+      );
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+});
+
+test("firecrawlFetch still requires apiKey against the default cloud base URL", async () => {
+  await withEnv({ FIRECRAWL_BASE_URL: undefined }, async () => {
+    const result = await firecrawlFetch({
+      url: "https://example.com",
+      format: "markdown",
+      depth: 0,
+      includeMetadata: false,
+      credentials: {},
+    });
+
+    assert.equal(result.success, false);
+    assert.equal(result.status, 401);
+    assert.ok(!result.error?.includes("at /"), "error must not contain stack trace");
+  });
+});
+
+test("firecrawlFetch honors FIRECRAWL_TIMEOUT_MS override", async () => {
+  await withEnv(
+    { FIRECRAWL_BASE_URL: "http://127.0.0.1:3002", FIRECRAWL_TIMEOUT_MS: "5" },
+    async () => {
+      const originalFetch = globalThis.fetch;
+
+      globalThis.fetch = async (_url, init = {}) => {
+        const signal = (init as RequestInit).signal;
+        return new Promise((resolve, reject) => {
+          const timer = setTimeout(
+            () =>
+              resolve(
+                new Response(JSON.stringify({ data: { markdown: "late" } }), {
+                  status: 200,
+                  headers: { "content-type": "application/json" },
+                })
+              ),
+            50
+          );
+          signal?.addEventListener("abort", () => {
+            clearTimeout(timer);
+            reject(new DOMException("Aborted", "AbortError"));
+          });
+        });
+      };
+
+      try {
+        const result = await firecrawlFetch({
+          url: "https://example.com",
+          format: "markdown",
+          depth: 0,
+          includeMetadata: false,
+          credentials: {},
+        });
+
+        assert.equal(result.success, false);
+        assert.equal(result.status, 504);
+      } finally {
+        globalThis.fetch = originalFetch;
+      }
+    }
+  );
+});