feat(providers): add comprehensive support for self-hosted Firecrawl via FIRECRAWL_BASE_URL and custom base URLs (#9052)

Validated in post-merge-train sweep (boards clean on release/v3.8.50 tip)

open-sse/executors/firecrawl-fetch.ts

@@ -20,9 +20,15 @@ const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev";
 const FIRECRAWL_DEFAULT_TIMEOUT_MS = 30_000;
 
 /** Resolve the configured Firecrawl base URL, falling back to the public cloud API. */
-function getFirecrawlBaseUrl(): string {
+function getFirecrawlBaseUrl(credentials?: WebFetchCredentials): string {
   const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
-  return envBase ? envBase.replace(/\/+$/, "") : FIRECRAWL_DEFAULT_BASE_URL;
+  if (envBase) return envBase.replace(/\/+$/, "");
+  const providerData = credentials?.providerSpecificData;
+  const credBase = typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl;
+  if (typeof credBase === "string" && credBase.trim()) {
+    return credBase.trim().replace(/\/+$/, "");
+  }
+  return FIRECRAWL_DEFAULT_BASE_URL;
 }
 
 /** Whether the given base URL is the default Firecrawl cloud endpoint. */
@@ -67,7 +73,7 @@ interface FirecrawlScrapeOptions {
 export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebFetchResult> {
   const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts;
 
-  const baseUrl = getFirecrawlBaseUrl();
+  const baseUrl = getFirecrawlBaseUrl(credentials);
   const isDefaultBaseUrl = isDefaultFirecrawlBaseUrl(baseUrl);
 
   // The API key is mandatory for the public Firecrawl cloud API, but optional

open-sse/handlers/search/firecrawlSearch.ts

@@ -5,6 +5,8 @@ export interface FirecrawlSearchParams {
   searchType: string;
   maxResults: number;
   token?: string;
+  baseUrl?: string;
+  providerSpecificData?: Record<string, unknown>;
   country?: string;
   language?: string;
   timeRange?: string;
@@ -68,7 +70,11 @@ export function buildFirecrawlSearchRequest(
   params: FirecrawlSearchParams
 ): { url: string; init: RequestInit } {
   const envBase = process.env.FIRECRAWL_BASE_URL?.trim().replace(/\/+$/, "");
-  const url = envBase ? `${envBase}/v2/search` : config.baseUrl;
+  const providerData = params.providerSpecificData as Record<string, unknown> | undefined;
+  const paramBase = typeof params.baseUrl === "string" ? params.baseUrl : providerData?.baseUrl;
+  const customBase = typeof paramBase === "string" && paramBase.trim() ? paramBase.trim().replace(/\/+$/, "") : undefined;
+  const rawBase = envBase || customBase;
+  const url = rawBase ? `${rawBase}/v2/search` : config.baseUrl;
   const { includes, excludes } = parseDomainFilter(params.domainFilter);
   const source = params.searchType === "news" ? "news" : "web";
 

open-sse/handlers/webFetch.ts

@@ -50,6 +50,8 @@ export interface WebFetchResult {
 
 export interface WebFetchCredentials {
   apiKey?: string;
+  baseUrl?: string;
+  providerSpecificData?: Record<string, unknown>;
 }
 
 const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const;

open-sse/services/firecrawlQuotaFetcher.ts

@@ -104,6 +104,19 @@ export function parseFirecrawlCreditUsage(data: unknown): FirecrawlQuota | null
   };
 }
 
+export function getFirecrawlBaseUrl(connection?: Record<string, unknown>): string | null {
+  const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
+  if (envBase && !envBase.includes("api.firecrawl.dev")) {
+    return envBase.replace(/\/+$/, "");
+  }
+  const providerData = toRecord(connection?.providerSpecificData);
+  const connBase = typeof connection?.baseUrl === "string" ? connection.baseUrl : providerData?.baseUrl;
+  if (typeof connBase === "string" && connBase.trim() && !connBase.includes("api.firecrawl.dev")) {
+    return connBase.trim().replace(/\/+$/, "");
+  }
+  return null;
+}
+
 export async function fetchFirecrawlQuota(
   connectionId: string,
   connection?: Record<string, unknown>
@@ -113,6 +126,21 @@ export async function fetchFirecrawlQuota(
     return cached.quota;
   }
 
+  const customBase = getFirecrawlBaseUrl(connection);
+  if (customBase) {
+    return {
+      used: 0,
+      total: 0,
+      percentUsed: 0,
+      resetAt: null,
+      remainingCredits: 0,
+      planCredits: 0,
+      extraCreditsInferred: 0,
+      overPlan: false,
+      limitReached: false,
+    };
+  }
+
   const apiKey = extractFirecrawlApiKey(connection);
   if (!apiKey) {
     quotaCache.set(connectionId, { quota: null, fetchedAt: Date.now() });

open-sse/services/usage.ts

@@ -228,7 +228,7 @@ export async function getUsageForProvider(
     case "ha":
       return await getHyperAgentUsage(apiKey || accessToken, providerSpecificData);
     case "firecrawl":
-      return await getFirecrawlUsage(id || "", apiKey);
+      return await getFirecrawlUsage(id || "", apiKey, connection);
     default:
       return { message: `Usage API not implemented for ${provider}` };
   }

open-sse/services/usage/firecrawl.ts

@@ -5,7 +5,7 @@
  * credits into the standard `{ plan, quotas }` response.
  */
 
-import { fetchFirecrawlQuota, type FirecrawlQuota } from "../firecrawlQuotaFetcher.ts";
+import { fetchFirecrawlQuota, getFirecrawlBaseUrl, type FirecrawlQuota } from "../firecrawlQuotaFetcher.ts";
 import { createQuotaFromUsage, parseResetTime } from "./quota.ts";
 
 function createFirecrawlPlanQuota(q: FirecrawlQuota) {
@@ -29,13 +29,22 @@ function createFirecrawlPlanQuota(q: FirecrawlQuota) {
   };
 }
 
-export async function getFirecrawlUsage(connectionId: string, apiKey?: string) {
+export async function getFirecrawlUsage(connectionId: string, apiKey?: string, connection?: Record<string, unknown>) {
   if (!connectionId) {
     return { message: "Firecrawl: connection id unavailable." };
   }
 
+  const customBase = getFirecrawlBaseUrl(connection);
+  if (customBase) {
+    return {
+      plan: "Firecrawl · Self-Hosted Local",
+      quotas: {},
+      message: `Connected to self-hosted Firecrawl instance (${customBase})`,
+    };
+  }
+
   try {
-    const live = await fetchFirecrawlQuota(connectionId, { apiKey });
+    const live = await fetchFirecrawlQuota(connectionId, connection);
     if (!live) {
       return { message: "Firecrawl API key not available or credit usage unavailable." };
     }

src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts

@@ -229,6 +229,7 @@ export const CONFIGURABLE_BASE_URL_PROVIDERS = new Set([
   "databricks",
   "snowflake",
   "searxng-search",
+  "firecrawl",
   "petals",
   "comfyui",
   // #7447 — Moonshot/Kimi's international host (api.moonshot.ai) rejects
@@ -250,6 +251,7 @@ export const DEFAULT_PROVIDER_BASE_URLS: Record<string, string> = {
   "xiaomi-mimo": "https://token-plan-sgp.xiaomimimo.com/v1",
   siliconflow: "https://api.siliconflow.com/v1",
   "searxng-search": "http://localhost:8888/search",
+  firecrawl: "https://api.firecrawl.dev",
   petals: "https://chat.petals.dev/api/v1/generate",
   comfyui: "http://localhost:8188",
   // #7447 — default stays the international host so existing/new
@@ -334,6 +336,8 @@ export function getProviderBaseUrlHint(
       return t ? t("snowflakeBaseUrlHint") : undefined;
     case "searxng-search":
       return t ? t("searxngBaseUrlHint") : undefined;
+    case "firecrawl":
+      return t ? t("firecrawlBaseUrlHint") : undefined;
     default:
       return undefined;
   }
@@ -349,6 +353,7 @@ export function getProviderBaseUrlPlaceholder(providerId?: string | null) {
     case "bailian-coding-plan":
     case "xiaomi-mimo":
     case "comfyui":
+    case "firecrawl":
       return getProviderBaseUrlDefault(providerId);
     case "siliconflow":
       return "https://api.siliconflow.cn/v1";

src/lib/providers/validation/searchProviders.ts

@@ -173,14 +173,26 @@ export const SEARCH_VALIDATOR_CONFIGS: Record<
   // Probe each provider's real fetch endpoint with the same Bearer auth the executor
   // uses; validateSearchProvider maps 200/<500 → valid, 401/403 → invalid key,
   // >=500 → failure (a credit-exhausted / rate-limited key still validates).
-  firecrawl: (apiKey) => ({
+  firecrawl: (apiKey, providerSpecificData = {}) => {
-    url: "https://api.firecrawl.dev/v1/scrape",
+    const envBase = process.env.FIRECRAWL_BASE_URL?.trim();
-    init: {
+    const baseUrl = envBase
-      method: "POST",
+      ? envBase.replace(/\/+$/, "")
-      headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` },
+      : typeof providerSpecificData?.baseUrl === "string" && providerSpecificData.baseUrl.trim()
-      body: JSON.stringify({ url: "https://example.com", formats: ["markdown"] }),
+        ? providerSpecificData.baseUrl.trim().replace(/\/+$/, "")
-    },
+        : "https://api.firecrawl.dev";
-  }),
+    const headers: Record<string, string> = { "Content-Type": "application/json" };
+    if (apiKey) {
+      headers["Authorization"] = `Bearer ${apiKey}`;
+    }
+    return {
+      url: `${baseUrl}/v1/scrape`,
+      init: {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ url: "https://example.com", formats: ["markdown"] }),
+      },
+    };
+  },
   "jina-reader": (apiKey) => ({
     url: "https://r.jina.ai/https://example.com",
     init: {

src/shared/constants/providers.ts

@@ -179,6 +179,7 @@ export function isSelfHostedChatProvider(providerId: unknown): boolean {
 // cyclomatic complexity flat as this list grows — see g4f.space (#6650).
 const EXPLICIT_OPTIONAL_APIKEY_PROVIDER_IDS = new Set([
   "searxng-search",
+  "firecrawl",
   "pollinations",
   "copilot-web",
   "hackclub",

src/shared/constants/providers/search.ts

@@ -69,6 +69,7 @@ export const SEARCH_PROVIDERS = {
     textIcon: "FC",
     website: "https://firecrawl.dev",
     hasFree: true,
+    authHint: "API key from firecrawl.dev/app/api-keys (or set your self-hosted Firecrawl base URL)",
     notice: {
       text: "Free tier: 1,000 credits/month. Powers /v1/web/fetch and /v1/search.",
       apiKeyUrl: "https://firecrawl.dev/app/api-keys",

tests/unit/firecrawl-quota-fetcher.test.ts

@@ -153,3 +153,25 @@ test("registerFirecrawlQuotaFetcher registers firecrawl for preflight", async ()
 
   invalidateFirecrawlQuotaCache(connectionId);
 });
+
+test("fetchFirecrawlQuota bypasses cloud fetch when FIRECRAWL_BASE_URL is set", async () => {
+  const originalEnv = process.env.FIRECRAWL_BASE_URL;
+  try {
+    process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/";
+    const connectionId = `fc-selfhosted-${Date.now()}`;
+    let fetchCalled = false;
+    globalThis.fetch = async () => {
+      fetchCalled = true;
+      return creditUsageResponse(100, 1000);
+    };
+
+    const quota = await fetchFirecrawlQuota(connectionId, { apiKey: "local-key" });
+    assert.ok(quota);
+    assert.equal(quota!.used, 0);
+    assert.equal(quota!.total, 0);
+    assert.equal(fetchCalled, false);
+    invalidateFirecrawlQuotaCache(connectionId);
+  } finally {
+    process.env.FIRECRAWL_BASE_URL = originalEnv;
+  }
+});