From 91bb6aa6198463c9e04dcbfc0e0bc43a663957a2 Mon Sep 17 00:00:00 2001 From: Alexey Gusev Date: Thu, 6 Aug 2026 17:07:42 +0300 Subject: [PATCH] feat(providers): add comprehensive support for self-hosted Firecrawl via FIRECRAWL_BASE_URL and custom base URLs (#9052) Validated in post-merge-train sweep (boards clean on release/v3.8.50 tip) --- open-sse/executors/firecrawl-fetch.ts | 12 ++++++-- open-sse/handlers/search/firecrawlSearch.ts | 8 +++++- open-sse/handlers/webFetch.ts | 2 ++ open-sse/services/firecrawlQuotaFetcher.ts | 28 +++++++++++++++++++ open-sse/services/usage.ts | 2 +- open-sse/services/usage/firecrawl.ts | 15 ++++++++-- .../providers/[id]/providerPageHelpers.ts | 5 ++++ .../providers/validation/searchProviders.ts | 28 +++++++++++++------ src/shared/constants/providers.ts | 1 + src/shared/constants/providers/search.ts | 1 + tests/unit/firecrawl-quota-fetcher.test.ts | 22 +++++++++++++++ 11 files changed, 108 insertions(+), 16 deletions(-) diff --git a/open-sse/executors/firecrawl-fetch.ts b/open-sse/executors/firecrawl-fetch.ts index 00c6c3cbea..aa54ee89c4 100644 --- a/open-sse/executors/firecrawl-fetch.ts +++ b/open-sse/executors/firecrawl-fetch.ts @@ -20,9 +20,15 @@ const FIRECRAWL_DEFAULT_BASE_URL = "https://api.firecrawl.dev"; const FIRECRAWL_DEFAULT_TIMEOUT_MS = 30_000; /** Resolve the configured Firecrawl base URL, falling back to the public cloud API. */ -function getFirecrawlBaseUrl(): string { +function getFirecrawlBaseUrl(credentials?: WebFetchCredentials): string { const envBase = process.env.FIRECRAWL_BASE_URL?.trim(); - return envBase ? envBase.replace(/\/+$/, "") : FIRECRAWL_DEFAULT_BASE_URL; + if (envBase) return envBase.replace(/\/+$/, ""); + const providerData = credentials?.providerSpecificData; + const credBase = typeof credentials?.baseUrl === "string" ? credentials.baseUrl : providerData?.baseUrl; + if (typeof credBase === "string" && credBase.trim()) { + return credBase.trim().replace(/\/+$/, ""); + } + return FIRECRAWL_DEFAULT_BASE_URL; } /** Whether the given base URL is the default Firecrawl cloud endpoint. */ @@ -67,7 +73,7 @@ interface FirecrawlScrapeOptions { export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise { const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts; - const baseUrl = getFirecrawlBaseUrl(); + const baseUrl = getFirecrawlBaseUrl(credentials); const isDefaultBaseUrl = isDefaultFirecrawlBaseUrl(baseUrl); // The API key is mandatory for the public Firecrawl cloud API, but optional diff --git a/open-sse/handlers/search/firecrawlSearch.ts b/open-sse/handlers/search/firecrawlSearch.ts index 350f7b2bb8..9885486e94 100644 --- a/open-sse/handlers/search/firecrawlSearch.ts +++ b/open-sse/handlers/search/firecrawlSearch.ts @@ -5,6 +5,8 @@ export interface FirecrawlSearchParams { searchType: string; maxResults: number; token?: string; + baseUrl?: string; + providerSpecificData?: Record; country?: string; language?: string; timeRange?: string; @@ -68,7 +70,11 @@ export function buildFirecrawlSearchRequest( params: FirecrawlSearchParams ): { url: string; init: RequestInit } { const envBase = process.env.FIRECRAWL_BASE_URL?.trim().replace(/\/+$/, ""); - const url = envBase ? `${envBase}/v2/search` : config.baseUrl; + const providerData = params.providerSpecificData as Record | undefined; + const paramBase = typeof params.baseUrl === "string" ? params.baseUrl : providerData?.baseUrl; + const customBase = typeof paramBase === "string" && paramBase.trim() ? paramBase.trim().replace(/\/+$/, "") : undefined; + const rawBase = envBase || customBase; + const url = rawBase ? `${rawBase}/v2/search` : config.baseUrl; const { includes, excludes } = parseDomainFilter(params.domainFilter); const source = params.searchType === "news" ? "news" : "web"; diff --git a/open-sse/handlers/webFetch.ts b/open-sse/handlers/webFetch.ts index 27179ab289..8b6c25660a 100644 --- a/open-sse/handlers/webFetch.ts +++ b/open-sse/handlers/webFetch.ts @@ -50,6 +50,8 @@ export interface WebFetchResult { export interface WebFetchCredentials { apiKey?: string; + baseUrl?: string; + providerSpecificData?: Record; } const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const; diff --git a/open-sse/services/firecrawlQuotaFetcher.ts b/open-sse/services/firecrawlQuotaFetcher.ts index d86e1ad116..9f0784fa06 100644 --- a/open-sse/services/firecrawlQuotaFetcher.ts +++ b/open-sse/services/firecrawlQuotaFetcher.ts @@ -104,6 +104,19 @@ export function parseFirecrawlCreditUsage(data: unknown): FirecrawlQuota | null }; } +export function getFirecrawlBaseUrl(connection?: Record): string | null { + const envBase = process.env.FIRECRAWL_BASE_URL?.trim(); + if (envBase && !envBase.includes("api.firecrawl.dev")) { + return envBase.replace(/\/+$/, ""); + } + const providerData = toRecord(connection?.providerSpecificData); + const connBase = typeof connection?.baseUrl === "string" ? connection.baseUrl : providerData?.baseUrl; + if (typeof connBase === "string" && connBase.trim() && !connBase.includes("api.firecrawl.dev")) { + return connBase.trim().replace(/\/+$/, ""); + } + return null; +} + export async function fetchFirecrawlQuota( connectionId: string, connection?: Record @@ -113,6 +126,21 @@ export async function fetchFirecrawlQuota( return cached.quota; } + const customBase = getFirecrawlBaseUrl(connection); + if (customBase) { + return { + used: 0, + total: 0, + percentUsed: 0, + resetAt: null, + remainingCredits: 0, + planCredits: 0, + extraCreditsInferred: 0, + overPlan: false, + limitReached: false, + }; + } + const apiKey = extractFirecrawlApiKey(connection); if (!apiKey) { quotaCache.set(connectionId, { quota: null, fetchedAt: Date.now() }); diff --git a/open-sse/services/usage.ts b/open-sse/services/usage.ts index 1123ca4951..679d4f263d 100644 --- a/open-sse/services/usage.ts +++ b/open-sse/services/usage.ts @@ -228,7 +228,7 @@ export async function getUsageForProvider( case "ha": return await getHyperAgentUsage(apiKey || accessToken, providerSpecificData); case "firecrawl": - return await getFirecrawlUsage(id || "", apiKey); + return await getFirecrawlUsage(id || "", apiKey, connection); default: return { message: `Usage API not implemented for ${provider}` }; } diff --git a/open-sse/services/usage/firecrawl.ts b/open-sse/services/usage/firecrawl.ts index ea2ae1618f..09ea1f5211 100644 --- a/open-sse/services/usage/firecrawl.ts +++ b/open-sse/services/usage/firecrawl.ts @@ -5,7 +5,7 @@ * credits into the standard `{ plan, quotas }` response. */ -import { fetchFirecrawlQuota, type FirecrawlQuota } from "../firecrawlQuotaFetcher.ts"; +import { fetchFirecrawlQuota, getFirecrawlBaseUrl, type FirecrawlQuota } from "../firecrawlQuotaFetcher.ts"; import { createQuotaFromUsage, parseResetTime } from "./quota.ts"; function createFirecrawlPlanQuota(q: FirecrawlQuota) { @@ -29,13 +29,22 @@ function createFirecrawlPlanQuota(q: FirecrawlQuota) { }; } -export async function getFirecrawlUsage(connectionId: string, apiKey?: string) { +export async function getFirecrawlUsage(connectionId: string, apiKey?: string, connection?: Record) { if (!connectionId) { return { message: "Firecrawl: connection id unavailable." }; } + const customBase = getFirecrawlBaseUrl(connection); + if (customBase) { + return { + plan: "Firecrawl · Self-Hosted Local", + quotas: {}, + message: `Connected to self-hosted Firecrawl instance (${customBase})`, + }; + } + try { - const live = await fetchFirecrawlQuota(connectionId, { apiKey }); + const live = await fetchFirecrawlQuota(connectionId, connection); if (!live) { return { message: "Firecrawl API key not available or credit usage unavailable." }; } diff --git a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts index 7445f36206..7812293486 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts +++ b/src/app/(dashboard)/dashboard/providers/[id]/providerPageHelpers.ts @@ -229,6 +229,7 @@ export const CONFIGURABLE_BASE_URL_PROVIDERS = new Set([ "databricks", "snowflake", "searxng-search", + "firecrawl", "petals", "comfyui", // #7447 — Moonshot/Kimi's international host (api.moonshot.ai) rejects @@ -250,6 +251,7 @@ export const DEFAULT_PROVIDER_BASE_URLS: Record = { "xiaomi-mimo": "https://token-plan-sgp.xiaomimimo.com/v1", siliconflow: "https://api.siliconflow.com/v1", "searxng-search": "http://localhost:8888/search", + firecrawl: "https://api.firecrawl.dev", petals: "https://chat.petals.dev/api/v1/generate", comfyui: "http://localhost:8188", // #7447 — default stays the international host so existing/new @@ -334,6 +336,8 @@ export function getProviderBaseUrlHint( return t ? t("snowflakeBaseUrlHint") : undefined; case "searxng-search": return t ? t("searxngBaseUrlHint") : undefined; + case "firecrawl": + return t ? t("firecrawlBaseUrlHint") : undefined; default: return undefined; } @@ -349,6 +353,7 @@ export function getProviderBaseUrlPlaceholder(providerId?: string | null) { case "bailian-coding-plan": case "xiaomi-mimo": case "comfyui": + case "firecrawl": return getProviderBaseUrlDefault(providerId); case "siliconflow": return "https://api.siliconflow.cn/v1"; diff --git a/src/lib/providers/validation/searchProviders.ts b/src/lib/providers/validation/searchProviders.ts index ecfa79b594..302a792266 100644 --- a/src/lib/providers/validation/searchProviders.ts +++ b/src/lib/providers/validation/searchProviders.ts @@ -173,14 +173,26 @@ export const SEARCH_VALIDATOR_CONFIGS: Record< // Probe each provider's real fetch endpoint with the same Bearer auth the executor // uses; validateSearchProvider maps 200/<500 → valid, 401/403 → invalid key, // >=500 → failure (a credit-exhausted / rate-limited key still validates). - firecrawl: (apiKey) => ({ - url: "https://api.firecrawl.dev/v1/scrape", - init: { - method: "POST", - headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` }, - body: JSON.stringify({ url: "https://example.com", formats: ["markdown"] }), - }, - }), + firecrawl: (apiKey, providerSpecificData = {}) => { + const envBase = process.env.FIRECRAWL_BASE_URL?.trim(); + const baseUrl = envBase + ? envBase.replace(/\/+$/, "") + : typeof providerSpecificData?.baseUrl === "string" && providerSpecificData.baseUrl.trim() + ? providerSpecificData.baseUrl.trim().replace(/\/+$/, "") + : "https://api.firecrawl.dev"; + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) { + headers["Authorization"] = `Bearer ${apiKey}`; + } + return { + url: `${baseUrl}/v1/scrape`, + init: { + method: "POST", + headers, + body: JSON.stringify({ url: "https://example.com", formats: ["markdown"] }), + }, + }; + }, "jina-reader": (apiKey) => ({ url: "https://r.jina.ai/https://example.com", init: { diff --git a/src/shared/constants/providers.ts b/src/shared/constants/providers.ts index 7fdca3033d..af6535c149 100644 --- a/src/shared/constants/providers.ts +++ b/src/shared/constants/providers.ts @@ -179,6 +179,7 @@ export function isSelfHostedChatProvider(providerId: unknown): boolean { // cyclomatic complexity flat as this list grows — see g4f.space (#6650). const EXPLICIT_OPTIONAL_APIKEY_PROVIDER_IDS = new Set([ "searxng-search", + "firecrawl", "pollinations", "copilot-web", "hackclub", diff --git a/src/shared/constants/providers/search.ts b/src/shared/constants/providers/search.ts index f829964f4b..43fc61afb1 100644 --- a/src/shared/constants/providers/search.ts +++ b/src/shared/constants/providers/search.ts @@ -69,6 +69,7 @@ export const SEARCH_PROVIDERS = { textIcon: "FC", website: "https://firecrawl.dev", hasFree: true, + authHint: "API key from firecrawl.dev/app/api-keys (or set your self-hosted Firecrawl base URL)", notice: { text: "Free tier: 1,000 credits/month. Powers /v1/web/fetch and /v1/search.", apiKeyUrl: "https://firecrawl.dev/app/api-keys", diff --git a/tests/unit/firecrawl-quota-fetcher.test.ts b/tests/unit/firecrawl-quota-fetcher.test.ts index 3df6e224a2..0a98bb020c 100644 --- a/tests/unit/firecrawl-quota-fetcher.test.ts +++ b/tests/unit/firecrawl-quota-fetcher.test.ts @@ -153,3 +153,25 @@ test("registerFirecrawlQuotaFetcher registers firecrawl for preflight", async () invalidateFirecrawlQuotaCache(connectionId); }); + +test("fetchFirecrawlQuota bypasses cloud fetch when FIRECRAWL_BASE_URL is set", async () => { + const originalEnv = process.env.FIRECRAWL_BASE_URL; + try { + process.env.FIRECRAWL_BASE_URL = "http://localhost:3002/"; + const connectionId = `fc-selfhosted-${Date.now()}`; + let fetchCalled = false; + globalThis.fetch = async () => { + fetchCalled = true; + return creditUsageResponse(100, 1000); + }; + + const quota = await fetchFirecrawlQuota(connectionId, { apiKey: "local-key" }); + assert.ok(quota); + assert.equal(quota!.used, 0); + assert.equal(quota!.total, 0); + assert.equal(fetchCalled, false); + invalidateFirecrawlQuotaCache(connectionId); + } finally { + process.env.FIRECRAWL_BASE_URL = originalEnv; + } +});