diff --git a/changelog.d/features/8297-web-fetch-quota-aware-fallback.md b/changelog.d/features/8297-web-fetch-quota-aware-fallback.md new file mode 100644 index 0000000000..c916cf1767 --- /dev/null +++ b/changelog.d/features/8297-web-fetch-quota-aware-fallback.md @@ -0,0 +1 @@ +- feat(api): quota-aware fallback routing for web-fetch providers (#8297) diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index 32aab3fbe8..fbf41ea286 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -344,6 +344,32 @@ Web/search provider abstraction (Tavily, Brave, Exa, Serper, etc.). --- +## Web Fetch API + +Extract content from a URL via a configured web-fetch provider (Firecrawl, Jina +Reader, Tavily Extract, TinyFish Fetch). + +| Method | Path | Description | +| ------ | -------------- | ------------------------------------------------------------------------- | +| POST | `/v1/web/fetch` | Fetch/scrape a URL — body validated by `v1WebFetchSchema` | + +**Auth:** Bearer API key (`extractApiKey` + `isValidApiKey`). Policy enforced via `enforceApiKeyPolicy`. + +**Quota-aware fallback (#8297):** when no explicit `provider` is given, the pool +(`firecrawl` → `jina-reader` → `tavily-search` → `tinyfish`) is walked in fixed +priority order (fill-first) — a rate-limited-but-configured provider is skipped +instead of short-circuiting the request, and a retryable/quota upstream failure +(HTTP 429 always; 402/403 for Firecrawl/Tavily/TinyFish quota-style free tiers — +not for Jina Reader, and never for a plain 400 bad request) falls through to the +next untried credentialed provider at request time. When every provider in the +pool is exhausted, the endpoint returns a single `429` (with a `Retry-After` +header) instead of the previous generic `400`. When an explicit `provider` is +requested, there is **no** silent fallback — a rate-limited or failing explicit +provider surfaces its own error (`429` if rate-limited, otherwise the upstream +status). + +--- + ## WebSocket Streaming ```bash diff --git a/src/app/api/v1/web/fetch/route.ts b/src/app/api/v1/web/fetch/route.ts index 1fbfa0be5a..f7c49f8da1 100644 --- a/src/app/api/v1/web/fetch/route.ts +++ b/src/app/api/v1/web/fetch/route.ts @@ -6,11 +6,21 @@ * * Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? } * Response: { provider, url, content, links, metadata, screenshot_url } + * + * Quota-aware fallback (#8297): when no explicit provider is requested, the + * pool is walked in fixed priority order (fill-first) — a rate-limited or + * quota-exhausted provider is skipped instead of short-circuiting the whole + * request. When an explicit provider is requested, no silent fallback is + * performed — a rate-limited/failing explicit provider surfaces its own error. */ -import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; +import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; -import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts"; +import { + handleWebFetch, + type WebFetchCredentials, + type WebFetchResult, +} from "@omniroute/open-sse/handlers/webFetch.ts"; import * as log from "@/sse/utils/logger"; import { extractApiKey, @@ -21,6 +31,11 @@ import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; import { isRequireApiKeyEnabled } from "@/shared/utils/featureFlags"; import { v1WebFetchSchema } from "@/shared/validation/schemas"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { + isAllRateLimitedCredentials, + rateLimitedProviderResponse, + type RateLimitedCredentials, +} from "@/app/api/v1/_shared/rateLimit"; const CORS_HEADERS = { "Access-Control-Allow-Methods": "POST, OPTIONS", @@ -30,25 +45,192 @@ const CORS_HEADERS = { const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const; type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]; +// Providers whose free/low tiers surface quota exhaustion as 402/403 instead +// of (or in addition to) 429. jina-reader has no such quota-status signal — +// a 402/403 there is a real auth/bad-request failure, not exhaustion. +const QUOTA_STATUS_PROVIDERS = new Set([ + "firecrawl", + "tavily-search", + "tinyfish", +]); + +type CredentialsLookup = WebFetchCredentials | RateLimitedCredentials | null; + export async function OPTIONS() { return new Response(null, { headers: CORS_HEADERS }); } /** - * Resolve credentials for a web-fetch provider. Tries each known provider in - * priority order when no explicit provider is requested. + * Resolve credentials for a web-fetch provider (may be a rate-limited stub, + * real credentials, or null when unconfigured). */ -async function resolveCredentials( - providerId: WebFetchProviderId -): Promise<{ apiKey?: string } | null> { +async function resolveCredentials(providerId: WebFetchProviderId): Promise { try { const creds = await getProviderCredentialsWithQuotaPreflight(providerId); - return creds ?? null; + return (creds as CredentialsLookup) ?? null; } catch { return null; } } +/** A request-time upstream status that means "try the next provider" instead of giving up. */ +function isRetryableWebFetchStatus(providerId: WebFetchProviderId, status?: number): boolean { + if (status === HTTP_STATUS.RATE_LIMITED) return true; + if (status === HTTP_STATUS.PAYMENT_REQUIRED || status === HTTP_STATUS.FORBIDDEN) { + return QUOTA_STATUS_PROVIDERS.has(providerId); + } + return false; +} + +/** Find the next untried, non-rate-limited, credentialed provider in pool order. */ +async function findNextFallbackProvider( + tried: Set +): Promise<{ providerId: WebFetchProviderId; credentials: WebFetchCredentials } | null> { + for (const pid of WEB_FETCH_PROVIDERS) { + if (tried.has(pid)) continue; + const creds = await resolveCredentials(pid); + tried.add(pid); + if (creds && !isAllRateLimitedCredentials(creds)) { + return { providerId: pid, credentials: creds }; + } + } + return null; +} + +interface WebFetchExecutionInput { + url: string; + format: "markdown" | "html" | "links" | "screenshot"; + depth: 0 | 1 | 2; + wait_for_selector?: string; + include_metadata?: boolean; +} + +interface WebFetchExecutionResult { + result: WebFetchResult; + provider: WebFetchProviderId; + poolExhausted: boolean; +} + +/** + * Execute the web-fetch request. When `allowFallback` is true (auto-select), + * a retryable/quota upstream failure walks the remaining pool in order + * before giving up. Explicit-provider requests never fall back. + */ +async function executeWithFallback( + reqBody: WebFetchExecutionInput, + startProvider: WebFetchProviderId, + startCredentials: WebFetchCredentials, + allowFallback: boolean, + triedProviders: Set +): Promise { + let provider = startProvider; + let credentials = startCredentials; + let result = await handleWebFetch(reqBody, credentials, provider); + + if (!allowFallback) { + return { result, provider, poolExhausted: false }; + } + + while (!result.success && isRetryableWebFetchStatus(provider, result.status)) { + const next = await findNextFallbackProvider(triedProviders); + if (!next) { + return { result, provider, poolExhausted: true }; + } + provider = next.providerId; + credentials = next.credentials; + result = await handleWebFetch(reqBody, credentials, provider); + } + + return { result, provider, poolExhausted: false }; +} + +type ResolvedWebFetchTarget = + | { + ok: true; + provider: WebFetchProviderId; + credentials: WebFetchCredentials; + tried: Set; + isExplicit: boolean; + } + | { ok: false; response: Response }; + +/** Resolve credentials for an explicitly requested provider (no fallback allowed). */ +async function resolveExplicitTarget( + providerId: WebFetchProviderId +): Promise { + const creds = await resolveCredentials(providerId); + if (isAllRateLimitedCredentials(creds)) { + return { ok: false, response: rateLimitedProviderResponse(providerId, creds) }; + } + if (!creds) { + return { + ok: false, + response: errorResponse( + HTTP_STATUS.BAD_REQUEST, + `No credentials configured for web-fetch provider: ${providerId}. ` + + `Add an API key for "${providerId}" in the dashboard.` + ), + }; + } + return { + ok: true, + provider: providerId, + credentials: creds, + tried: new Set([providerId]), + isExplicit: true, + }; +} + +/** + * Auto-select: walk the pool in fixed priority order (fill-first), skipping + * rate-limited stubs instead of letting them short-circuit the loop (#8297). + */ +async function resolveAutoSelectTarget(): Promise { + let firstRateLimited: { + providerId: WebFetchProviderId; + credentials: RateLimitedCredentials; + } | null = null; + + for (const pid of WEB_FETCH_PROVIDERS) { + const creds = await resolveCredentials(pid); + if (isAllRateLimitedCredentials(creds)) { + firstRateLimited ??= { providerId: pid, credentials: creds }; + continue; + } + if (creds) { + return { ok: true, provider: pid, credentials: creds, tried: new Set([pid]), isExplicit: false }; + } + } + + if (firstRateLimited) { + return { + ok: false, + response: rateLimitedProviderResponse( + firstRateLimited.providerId, + firstRateLimited.credentials + ), + }; + } + return { + ok: false, + response: errorResponse( + HTTP_STATUS.BAD_REQUEST, + `No credentials configured for any web-fetch provider. ` + + `Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.` + ), + }; +} + +/** Resolve the provider + credentials to use for this request (explicit or auto-select). */ +async function resolveWebFetchTarget( + requestedProvider: string | undefined +): Promise { + if (requestedProvider) { + return resolveExplicitTarget(requestedProvider as WebFetchProviderId); + } + return resolveAutoSelectTarget(); +} + export async function POST(request: Request) { let rawBody: unknown; try { @@ -79,43 +261,13 @@ export async function POST(request: Request) { const policy = await enforceApiKeyPolicy(request, "web-fetch"); if (policy.rejection) return policy.rejection; - // Resolve provider + credentials - let resolvedProvider: WebFetchProviderId | undefined; - let credentials: { apiKey?: string } = {}; + // Resolve provider + credentials (explicit provider never falls back; #8297) + const target = await resolveWebFetchTarget(body.provider); + if (!target.ok) return target.response; - if (body.provider) { - resolvedProvider = body.provider as WebFetchProviderId; - const creds = await resolveCredentials(resolvedProvider); - if (!creds) { - return errorResponse( - HTTP_STATUS.BAD_REQUEST, - `No credentials configured for web-fetch provider: ${resolvedProvider}. ` + - `Add an API key for "${resolvedProvider}" in the dashboard.` - ); - } - credentials = creds; - } else { - // Auto-select: try providers in priority order - for (const pid of WEB_FETCH_PROVIDERS) { - const creds = await resolveCredentials(pid); - if (creds) { - resolvedProvider = pid; - credentials = creds; - break; - } - } - if (!resolvedProvider) { - return errorResponse( - HTTP_STATUS.BAD_REQUEST, - `No credentials configured for any web-fetch provider. ` + - `Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.` - ); - } - } + log.info("WEB_FETCH", `${target.provider} | ${body.url} | format=${body.format}`); - log.info("WEB_FETCH", `${resolvedProvider} | ${body.url} | format=${body.format}`); - - const result = await handleWebFetch( + const { result, provider: finalProvider, poolExhausted } = await executeWithFallback( { url: body.url, format: body.format, @@ -123,10 +275,19 @@ export async function POST(request: Request) { wait_for_selector: body.wait_for_selector, include_metadata: body.include_metadata, }, - credentials, - resolvedProvider + target.provider, + target.credentials, + !target.isExplicit, + target.tried ); + if (poolExhausted) { + return unavailableResponse( + HTTP_STATUS.RATE_LIMITED, + "All configured web-fetch providers are rate limited or quota-exhausted" + ); + } + if (!result.success) { return new Response( JSON.stringify({ @@ -139,6 +300,10 @@ export async function POST(request: Request) { ); } + if (finalProvider !== target.provider) { + log.info("WEB_FETCH", `Fell back from ${target.provider} to ${finalProvider}`); + } + return new Response(JSON.stringify(result.data), { status: 200, headers: { "Content-Type": "application/json", ...CORS_HEADERS }, diff --git a/tests/unit/web-fetch-quota-fallback.test.ts b/tests/unit/web-fetch-quota-fallback.test.ts new file mode 100644 index 0000000000..4c7a77f595 --- /dev/null +++ b/tests/unit/web-fetch-quota-fallback.test.ts @@ -0,0 +1,274 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-web-fetch-fallback-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const webFetchRoute = await import("../../src/app/api/v1/web/fetch/route.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +async function seedConnection( + provider: string, + overrides: { + apiKey?: string | null; + rateLimitedUntil?: string | null; + } = {} +) { + return providersDb.createProviderConnection({ + provider, + authType: "apikey", + name: `${provider}-${Math.random().toString(16).slice(2, 8)}`, + apiKey: overrides.apiKey ?? "test-key", + isActive: true, + testStatus: "active", + rateLimitedUntil: overrides.rateLimitedUntil ?? null, + providerSpecificData: {}, + }); +} + +function postWebFetch(body: Record) { + return webFetchRoute.POST( + new Request("http://localhost/api/v1/web/fetch", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url: "https://example.com", ...body }), + }) + ); +} + +interface WebFetchTestBody { + provider?: string; + content?: string; + error?: { message: string }; +} + +async function readJson(response: Response): Promise { + return (await response.json()) as WebFetchTestBody; +} + +const FUTURE_ISO = new Date(Date.now() + 5 * 60 * 1000).toISOString(); + +test.beforeEach(async () => { + await resetStorage(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +// ── (a) credential-time: rate-limited stub is skipped, not short-circuited ── + +test("auto-select skips a rate-limited firecrawl and falls to jina-reader", async () => { + await seedConnection("firecrawl", { rateLimitedUntil: FUTURE_ISO }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = String(url); + if (u.includes("api.firecrawl.dev")) { + throw new Error("firecrawl should never be called once rate-limited"); + } + if (u.includes("r.jina.ai")) { + return new Response( + JSON.stringify({ data: { content: "jina content", links: [] } }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + throw new Error(`unexpected fetch to ${u}`); + }; + + try { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 200); + assert.equal(body.provider, "jina-reader"); + assert.equal(body.content, "jina content"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ── (b) request-time: credentialed provider returns 429 → falls through ──── + +test("auto-select falls through to jina-reader when firecrawl returns 429 at request time", async () => { + await seedConnection("firecrawl", { apiKey: "fc-key" }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = String(url); + if (u.includes("api.firecrawl.dev")) { + return new Response(JSON.stringify({ error: "rate limited" }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + } + if (u.includes("r.jina.ai")) { + return new Response( + JSON.stringify({ data: { content: "jina content", links: [] } }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + throw new Error(`unexpected fetch to ${u}`); + }; + + try { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 200); + assert.equal(body.provider, "jina-reader"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ── (c) provider-specific quota status (402/403) triggers fallback; plain 400 does NOT ── + +test("auto-select falls through to jina-reader when firecrawl returns 403 (quota-style)", async () => { + await seedConnection("firecrawl", { apiKey: "fc-key" }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = String(url); + if (u.includes("api.firecrawl.dev")) { + return new Response(JSON.stringify({ error: "quota exceeded" }), { + status: 403, + headers: { "content-type": "application/json" }, + }); + } + if (u.includes("r.jina.ai")) { + return new Response( + JSON.stringify({ data: { content: "jina content", links: [] } }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + throw new Error(`unexpected fetch to ${u}`); + }; + + try { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 200); + assert.equal(body.provider, "jina-reader"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("auto-select does NOT fall through when firecrawl returns a plain 400 bad request", async () => { + await seedConnection("firecrawl", { apiKey: "fc-key" }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + + let jinaWasCalled = false; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = String(url); + if (u.includes("api.firecrawl.dev")) { + return new Response(JSON.stringify({ error: "bad url" }), { + status: 400, + headers: { "content-type": "application/json" }, + }); + } + if (u.includes("r.jina.ai")) { + jinaWasCalled = true; + return new Response( + JSON.stringify({ data: { content: "jina content", links: [] } }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + throw new Error(`unexpected fetch to ${u}`); + }; + + try { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 400); + assert.equal(jinaWasCalled, false, "jina-reader must not be tried for a non-quota 400"); + assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ── (d) explicit rate-limited provider → 429, no silent fallback ─────────── + +test("explicit rate-limited provider request returns 429 without falling back", async () => { + await seedConnection("firecrawl", { rateLimitedUntil: FUTURE_ISO }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + + let jinaWasCalled = false; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const u = String(url); + if (u.includes("r.jina.ai")) { + jinaWasCalled = true; + } + throw new Error(`unexpected fetch to ${u}`); + }; + + try { + const response = await postWebFetch({ provider: "firecrawl" }); + const body = await readJson(response); + + assert.equal(response.status, 429); + assert.equal(jinaWasCalled, false, "explicit provider request must never fall back"); + assert.ok(response.headers.get("Retry-After"), "should include a Retry-After header"); + assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ── (e) whole pool exhausted at request time → single 429 with retry-after ─ + +test("auto-select returns a single 429 with retry-after when the whole pool is exhausted", async () => { + await seedConnection("firecrawl", { apiKey: "fc-key" }); + await seedConnection("jina-reader", { apiKey: "jina-key" }); + await seedConnection("tavily-search", { apiKey: "tavily-key" }); + await seedConnection("tinyfish", { apiKey: "tf-key" }); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => { + return new Response(JSON.stringify({ error: "rate limited" }), { + status: 429, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 429); + assert.ok(response.headers.get("Retry-After"), "should include a Retry-After header"); + assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +// ── No credentials at all → generic 400 (unchanged behavior) ────────────── + +test("auto-select returns 400 when no web-fetch provider is configured", async () => { + const response = await postWebFetch({}); + const body = await readJson(response); + + assert.equal(response.status, 400); + assert.ok((body.error?.message ?? "").includes("No credentials configured")); +});