feat(api): quota-aware fallback routing for web-fetch providers (#8297) (#8335)

Mirror the search/route.ts pattern for /v1/web/fetch: skip rate-limited
stubs instead of letting them short-circuit auto-select, walk the
fixed-priority pool (fill-first) with a request-time fallback on
retryable/quota upstream statuses (429 always; 402/403 for
Firecrawl/Tavily/TinyFish quota-style tiers), and return a proper 429
(with Retry-After) when the whole pool is exhausted instead of a
generic 400. Explicit-provider requests never silently fall back.

changelog.d/features/8297-web-fetch-quota-aware-fallback.md

@@ -0,0 +1 @@
+- feat(api): quota-aware fallback routing for web-fetch providers (#8297)

docs/reference/API_REFERENCE.md

@@ -344,6 +344,32 @@ Web/search provider abstraction (Tavily, Brave, Exa, Serper, etc.).
 
 ---
 
+## Web Fetch API
+
+Extract content from a URL via a configured web-fetch provider (Firecrawl, Jina
+Reader, Tavily Extract, TinyFish Fetch).
+
+| Method | Path           | Description                                                              |
+| ------ | -------------- | ------------------------------------------------------------------------- |
+| POST   | `/v1/web/fetch` | Fetch/scrape a URL — body validated by `v1WebFetchSchema`               |
+
+**Auth:** Bearer API key (`extractApiKey` + `isValidApiKey`). Policy enforced via `enforceApiKeyPolicy`.
+
+**Quota-aware fallback (#8297):** when no explicit `provider` is given, the pool
+(`firecrawl` → `jina-reader` → `tavily-search` → `tinyfish`) is walked in fixed
+priority order (fill-first) — a rate-limited-but-configured provider is skipped
+instead of short-circuiting the request, and a retryable/quota upstream failure
+(HTTP 429 always; 402/403 for Firecrawl/Tavily/TinyFish quota-style free tiers —
+not for Jina Reader, and never for a plain 400 bad request) falls through to the
+next untried credentialed provider at request time. When every provider in the
+pool is exhausted, the endpoint returns a single `429` (with a `Retry-After`
+header) instead of the previous generic `400`. When an explicit `provider` is
+requested, there is **no** silent fallback — a rate-limited or failing explicit
+provider surfaces its own error (`429` if rate-limited, otherwise the upstream
+status).
+
+---
+
 ## WebSocket Streaming
 
 ```bash

src/app/api/v1/web/fetch/route.ts

@@ -6,11 +6,21 @@
  *
  * Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? }
  * Response: { provider, url, content, links, metadata, screenshot_url }
+ *
+ * Quota-aware fallback (#8297): when no explicit provider is requested, the
+ * pool is walked in fixed priority order (fill-first) — a rate-limited or
+ * quota-exhausted provider is skipped instead of short-circuiting the whole
+ * request. When an explicit provider is requested, no silent fallback is
+ * performed — a rate-limited/failing explicit provider surfaces its own error.
  */
 
-import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
+import { errorResponse, unavailableResponse } from "@omniroute/open-sse/utils/error.ts";
 import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
-import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts";
+import {
+  handleWebFetch,
+  type WebFetchCredentials,
+  type WebFetchResult,
+} from "@omniroute/open-sse/handlers/webFetch.ts";
 import * as log from "@/sse/utils/logger";
 import {
   extractApiKey,
@@ -21,6 +31,11 @@ import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
 import { isRequireApiKeyEnabled } from "@/shared/utils/featureFlags";
 import { v1WebFetchSchema } from "@/shared/validation/schemas";
 import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
+import {
+  isAllRateLimitedCredentials,
+  rateLimitedProviderResponse,
+  type RateLimitedCredentials,
+} from "@/app/api/v1/_shared/rateLimit";
 
 const CORS_HEADERS = {
   "Access-Control-Allow-Methods": "POST, OPTIONS",
@@ -30,25 +45,192 @@ const CORS_HEADERS = {
 const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const;
 type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number];
 
+// Providers whose free/low tiers surface quota exhaustion as 402/403 instead
+// of (or in addition to) 429. jina-reader has no such quota-status signal —
+// a 402/403 there is a real auth/bad-request failure, not exhaustion.
+const QUOTA_STATUS_PROVIDERS = new Set<WebFetchProviderId>([
+  "firecrawl",
+  "tavily-search",
+  "tinyfish",
+]);
+
+type CredentialsLookup = WebFetchCredentials | RateLimitedCredentials | null;
+
 export async function OPTIONS() {
   return new Response(null, { headers: CORS_HEADERS });
 }
 
 /**
- * Resolve credentials for a web-fetch provider. Tries each known provider in
- * priority order when no explicit provider is requested.
+ * Resolve credentials for a web-fetch provider (may be a rate-limited stub,
+ * real credentials, or null when unconfigured).
  */
-async function resolveCredentials(
-  providerId: WebFetchProviderId
-): Promise<{ apiKey?: string } | null> {
+async function resolveCredentials(providerId: WebFetchProviderId): Promise<CredentialsLookup> {
   try {
     const creds = await getProviderCredentialsWithQuotaPreflight(providerId);
-    return creds ?? null;
+    return (creds as CredentialsLookup) ?? null;
   } catch {
     return null;
   }
 }
 
+/** A request-time upstream status that means "try the next provider" instead of giving up. */
+function isRetryableWebFetchStatus(providerId: WebFetchProviderId, status?: number): boolean {
+  if (status === HTTP_STATUS.RATE_LIMITED) return true;
+  if (status === HTTP_STATUS.PAYMENT_REQUIRED || status === HTTP_STATUS.FORBIDDEN) {
+    return QUOTA_STATUS_PROVIDERS.has(providerId);
+  }
+  return false;
+}
+
+/** Find the next untried, non-rate-limited, credentialed provider in pool order. */
+async function findNextFallbackProvider(
+  tried: Set<WebFetchProviderId>
+): Promise<{ providerId: WebFetchProviderId; credentials: WebFetchCredentials } | null> {
+  for (const pid of WEB_FETCH_PROVIDERS) {
+    if (tried.has(pid)) continue;
+    const creds = await resolveCredentials(pid);
+    tried.add(pid);
+    if (creds && !isAllRateLimitedCredentials(creds)) {
+      return { providerId: pid, credentials: creds };
+    }
+  }
+  return null;
+}
+
+interface WebFetchExecutionInput {
+  url: string;
+  format: "markdown" | "html" | "links" | "screenshot";
+  depth: 0 | 1 | 2;
+  wait_for_selector?: string;
+  include_metadata?: boolean;
+}
+
+interface WebFetchExecutionResult {
+  result: WebFetchResult;
+  provider: WebFetchProviderId;
+  poolExhausted: boolean;
+}
+
+/**
+ * Execute the web-fetch request. When `allowFallback` is true (auto-select),
+ * a retryable/quota upstream failure walks the remaining pool in order
+ * before giving up. Explicit-provider requests never fall back.
+ */
+async function executeWithFallback(
+  reqBody: WebFetchExecutionInput,
+  startProvider: WebFetchProviderId,
+  startCredentials: WebFetchCredentials,
+  allowFallback: boolean,
+  triedProviders: Set<WebFetchProviderId>
+): Promise<WebFetchExecutionResult> {
+  let provider = startProvider;
+  let credentials = startCredentials;
+  let result = await handleWebFetch(reqBody, credentials, provider);
+
+  if (!allowFallback) {
+    return { result, provider, poolExhausted: false };
+  }
+
+  while (!result.success && isRetryableWebFetchStatus(provider, result.status)) {
+    const next = await findNextFallbackProvider(triedProviders);
+    if (!next) {
+      return { result, provider, poolExhausted: true };
+    }
+    provider = next.providerId;
+    credentials = next.credentials;
+    result = await handleWebFetch(reqBody, credentials, provider);
+  }
+
+  return { result, provider, poolExhausted: false };
+}
+
+type ResolvedWebFetchTarget =
+  | {
+      ok: true;
+      provider: WebFetchProviderId;
+      credentials: WebFetchCredentials;
+      tried: Set<WebFetchProviderId>;
+      isExplicit: boolean;
+    }
+  | { ok: false; response: Response };
+
+/** Resolve credentials for an explicitly requested provider (no fallback allowed). */
+async function resolveExplicitTarget(
+  providerId: WebFetchProviderId
+): Promise<ResolvedWebFetchTarget> {
+  const creds = await resolveCredentials(providerId);
+  if (isAllRateLimitedCredentials(creds)) {
+    return { ok: false, response: rateLimitedProviderResponse(providerId, creds) };
+  }
+  if (!creds) {
+    return {
+      ok: false,
+      response: errorResponse(
+        HTTP_STATUS.BAD_REQUEST,
+        `No credentials configured for web-fetch provider: ${providerId}. ` +
+          `Add an API key for "${providerId}" in the dashboard.`
+      ),
+    };
+  }
+  return {
+    ok: true,
+    provider: providerId,
+    credentials: creds,
+    tried: new Set([providerId]),
+    isExplicit: true,
+  };
+}
+
+/**
+ * Auto-select: walk the pool in fixed priority order (fill-first), skipping
+ * rate-limited stubs instead of letting them short-circuit the loop (#8297).
+ */
+async function resolveAutoSelectTarget(): Promise<ResolvedWebFetchTarget> {
+  let firstRateLimited: {
+    providerId: WebFetchProviderId;
+    credentials: RateLimitedCredentials;
+  } | null = null;
+
+  for (const pid of WEB_FETCH_PROVIDERS) {
+    const creds = await resolveCredentials(pid);
+    if (isAllRateLimitedCredentials(creds)) {
+      firstRateLimited ??= { providerId: pid, credentials: creds };
+      continue;
+    }
+    if (creds) {
+      return { ok: true, provider: pid, credentials: creds, tried: new Set([pid]), isExplicit: false };
+    }
+  }
+
+  if (firstRateLimited) {
+    return {
+      ok: false,
+      response: rateLimitedProviderResponse(
+        firstRateLimited.providerId,
+        firstRateLimited.credentials
+      ),
+    };
+  }
+  return {
+    ok: false,
+    response: errorResponse(
+      HTTP_STATUS.BAD_REQUEST,
+      `No credentials configured for any web-fetch provider. ` +
+        `Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.`
+    ),
+  };
+}
+
+/** Resolve the provider + credentials to use for this request (explicit or auto-select). */
+async function resolveWebFetchTarget(
+  requestedProvider: string | undefined
+): Promise<ResolvedWebFetchTarget> {
+  if (requestedProvider) {
+    return resolveExplicitTarget(requestedProvider as WebFetchProviderId);
+  }
+  return resolveAutoSelectTarget();
+}
+
 export async function POST(request: Request) {
   let rawBody: unknown;
   try {
@@ -79,43 +261,13 @@ export async function POST(request: Request) {
   const policy = await enforceApiKeyPolicy(request, "web-fetch");
   if (policy.rejection) return policy.rejection;
 
-  // Resolve provider + credentials
-  let resolvedProvider: WebFetchProviderId | undefined;
-  let credentials: { apiKey?: string } = {};
+  // Resolve provider + credentials (explicit provider never falls back; #8297)
+  const target = await resolveWebFetchTarget(body.provider);
+  if (!target.ok) return target.response;
 
-  if (body.provider) {
-    resolvedProvider = body.provider as WebFetchProviderId;
-    const creds = await resolveCredentials(resolvedProvider);
-    if (!creds) {
-      return errorResponse(
-        HTTP_STATUS.BAD_REQUEST,
-        `No credentials configured for web-fetch provider: ${resolvedProvider}. ` +
-          `Add an API key for "${resolvedProvider}" in the dashboard.`
-      );
-    }
-    credentials = creds;
-  } else {
-    // Auto-select: try providers in priority order
-    for (const pid of WEB_FETCH_PROVIDERS) {
-      const creds = await resolveCredentials(pid);
-      if (creds) {
-        resolvedProvider = pid;
-        credentials = creds;
-        break;
-      }
-    }
-    if (!resolvedProvider) {
-      return errorResponse(
-        HTTP_STATUS.BAD_REQUEST,
-        `No credentials configured for any web-fetch provider. ` +
-          `Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.`
-      );
-    }
-  }
+  log.info("WEB_FETCH", `${target.provider} | ${body.url} | format=${body.format}`);
 
-  log.info("WEB_FETCH", `${resolvedProvider} | ${body.url} | format=${body.format}`);
-
-  const result = await handleWebFetch(
+  const { result, provider: finalProvider, poolExhausted } = await executeWithFallback(
     {
       url: body.url,
       format: body.format,
@@ -123,10 +275,19 @@ export async function POST(request: Request) {
       wait_for_selector: body.wait_for_selector,
       include_metadata: body.include_metadata,
     },
-    credentials,
-    resolvedProvider
+    target.provider,
+    target.credentials,
+    !target.isExplicit,
+    target.tried
   );
 
+  if (poolExhausted) {
+    return unavailableResponse(
+      HTTP_STATUS.RATE_LIMITED,
+      "All configured web-fetch providers are rate limited or quota-exhausted"
+    );
+  }
+
   if (!result.success) {
     return new Response(
       JSON.stringify({
@@ -139,6 +300,10 @@ export async function POST(request: Request) {
     );
   }
 
+  if (finalProvider !== target.provider) {
+    log.info("WEB_FETCH", `Fell back from ${target.provider} to ${finalProvider}`);
+  }
+
   return new Response(JSON.stringify(result.data), {
     status: 200,
     headers: { "Content-Type": "application/json", ...CORS_HEADERS },

tests/unit/web-fetch-quota-fallback.test.ts

@@ -0,0 +1,274 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-web-fetch-fallback-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const webFetchRoute = await import("../../src/app/api/v1/web/fetch/route.ts");
+
+async function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+async function seedConnection(
+  provider: string,
+  overrides: {
+    apiKey?: string | null;
+    rateLimitedUntil?: string | null;
+  } = {}
+) {
+  return providersDb.createProviderConnection({
+    provider,
+    authType: "apikey",
+    name: `${provider}-${Math.random().toString(16).slice(2, 8)}`,
+    apiKey: overrides.apiKey ?? "test-key",
+    isActive: true,
+    testStatus: "active",
+    rateLimitedUntil: overrides.rateLimitedUntil ?? null,
+    providerSpecificData: {},
+  });
+}
+
+function postWebFetch(body: Record<string, unknown>) {
+  return webFetchRoute.POST(
+    new Request("http://localhost/api/v1/web/fetch", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ url: "https://example.com", ...body }),
+    })
+  );
+}
+
+interface WebFetchTestBody {
+  provider?: string;
+  content?: string;
+  error?: { message: string };
+}
+
+async function readJson(response: Response): Promise<WebFetchTestBody> {
+  return (await response.json()) as WebFetchTestBody;
+}
+
+const FUTURE_ISO = new Date(Date.now() + 5 * 60 * 1000).toISOString();
+
+test.beforeEach(async () => {
+  await resetStorage();
+});
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+// ── (a) credential-time: rate-limited stub is skipped, not short-circuited ──
+
+test("auto-select skips a rate-limited firecrawl and falls to jina-reader", async () => {
+  await seedConnection("firecrawl", { rateLimitedUntil: FUTURE_ISO });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const u = String(url);
+    if (u.includes("api.firecrawl.dev")) {
+      throw new Error("firecrawl should never be called once rate-limited");
+    }
+    if (u.includes("r.jina.ai")) {
+      return new Response(
+        JSON.stringify({ data: { content: "jina content", links: [] } }),
+        { status: 200, headers: { "content-type": "application/json" } }
+      );
+    }
+    throw new Error(`unexpected fetch to ${u}`);
+  };
+
+  try {
+    const response = await postWebFetch({});
+    const body = await readJson(response);
+
+    assert.equal(response.status, 200);
+    assert.equal(body.provider, "jina-reader");
+    assert.equal(body.content, "jina content");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+// ── (b) request-time: credentialed provider returns 429 → falls through ────
+
+test("auto-select falls through to jina-reader when firecrawl returns 429 at request time", async () => {
+  await seedConnection("firecrawl", { apiKey: "fc-key" });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const u = String(url);
+    if (u.includes("api.firecrawl.dev")) {
+      return new Response(JSON.stringify({ error: "rate limited" }), {
+        status: 429,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    if (u.includes("r.jina.ai")) {
+      return new Response(
+        JSON.stringify({ data: { content: "jina content", links: [] } }),
+        { status: 200, headers: { "content-type": "application/json" } }
+      );
+    }
+    throw new Error(`unexpected fetch to ${u}`);
+  };
+
+  try {
+    const response = await postWebFetch({});
+    const body = await readJson(response);
+
+    assert.equal(response.status, 200);
+    assert.equal(body.provider, "jina-reader");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+// ── (c) provider-specific quota status (402/403) triggers fallback; plain 400 does NOT ──
+
+test("auto-select falls through to jina-reader when firecrawl returns 403 (quota-style)", async () => {
+  await seedConnection("firecrawl", { apiKey: "fc-key" });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const u = String(url);
+    if (u.includes("api.firecrawl.dev")) {
+      return new Response(JSON.stringify({ error: "quota exceeded" }), {
+        status: 403,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    if (u.includes("r.jina.ai")) {
+      return new Response(
+        JSON.stringify({ data: { content: "jina content", links: [] } }),
+        { status: 200, headers: { "content-type": "application/json" } }
+      );
+    }
+    throw new Error(`unexpected fetch to ${u}`);
+  };
+
+  try {
+    const response = await postWebFetch({});
+    const body = await readJson(response);
+
+    assert.equal(response.status, 200);
+    assert.equal(body.provider, "jina-reader");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test("auto-select does NOT fall through when firecrawl returns a plain 400 bad request", async () => {
+  await seedConnection("firecrawl", { apiKey: "fc-key" });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+
+  let jinaWasCalled = false;
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const u = String(url);
+    if (u.includes("api.firecrawl.dev")) {
+      return new Response(JSON.stringify({ error: "bad url" }), {
+        status: 400,
+        headers: { "content-type": "application/json" },
+      });
+    }
+    if (u.includes("r.jina.ai")) {
+      jinaWasCalled = true;
+      return new Response(
+        JSON.stringify({ data: { content: "jina content", links: [] } }),
+        { status: 200, headers: { "content-type": "application/json" } }
+      );
+    }
+    throw new Error(`unexpected fetch to ${u}`);
+  };
+
+  try {
+    const response = await postWebFetch({});
+    const body = await readJson(response);
+
+    assert.equal(response.status, 400);
+    assert.equal(jinaWasCalled, false, "jina-reader must not be tried for a non-quota 400");
+    assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+// ── (d) explicit rate-limited provider → 429, no silent fallback ───────────
+
+test("explicit rate-limited provider request returns 429 without falling back", async () => {
+  await seedConnection("firecrawl", { rateLimitedUntil: FUTURE_ISO });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+
+  let jinaWasCalled = false;
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async (url) => {
+    const u = String(url);
+    if (u.includes("r.jina.ai")) {
+      jinaWasCalled = true;
+    }
+    throw new Error(`unexpected fetch to ${u}`);
+  };
+
+  try {
+    const response = await postWebFetch({ provider: "firecrawl" });
+    const body = await readJson(response);
+
+    assert.equal(response.status, 429);
+    assert.equal(jinaWasCalled, false, "explicit provider request must never fall back");
+    assert.ok(response.headers.get("Retry-After"), "should include a Retry-After header");
+    assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+// ── (e) whole pool exhausted at request time → single 429 with retry-after ─
+
+test("auto-select returns a single 429 with retry-after when the whole pool is exhausted", async () => {
+  await seedConnection("firecrawl", { apiKey: "fc-key" });
+  await seedConnection("jina-reader", { apiKey: "jina-key" });
+  await seedConnection("tavily-search", { apiKey: "tavily-key" });
+  await seedConnection("tinyfish", { apiKey: "tf-key" });
+
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async () => {
+    return new Response(JSON.stringify({ error: "rate limited" }), {
+      status: 429,
+      headers: { "content-type": "application/json" },
+    });
+  };
+
+  try {
+    const response = await postWebFetch({});
+    const body = await readJson(response);
+
+    assert.equal(response.status, 429);
+    assert.ok(response.headers.get("Retry-After"), "should include a Retry-After header");
+    assert.ok(!(body.error?.message ?? "").includes("at /"), "error must not leak stack paths");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+// ── No credentials at all → generic 400 (unchanged behavior) ──────────────
+
+test("auto-select returns 400 when no web-fetch provider is configured", async () => {
+  const response = await postWebFetch({});
+  const body = await readJson(response);
+
+  assert.equal(response.status, 400);
+  assert.ok((body.error?.message ?? "").includes("No credentials configured"));
+});