diff --git a/CHANGELOG.md b/CHANGELOG.md index 081d18a96c..b84a32d6ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ ### πŸ› Bug Fixes +- **feat(providers):** add **TinyFish** web-fetch/search support β€” a `tinyfish-fetch` executor + `/v1/web/fetch` route + MCP web-fetch tool, registered as a specialty-media provider with request-validation and a search-provider catalog entry. Regression guards: `tests/unit/executor-tinyfish-fetch.test.ts`, `web-fetch-handler.test.ts`, `mcp-web-fetch-tool.test.ts`, `provider-validation-tinyfish.test.ts`. (thanks @dtybnrj) - **fix(cli):** `omniroute launch-codex` now spawns `codex.cmd` through a shell on Windows (the npm `.cmd` shim is unresolvable by bare `spawn` β†’ ENOENT), mirroring the qodercli Windows fix (#6263). Regression guard: `tests/unit/launch-codex-windows-spawn-6312.test.ts`. (thanks @swingtempo) - **fix(codex):** isolate the **Spark** quota from the shared Codex quota and stabilize the quota UI ordering / hydration so per-scope limits render consistently. Regression guards: `tests/unit/codex-quota-selection-hydration.test.ts`, `provider-limits-ui.test.ts` + 3 more. (thanks @xz-dev) - **feat(api):** add a `hidePaidModels` setting that filters paid-only models out of the `/v1/models` catalog. Regression guard: `tests/unit/models-catalog-hide-paid.test.ts`. (thanks @chirag127) diff --git a/docs/frameworks/SEARCH_TOOLS_STUDIO.md b/docs/frameworks/SEARCH_TOOLS_STUDIO.md index dc51929e63..3b4148894a 100644 --- a/docs/frameworks/SEARCH_TOOLS_STUDIO.md +++ b/docs/frameworks/SEARCH_TOOLS_STUDIO.md @@ -57,7 +57,7 @@ New tab for extracting content from a URL via `POST /v1/web/fetch` (created in p - Submit β†’ fetch β†’ render `ScrapeResult.tsx`. - `ScrapeResult` renders markdown preview + raw toggle. - Cap: if response body > **256 KB**, UI shows `(truncated, view raw)` and opens raw in a Monaco modal (D21). -- Metadata panel: provider (firecrawl/jina-reader/tavily-search), latency, cost, response size, links count. +- Metadata panel: provider (firecrawl/jina-reader/tavily-search/tinyfish), latency, cost, response size, links count. - Uses `useScrapeFetch.ts` hook. ### Compare Tab @@ -108,7 +108,7 @@ Runs the same query/URL across up to **4 providers in parallel** (D22): | Field | Source | | ------------------------------ | ------------------------------------------------------------------------------------------ | | `id`, `name` | `searchRegistry.ts` | -| `kind` | `"search"` (12 providers) or `"fetch"` (firecrawl, jina-reader, tavily-search) | +| `kind` | `"search"` (12 providers) or `"fetch"` (firecrawl, jina-reader, tavily-search, tinyfish) | | `costPerQuery` | Registry data | | `freeMonthlyQuota` | Registry data | | `searchTypes` / `fetchFormats` | Registry data | @@ -136,7 +136,7 @@ Only one backend change was needed for this feature: `src/app/api/search/providers/route.ts` was extended to: -- Include all 3 fetch providers (`firecrawl`, `jina-reader`, `tavily-search`) in the array. +- Include all 4 fetch providers (`firecrawl`, `jina-reader`, `tavily-search`, `tinyfish`) in the array. - Add `kind: "search" | "fetch"` to every item. - Add `status: "configured" | "missing" | "rate_limited"` derived from live credential state. - Maintain backward compatibility β€” existing fields (`id`, `name`, etc.) unchanged. diff --git a/open-sse/executors/tinyfish-fetch.ts b/open-sse/executors/tinyfish-fetch.ts new file mode 100644 index 0000000000..80952461a6 --- /dev/null +++ b/open-sse/executors/tinyfish-fetch.ts @@ -0,0 +1,131 @@ +/** + * TinyFish Fetch Executor + * + * Fetches content from a URL using the TinyFish Fetch API. + * POST https://api.fetch.tinyfish.ai + * + * "Fetch does not use credits" per TinyFish docs β€” no explicit pricing tier. + * Docs: https://docs.tinyfish.ai/fetch-api + * + * Unlike Firecrawl, TinyFish has no "links" or "screenshot" output modes β€” + * only markdown, html, and json. Requests for those formats fall back to + * markdown and return an empty links array / null screenshot, mirroring how + * jina-reader-fetch.ts and tavily-fetch.ts handle formats they don't support. + */ + +import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts"; +import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts"; + +const TINYFISH_FETCH_URL = "https://api.fetch.tinyfish.ai"; +const TINYFISH_TIMEOUT_MS = 30_000; + +function mapFormat(format: WebFetchFormat): "markdown" | "html" { + return format === "html" ? "html" : "markdown"; +} + +interface TinyFishFetchOptions { + url: string; + format: WebFetchFormat; + includeMetadata: boolean; + credentials: WebFetchCredentials; +} + +interface TinyFishResultEntry { + url?: string; + final_url?: string; + title?: string; + description?: string; + text?: string; +} + +interface TinyFishErrorEntry { + url?: string; + message?: string; + error?: string; +} + +/** + * Execute a TinyFish Fetch API request. + */ +export async function tinyfishFetch(opts: TinyFishFetchOptions): Promise { + const { url, format, includeMetadata, credentials } = opts; + + if (!credentials.apiKey) { + const body = buildErrorBody(401, "TinyFish API key required"); + return { success: false, status: 401, error: body.error.message }; + } + + const requestBody = { + urls: [url], + format: mapFormat(format), + ttl: 0, + }; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), TINYFISH_TIMEOUT_MS); + + try { + const response = await fetch(TINYFISH_FETCH_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-API-Key": credentials.apiKey, + }, + body: JSON.stringify(requestBody), + signal: controller.signal, + }); + + if (!response.ok) { + const rawError = await response.text().catch(() => `HTTP ${response.status}`); + const msg = sanitizeErrorMessage(`TinyFish error ${response.status}: ${rawError}`); + const body = buildErrorBody(response.status, msg); + return { success: false, status: response.status, error: body.error.message }; + } + + const data = (await response.json()) as { + results?: TinyFishResultEntry[]; + errors?: TinyFishErrorEntry[]; + }; + + const result = data.results?.[0]; + + if (!result) { + const errorEntry = data.errors?.[0]; + const msg = sanitizeErrorMessage( + errorEntry?.message ?? errorEntry?.error ?? "TinyFish could not fetch the requested URL" + ); + const body = buildErrorBody(502, msg); + return { success: false, status: 502, error: body.error.message }; + } + + const metadata = includeMetadata + ? { + title: result.title != null ? String(result.title) : null, + description: result.description != null ? String(result.description) : null, + } + : null; + + return { + success: true, + data: { + provider: "tinyfish", + url, + content: String(result.text ?? ""), + links: [], + metadata, + screenshot_url: null, + }, + }; + } catch (err: unknown) { + if (err instanceof Error && err.name === "AbortError") { + const body = buildErrorBody(504, "TinyFish request timed out"); + return { success: false, status: 504, error: body.error.message }; + } + const msg = + err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err)); + const body = buildErrorBody(502, msg); + return { success: false, status: 502, error: body.error.message }; + } finally { + clearTimeout(timeoutId); + } +} diff --git a/open-sse/handlers/webFetch.ts b/open-sse/handlers/webFetch.ts index 6e03f11b26..27179ab289 100644 --- a/open-sse/handlers/webFetch.ts +++ b/open-sse/handlers/webFetch.ts @@ -2,12 +2,12 @@ * Web Fetch Handler * * Handles POST /v1/web/fetch requests. - * Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, or Tavily). + * Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, Tavily, or TinyFish). * * Request format: * { * "url": "https://example.com", - * "provider": "firecrawl" | "jina-reader" | "tavily-search", // optional + * "provider": "firecrawl" | "jina-reader" | "tavily-search" | "tinyfish", // optional * "format": "markdown" | "html" | "links" | "screenshot", * "depth": 0 | 1 | 2, * "wait_for_selector": "main", @@ -19,12 +19,13 @@ import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; import { firecrawlFetch } from "../executors/firecrawl-fetch.ts"; import { jinaReaderFetch } from "../executors/jina-reader-fetch.ts"; import { tavilyFetch } from "../executors/tavily-fetch.ts"; +import { tinyfishFetch } from "../executors/tinyfish-fetch.ts"; export type WebFetchFormat = "markdown" | "html" | "links" | "screenshot"; export interface WebFetchRequest { url: string; - provider?: "firecrawl" | "jina-reader" | "tavily-search"; + provider?: "firecrawl" | "jina-reader" | "tavily-search" | "tinyfish"; format?: WebFetchFormat; depth?: 0 | 1 | 2; wait_for_selector?: string; @@ -51,7 +52,7 @@ export interface WebFetchCredentials { apiKey?: string; } -const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const; +const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const; type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]; /** @@ -99,6 +100,14 @@ export async function handleWebFetch( credentials, }); + case "tinyfish": + return await tinyfishFetch({ + url: req.url, + format, + includeMetadata, + credentials, + }); + default: { const _exhaustive: never = provider; return { diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index 85b5c3edd7..a2a3e76964 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -453,7 +453,7 @@ export const webFetchInput = z.object({ .min(1, "URL is required") .describe("The URL to fetch content from"), provider: z - .enum(["firecrawl", "jina-reader", "tavily-search"]) + .enum(["firecrawl", "jina-reader", "tavily-search", "tinyfish"]) .optional() .describe("Specific fetch provider to use (default: first available)"), format: z @@ -496,7 +496,7 @@ export const webFetchOutput = z.object({ export const webFetchTool: McpToolDefinition = { name: "omniroute_web_fetch", description: - "Fetches and extracts content from a URL using OmniRoute's web fetch gateway. Supports multiple providers (Firecrawl, Jina Reader, Tavily) with automatic failover. Returns the page content as markdown, HTML, links, or screenshot, along with metadata.", + "Fetches and extracts content from a URL using OmniRoute's web fetch gateway. Supports multiple providers (Firecrawl, Jina Reader, Tavily, TinyFish) with automatic failover. Returns the page content as markdown, HTML, links, or screenshot, along with metadata.", inputSchema: webFetchInput, outputSchema: webFetchOutput, scopes: ["execute:search"], diff --git a/open-sse/mcp-server/server.ts b/open-sse/mcp-server/server.ts index 77ee0706b8..a9428ba310 100644 --- a/open-sse/mcp-server/server.ts +++ b/open-sse/mcp-server/server.ts @@ -578,7 +578,7 @@ async function handleWebSearch(args: { async function handleWebFetch(args: { url: string; - provider?: "firecrawl" | "jina-reader" | "tavily-search"; + provider?: "firecrawl" | "jina-reader" | "tavily-search" | "tinyfish"; format?: "markdown" | "html" | "links" | "screenshot"; include_metadata?: boolean; depth?: number; @@ -866,7 +866,16 @@ export function createMcpServer(): McpServer { ) ); - server.registerTool("omniroute_pick_fastest_model", { description: "Picks the fastest reliable provider-model pair from live telemetry.", inputSchema: pickFastestModelInput }, withScopeEnforcement("omniroute_pick_fastest_model", (args) => handlePickFastestModel(pickFastestModelInput.parse(args)))); + server.registerTool( + "omniroute_pick_fastest_model", + { + description: "Picks the fastest reliable provider-model pair from live telemetry.", + inputSchema: pickFastestModelInput, + }, + withScopeEnforcement("omniroute_pick_fastest_model", (args) => + handlePickFastestModel(pickFastestModelInput.parse(args)) + ) + ); server.registerTool( "omniroute_get_session_snapshot", @@ -1073,17 +1082,21 @@ export function createMcpServer(): McpServer { // @ts-ignore: dynamic zod access inputSchema: toolDef.inputSchema, }, - withScopeEnforcement(toolDef.name, async (args) => { - try { - const parsedArgs = toolDef.inputSchema.parse(args ?? {}); - // @ts-expect-error - handler type lost through dynamic Object.values() access - const result = await toolDef.handler(parsedArgs); - return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; - } - }, toolDef.scopes) + withScopeEnforcement( + toolDef.name, + async (args) => { + try { + const parsedArgs = toolDef.inputSchema.parse(args ?? {}); + // @ts-expect-error - handler type lost through dynamic Object.values() access + const result = await toolDef.handler(parsedArgs); + return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true }; + } + }, + toolDef.scopes + ) ); }); diff --git a/src/app/api/search/providers/route.ts b/src/app/api/search/providers/route.ts index 0d85618dae..cc87b54066 100644 --- a/src/app/api/search/providers/route.ts +++ b/src/app/api/search/providers/route.ts @@ -14,7 +14,7 @@ import { import * as log from "@/sse/utils/logger"; // --------------------------------------------------------------------------- -// Fetch provider metadata (hardcoded β€” no registry for these 3) +// Fetch provider metadata (hardcoded β€” no registry for these 4) // --------------------------------------------------------------------------- interface FetchProviderDef { @@ -47,6 +47,13 @@ const FETCH_PROVIDERS: FetchProviderDef[] = [ freeMonthlyQuota: 1000, fetchFormats: ["markdown", "text"], }, + { + id: "tinyfish", + name: "TinyFish Fetch", + costPerQuery: 0, + freeMonthlyQuota: 0, + fetchFormats: ["markdown", "html"], + }, ]; // --------------------------------------------------------------------------- @@ -115,10 +122,7 @@ async function resolveProviderStatus( export async function GET(request: Request) { if (!(await isAuthenticated(request))) { - return NextResponse.json( - buildErrorBody(401, "Unauthorized"), - { status: 401 } - ); + return NextResponse.json(buildErrorBody(401, "Unauthorized"), { status: 401 }); } try { @@ -133,19 +137,21 @@ export async function GET(request: Request) { ) ); - const searchItems: SearchProviderCatalogItem[] = searchProviderStatuses.map(({ p, status }) => ({ - id: p.id, - name: p.name, - kind: "search" as const, - costPerQuery: p.costPerQuery, - freeMonthlyQuota: p.freeMonthlyQuota, - searchTypes: p.searchTypes, - status, - configureHref: "/dashboard/providers", - })); + const searchItems: SearchProviderCatalogItem[] = searchProviderStatuses.map( + ({ p, status }) => ({ + id: p.id, + name: p.name, + kind: "search" as const, + costPerQuery: p.costPerQuery, + freeMonthlyQuota: p.freeMonthlyQuota, + searchTypes: p.searchTypes, + status, + configureHref: "/dashboard/providers", + }) + ); // ----------------------------------------------------------------------- - // 2. Build fetch providers (3 hardcoded) + // 2. Build fetch providers (4 hardcoded) // ----------------------------------------------------------------------- const fetchProviderStatuses = await Promise.all( FETCH_PROVIDERS.map((fp) => diff --git a/src/app/api/v1/web/fetch/route.ts b/src/app/api/v1/web/fetch/route.ts index 74111e22e5..77ffabe2a9 100644 --- a/src/app/api/v1/web/fetch/route.ts +++ b/src/app/api/v1/web/fetch/route.ts @@ -2,7 +2,7 @@ * POST /v1/web/fetch * * Extract content from a URL using a configured web-fetch provider. - * Supports Firecrawl, Jina Reader, and Tavily Extract. + * Supports Firecrawl, Jina Reader, Tavily Extract, and TinyFish Fetch. * * Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? } * Response: { provider, url, content, links, metadata, screenshot_url } @@ -23,7 +23,7 @@ const CORS_HEADERS = { "Access-Control-Allow-Headers": "*", }; -const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const; +const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search", "tinyfish"] as const; type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]; export async function OPTIONS() { diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index da0948cd4d..16c47096c2 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1968,7 +1968,7 @@ "scrapeProvider": "Provider", "scrapeSize": "Size", "scrapeEmptyState": "Enter a URL to extract its content", - "scrapeProvidersAvailable": "Available providers: Firecrawl, Jina Reader, Tavily.", + "scrapeProvidersAvailable": "Available providers: Firecrawl, Jina Reader, Tavily, TinyFish.", "compareRun": "Compare", "compareRunning": "Comparing…", "autoProvider": "Auto (cheapest)", diff --git a/src/lib/providers/validation/searchProviders.ts b/src/lib/providers/validation/searchProviders.ts index cc27aafd96..b49f77ed63 100644 --- a/src/lib/providers/validation/searchProviders.ts +++ b/src/lib/providers/validation/searchProviders.ts @@ -188,4 +188,12 @@ export const SEARCH_VALIDATOR_CONFIGS: Record< headers: { Authorization: `Bearer ${apiKey}` }, }, }), + tinyfish: (apiKey) => ({ + url: "https://api.fetch.tinyfish.ai", + init: { + method: "POST", + headers: { "Content-Type": "application/json", "X-API-Key": apiKey }, + body: JSON.stringify({ urls: ["https://example.com"], format: "markdown" }), + }, + }), }; diff --git a/src/shared/constants/providers/apikey/specialty-media.ts b/src/shared/constants/providers/apikey/specialty-media.ts index c72bf69454..0043800cca 100644 --- a/src/shared/constants/providers/apikey/specialty-media.ts +++ b/src/shared/constants/providers/apikey/specialty-media.ts @@ -248,4 +248,19 @@ export const APIKEY_PROVIDERS_SPECIALTY = { }, serviceKinds: ["webFetch"], }, + tinyfish: { + id: "tinyfish", + alias: "tf", + name: "TinyFish Fetch", + icon: "language", + color: "#0891B2", + textIcon: "TF", + website: "https://docs.tinyfish.ai/fetch-api", + notice: { + text: "Fetch does not use TinyFish credits. Submit up to 10 URLs per request (OmniRoute fetches one URL per call).", + apiKeyUrl: "https://agent.tinyfish.ai/api-keys", + }, + authHint: "X-API-Key from agent.tinyfish.ai/api-keys", + serviceKinds: ["webFetch"], + }, }; diff --git a/src/shared/validation/schemas/apiV1.ts b/src/shared/validation/schemas/apiV1.ts index d3d37009e5..c23d227f2f 100644 --- a/src/shared/validation/schemas/apiV1.ts +++ b/src/shared/validation/schemas/apiV1.ts @@ -289,7 +289,7 @@ export const v1BatchCreateSchema = z.object({ export const v1WebFetchSchema = z.object({ url: z.string().url("url must be a valid URL (http/https)"), - provider: z.enum(["firecrawl", "jina-reader", "tavily-search"]).optional(), + provider: z.enum(["firecrawl", "jina-reader", "tavily-search", "tinyfish"]).optional(), format: z.enum(["markdown", "html", "links", "screenshot"]).default("markdown"), depth: z.union([z.literal(0), z.literal(1), z.literal(2)]).default(0), wait_for_selector: z.string().max(256).optional(), diff --git a/tests/integration/search-providers-catalog.test.ts b/tests/integration/search-providers-catalog.test.ts index c9b41d3da2..79ce7b7d5b 100644 --- a/tests/integration/search-providers-catalog.test.ts +++ b/tests/integration/search-providers-catalog.test.ts @@ -27,9 +27,7 @@ import { // --------------------------------------------------------------------------- // Isolated temp DB for this test suite // --------------------------------------------------------------------------- -const TEST_DATA_DIR = fs.mkdtempSync( - path.join(os.tmpdir(), "omniroute-search-providers-catalog-") -); +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-search-providers-catalog-")); process.env.DATA_DIR = TEST_DATA_DIR; process.env.API_KEY_SECRET = "test-api-key-secret-search-catalog"; // Disable dashboard password requirement by default @@ -54,7 +52,7 @@ const route = await import("../../src/app/api/search/providers/route.ts"); // linkup, searchapi, youcom, searxng, ollama, zai + duckduckgo-free (added in the // v3.8.27 cycle, registry open-sse/config/searchRegistry.ts). const EXPECTED_SEARCH_COUNT = 13; -const EXPECTED_FETCH_COUNT = 3; +const EXPECTED_FETCH_COUNT = 4; const EXPECTED_TOTAL = EXPECTED_SEARCH_COUNT + EXPECTED_FETCH_COUNT; // --------------------------------------------------------------------------- @@ -271,11 +269,7 @@ test("search-providers-catalog: back-compat data field has legacy shape", async // Legacy shape: { id, object, created, name, search_types } assert.ok(Array.isArray(body.data), "`data` array must be present for back-compat"); - assert.equal( - body.data.length, - EXPECTED_TOTAL, - "data array should have same length as providers" - ); + assert.equal(body.data.length, EXPECTED_TOTAL, "data array should have same length as providers"); for (const item of body.data) { assert.ok(typeof item.id === "string", "data item must have id"); @@ -296,6 +290,7 @@ test("search-providers-catalog: fetch providers have correct metadata", async () assert.ok(ids.includes("firecrawl"), "firecrawl must be present"); assert.ok(ids.includes("jina-reader"), "jina-reader must be present"); assert.ok(ids.includes("tavily-search"), "tavily-search must be present"); + assert.ok(ids.includes("tinyfish"), "tinyfish must be present"); const firecrawl = fetchProviders.find((p: { id: string }) => p.id === "firecrawl"); assert.equal(firecrawl.name, "Firecrawl"); @@ -319,6 +314,14 @@ test("search-providers-catalog: fetch providers have correct metadata", async () const tavily = fetchProviders.find((p: { id: string }) => p.id === "tavily-search"); assert.equal(tavily.name, "Tavily Extract"); assert.equal(tavily.costPerQuery, 0.001); + + const tinyfish = fetchProviders.find((p: { id: string }) => p.id === "tinyfish"); + assert.equal(tinyfish.name, "TinyFish Fetch"); + assert.equal(tinyfish.costPerQuery, 0); + assert.ok( + tinyfish.fetchFormats.includes("markdown"), + "tinyfish fetchFormats must include markdown" + ); }); test("search-providers-catalog: search providers have correct fields", async () => { @@ -331,10 +334,7 @@ test("search-providers-catalog: search providers have correct fields", async () assert.ok(typeof item.id === "string", "search item must have id"); assert.ok(typeof item.name === "string", "search item must have name"); assert.ok(typeof item.costPerQuery === "number", "search item must have costPerQuery"); - assert.ok( - typeof item.freeMonthlyQuota === "number", - "search item must have freeMonthlyQuota" - ); + assert.ok(typeof item.freeMonthlyQuota === "number", "search item must have freeMonthlyQuota"); assert.ok(Array.isArray(item.searchTypes), "search item must have searchTypes array"); assert.equal( item.configureHref, @@ -355,9 +355,8 @@ test("search-providers-catalog: response validates against SearchProviderCatalog const res = await route.GET(req); const body = await res.json(); - const { SearchProviderCatalogResponseSchema } = await import( - "../../src/shared/schemas/searchTools.ts" - ); + const { SearchProviderCatalogResponseSchema } = + await import("../../src/shared/schemas/searchTools.ts"); const result = SearchProviderCatalogResponseSchema.safeParse({ providers: body.providers }); assert.ok( diff --git a/tests/unit/executor-tinyfish-fetch.test.ts b/tests/unit/executor-tinyfish-fetch.test.ts new file mode 100644 index 0000000000..9c614421a4 --- /dev/null +++ b/tests/unit/executor-tinyfish-fetch.test.ts @@ -0,0 +1,254 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { tinyfishFetch } = await import("../../open-sse/executors/tinyfish-fetch.ts"); + +// ── tinyfishFetch tests ───────────────────────────────────────────────────── + +test("tinyfishFetch posts to api.fetch.tinyfish.ai with X-API-Key auth and a urls array", async () => { + const originalFetch = globalThis.fetch; + let captured: { url: string; init: RequestInit } = { url: "", init: {} }; + + globalThis.fetch = async (url, init = {}) => { + captured = { url: String(url), init: init as RequestInit }; + return new Response( + JSON.stringify({ + results: [{ url: "https://example.com", text: "# Hello from TinyFish" }], + errors: [], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "tf-test-key" }, + }); + + assert.equal(result.success, true); + assert.equal(captured.url, "https://api.fetch.tinyfish.ai"); + assert.equal(captured.init.method, "POST"); + const headers = captured.init.headers as Record; + assert.equal(headers["X-API-Key"], "tf-test-key"); + const body = JSON.parse(String(captured.init.body)); + assert.deepEqual(body.urls, ["https://example.com"]); + assert.equal(body.format, "markdown"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch returns 401 error when no API key", async () => { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace"); +}); + +test("tinyfishFetch propagates non-200 status without stack trace", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => + new Response("Forbidden", { status: 403, headers: { "content-type": "text/plain" } }); + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "bad-key" }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 403); + assert.ok(result.error, "should have error message"); + assert.ok(!result.error.includes("at /"), "error must not contain stack trace"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch parses results[0].text as content and includes metadata when requested", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + results: [ + { + url: "https://example.com", + final_url: "https://example.com/", + title: "Example Domain", + description: "An example page", + text: "# Example content", + }, + ], + errors: [], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: true, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(result.success, true); + assert.ok(result.data, "should have data"); + assert.equal(result.data.provider, "tinyfish"); + assert.ok(result.data.content.includes("Example content")); + assert.equal(result.data.metadata?.title, "Example Domain"); + assert.equal(result.data.metadata?.description, "An example page"); + assert.equal(result.data.screenshot_url, null); + assert.deepEqual(result.data.links, []); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch omits metadata when includeMetadata is false", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + results: [{ url: "https://example.com", title: "Example", text: "content" }], + errors: [], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(result.success, true); + assert.equal(result.data?.metadata, null); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch maps 'html' format to the html request format", async () => { + const originalFetch = globalThis.fetch; + let capturedBody: Record = {}; + + globalThis.fetch = async (_url, init = {}) => { + capturedBody = JSON.parse(String((init as RequestInit).body)); + return new Response( + JSON.stringify({ results: [{ url: "https://example.com", text: "" }] }), + { status: 200 } + ); + }; + + try { + await tinyfishFetch({ + url: "https://example.com", + format: "html", + includeMetadata: false, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(capturedBody.format, "html"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch falls back to markdown for unsupported 'links'/'screenshot' formats", async () => { + const originalFetch = globalThis.fetch; + let capturedBody: Record = {}; + + globalThis.fetch = async (_url, init = {}) => { + capturedBody = JSON.parse(String((init as RequestInit).body)); + return new Response( + JSON.stringify({ results: [{ url: "https://example.com", text: "content" }] }), + { status: 200 } + ); + }; + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "links", + includeMetadata: false, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(capturedBody.format, "markdown"); + assert.equal(result.success, true); + assert.deepEqual(result.data?.links, []); + assert.equal(result.data?.screenshot_url, null); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch returns a failure when the URL is only present in the errors[] array", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => + new Response( + JSON.stringify({ + results: [], + errors: [{ url: "https://example.com", message: "could not reach host" }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 502); + assert.ok(result.error?.includes("could not reach host")); + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("tinyfishFetch maps AbortError to a 504 timeout", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + const err = new Error("aborted"); + err.name = "AbortError"; + throw err; + }; + + try { + const result = await tinyfishFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "tf-key" }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 504); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/mcp-web-fetch-tool.test.ts b/tests/unit/mcp-web-fetch-tool.test.ts index 2371dc72f8..beeac930ba 100644 --- a/tests/unit/mcp-web-fetch-tool.test.ts +++ b/tests/unit/mcp-web-fetch-tool.test.ts @@ -27,14 +27,8 @@ test("webFetchTool has the required McpToolDefinition shape", () => { test("webFetchTool is registered in MCP_TOOLS and MCP_TOOL_MAP", () => { const toolNames = MCP_TOOLS.map((t) => t.name); - assert.ok( - toolNames.includes("omniroute_web_fetch"), - "webFetchTool must be in MCP_TOOLS array" - ); - assert.ok( - "omniroute_web_fetch" in MCP_TOOL_MAP, - "webFetchTool must be in MCP_TOOL_MAP" - ); + assert.ok(toolNames.includes("omniroute_web_fetch"), "webFetchTool must be in MCP_TOOLS array"); + assert.ok("omniroute_web_fetch" in MCP_TOOL_MAP, "webFetchTool must be in MCP_TOOL_MAP"); }); // ── Scope mapping ── @@ -103,6 +97,11 @@ test("webFetchInput accepts depth values 0, 1, 2", () => { } }); +test("webFetchInput accepts provider=tinyfish", () => { + const parsed = webFetchInput.parse({ url: "https://example.com", provider: "tinyfish" }); + assert.equal(parsed.provider, "tinyfish"); +}); + test("webFetchInput rejects invalid provider", () => { assert.throws( () => webFetchInput.parse({ url: "https://example.com", provider: "unknown-provider" }), diff --git a/tests/unit/provider-validation-tinyfish.test.ts b/tests/unit/provider-validation-tinyfish.test.ts new file mode 100644 index 0000000000..1e4900d948 --- /dev/null +++ b/tests/unit/provider-validation-tinyfish.test.ts @@ -0,0 +1,47 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +// TinyFish Fetch API added as a webFetch-kind provider (docs.tinyfish.ai/fetch-api). +// These tests pin the validator dispatch (tinyfish -> POST api.fetch.tinyfish.ai with +// X-API-Key auth) and the auth-failure mapping, mirroring the firecrawl/jina-reader +// coverage in provider-validation-webfetch-4401.test.ts. + +const { validateProviderApiKey } = await import("../../src/lib/providers/validation.ts"); + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function headerValue(init: RequestInit | undefined, name: string): string | undefined { + const headers = (init?.headers || {}) as Record; + return headers[name]; +} + +test("tinyfish validator probes api.fetch.tinyfish.ai with X-API-Key auth and accepts a 200", async () => { + const calls: { url: string; init: RequestInit }[] = []; + globalThis.fetch = async (url, init = {}) => { + calls.push({ url: String(url), init }); + return new Response(JSON.stringify({ results: [{}], errors: [] }), { status: 200 }); + }; + + const result = await validateProviderApiKey({ provider: "tinyfish", apiKey: "tf-test-key" }); + + assert.equal(result.valid, true); + assert.equal(result.unsupported ?? false, false); + assert.equal(calls.length, 1); + assert.match(calls[0].url, /^https:\/\/api\.fetch\.tinyfish\.ai\/?$/); + assert.equal(calls[0].init.method, "POST"); + assert.equal(headerValue(calls[0].init, "X-API-Key"), "tf-test-key"); +}); + +test("tinyfish validator maps 401/403 to an invalid-key error", async () => { + globalThis.fetch = async () => + new Response(JSON.stringify({ error: "unauthorized" }), { status: 401 }); + + const result = await validateProviderApiKey({ provider: "tinyfish", apiKey: "bad" }); + + assert.equal(result.valid, false); + assert.equal(result.error, "Invalid API key"); +}); diff --git a/tests/unit/providers-constants-split.test.ts b/tests/unit/providers-constants-split.test.ts index 18e564fe1e..fcd6a272a9 100644 --- a/tests/unit/providers-constants-split.test.ts +++ b/tests/unit/providers-constants-split.test.ts @@ -1,7 +1,7 @@ // Characterization of the providers.ts catalog split (god-file decomposition): the host became a // barrel that re-exports 10 data catalogs now living under constants/providers/*, and APIKEY is // merged from 6 semantic family files (apikey/.ts). Locks: the public surface (every catalog -// + helpers still exported), the spread-merge integrity (170 APIKEY entries, no loss/dup), and that +// + helpers still exported), the spread-merge integrity (171 APIKEY entries, no loss/dup), and that // load-time Zod validation still runs. Pure-data move β†’ behavior must be identical. import { test } from "node:test"; import assert from "node:assert/strict"; @@ -31,12 +31,12 @@ test("barrel still exports every catalog + key helpers", () => { } }); -test("APIKEY_PROVIDERS merges the 6 family files into 170 entries (no loss / no dup)", async () => { +test("APIKEY_PROVIDERS merges the 6 family files into 171 entries (no loss / no dup)", async () => { const keys = Object.keys((P as Record).APIKEY_PROVIDERS); - assert.equal(keys.length, 170); - assert.equal(new Set(keys).size, 170, "duplicate keys after spread-merge"); + assert.equal(keys.length, 171); + assert.equal(new Set(keys).size, 171, "duplicate keys after spread-merge"); // the merged object's entry-count equals the sum of the 6 semantic family files; families are a - // strict partition (every provider in exactly one), so the sum must be exactly 170. + // strict partition (every provider in exactly one), so the sum must be exactly 171. const families: [string, string][] = [ ["gateways", "APIKEY_PROVIDERS_GATEWAYS"], ["frontier-labs", "APIKEY_PROVIDERS_FRONTIER"], @@ -56,7 +56,7 @@ test("APIKEY_PROVIDERS merges the 6 family files into 170 entries (no loss / no seen.add(k); } } - assert.equal(famTotal, 170, "families must partition all 170 providers"); + assert.equal(famTotal, 171, "families must partition all 171 providers"); }); test("AI_PROVIDERS Proxy aggregates all sections; lookups resolve", () => { diff --git a/tests/unit/web-fetch-handler.test.ts b/tests/unit/web-fetch-handler.test.ts index ca026c9bd0..edd4278010 100644 --- a/tests/unit/web-fetch-handler.test.ts +++ b/tests/unit/web-fetch-handler.test.ts @@ -70,6 +70,33 @@ test("handleWebFetch routes to jina-reader when provider=jina-reader", async () } }); +test("handleWebFetch routes to tinyfish when provider=tinyfish", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response( + JSON.stringify({ + results: [{ url: "https://example.com", title: "Test", text: "# TinyFish content" }], + errors: [], + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await handleWebFetch( + { url: "https://example.com", format: "markdown" }, + { apiKey: "tf-key" }, + "tinyfish" + ); + + assert.equal(result.success, true); + assert.equal(result.data?.provider, "tinyfish"); + } finally { + globalThis.fetch = originalFetch; + } +}); + test("handleWebFetch returns error 401 when no apiKey for firecrawl", async () => { const result = await handleWebFetch({ url: "https://example.com" }, {}, "firecrawl");