diff --git a/open-sse/executors/firecrawl-fetch.ts b/open-sse/executors/firecrawl-fetch.ts new file mode 100644 index 0000000000..a1938cd71c --- /dev/null +++ b/open-sse/executors/firecrawl-fetch.ts @@ -0,0 +1,143 @@ +/** + * Firecrawl Web Fetch Executor + * + * Fetches content from a URL using the Firecrawl scrape API. + * POST https://api.firecrawl.dev/v1/scrape + * + * Free tier: 500 fetches/month, no credit card required. + * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape + */ + +import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts"; +import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts"; + +const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1"; +const FIRECRAWL_TIMEOUT_MS = 30_000; + +function mapFormat(format: WebFetchFormat): string { + switch (format) { + case "html": + return "html"; + case "links": + return "links"; + case "screenshot": + return "screenshot"; + case "markdown": + default: + return "markdown"; + } +} + +interface FirecrawlScrapeOptions { + url: string; + format: WebFetchFormat; + depth: number; + waitForSelector?: string; + includeMetadata: boolean; + credentials: WebFetchCredentials; +} + +/** + * Execute a Firecrawl scrape request. + */ +export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise { + const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts; + + if (!credentials.apiKey) { + const body = buildErrorBody(401, "Firecrawl API key required"); + return { success: false, status: 401, error: body.error.message }; + } + + const formats = [mapFormat(format)]; + + const requestBody: Record = { + url, + formats, + }; + + if (includeMetadata) { + requestBody.includeTags = ["title", "description", "og:title", "og:description"]; + } + + if (depth > 0) { + requestBody.maxDepth = depth; + } + + if (waitForSelector) { + requestBody.waitFor = waitForSelector; + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS); + + try { + const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${credentials.apiKey}`, + }, + body: JSON.stringify(requestBody), + signal: controller.signal, + }); + + if (!response.ok) { + const rawError = await response.text().catch(() => `HTTP ${response.status}`); + const msg = sanitizeErrorMessage(`Firecrawl error ${response.status}: ${rawError}`); + const body = buildErrorBody(response.status, msg); + return { success: false, status: response.status, error: body.error.message }; + } + + const data = (await response.json()) as Record; + + const scraped = (data.data as Record | null) ?? {}; + + const content = + format === "html" + ? String(scraped.html ?? "") + : format === "links" + ? JSON.stringify(scraped.links ?? []) + : String(scraped.markdown ?? scraped.content ?? ""); + + const rawLinks = scraped.links; + const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : []; + + const rawMeta = scraped.metadata as Record | null | undefined; + const metadata = includeMetadata + ? { + title: rawMeta?.title != null ? String(rawMeta.title) : null, + description: rawMeta?.description != null ? String(rawMeta.description) : null, + } + : null; + + const screenshotUrl = + format === "screenshot" + ? scraped.screenshot != null + ? String(scraped.screenshot) + : null + : null; + + return { + success: true, + data: { + provider: "firecrawl", + url, + content, + links, + metadata, + screenshot_url: screenshotUrl, + }, + }; + } catch (err: unknown) { + if (err instanceof Error && err.name === "AbortError") { + const body = buildErrorBody(504, "Firecrawl request timed out"); + return { success: false, status: 504, error: body.error.message }; + } + const msg = + err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err)); + const body = buildErrorBody(502, msg); + return { success: false, status: 502, error: body.error.message }; + } finally { + clearTimeout(timeoutId); + } +} diff --git a/open-sse/executors/jina-reader-fetch.ts b/open-sse/executors/jina-reader-fetch.ts new file mode 100644 index 0000000000..cf5a8a2eb3 --- /dev/null +++ b/open-sse/executors/jina-reader-fetch.ts @@ -0,0 +1,119 @@ +/** + * Jina Reader Web Fetch Executor + * + * Fetches content from a URL using the Jina Reader API. + * GET https://r.jina.ai/{url} + * + * Free tier: 1M fetches/month. + * Docs: https://jina.ai/reader/ + */ + +import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts"; +import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts"; + +const JINA_READER_BASE = "https://r.jina.ai"; +const JINA_TIMEOUT_MS = 30_000; + +interface JinaReaderFetchOptions { + url: string; + format: WebFetchFormat; + includeMetadata: boolean; + credentials: WebFetchCredentials; +} + +/** + * Execute a Jina Reader fetch request. + * Jina Reader uses a URL-based approach: GET https://r.jina.ai/ + */ +export async function jinaReaderFetch(opts: JinaReaderFetchOptions): Promise { + const { url, format, includeMetadata, credentials } = opts; + + if (!credentials.apiKey) { + const body = buildErrorBody(401, "Jina Reader API key required"); + return { success: false, status: 401, error: body.error.message }; + } + + const headers: Record = { + Authorization: `Bearer ${credentials.apiKey}`, + Accept: "application/json", + "X-Return-Format": format === "html" ? "html" : "markdown", + }; + + if (includeMetadata) { + headers["X-With-Generated-Alt"] = "true"; + } + + if (format === "links") { + headers["X-Gather-All-Links-At-The-End"] = "true"; + } + + const encodedUrl = encodeURIComponent(url); + const requestUrl = `${JINA_READER_BASE}/${encodedUrl}`; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), JINA_TIMEOUT_MS); + + try { + const response = await fetch(requestUrl, { + method: "GET", + headers, + signal: controller.signal, + }); + + if (!response.ok) { + const rawError = await response.text().catch(() => `HTTP ${response.status}`); + const msg = sanitizeErrorMessage(`Jina Reader error ${response.status}: ${rawError}`); + const body = buildErrorBody(response.status, msg); + return { success: false, status: response.status, error: body.error.message }; + } + + // Jina Reader returns JSON with data.content or plain text + const contentType = response.headers.get("content-type") ?? ""; + let content = ""; + let links: string[] = []; + let metadata: { title: string | null; description: string | null } | null = null; + + if (contentType.includes("application/json")) { + const json = (await response.json()) as Record; + const responseData = (json.data as Record | null) ?? {}; + content = String(responseData.content ?? responseData.text ?? ""); + + if (includeMetadata) { + metadata = { + title: responseData.title != null ? String(responseData.title) : null, + description: responseData.description != null ? String(responseData.description) : null, + }; + } + + if (format === "links") { + const rawLinks = responseData.links; + links = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : []; + } + } else { + content = await response.text(); + } + + return { + success: true, + data: { + provider: "jina-reader", + url, + content, + links, + metadata, + screenshot_url: null, + }, + }; + } catch (err: unknown) { + if (err instanceof Error && err.name === "AbortError") { + const body = buildErrorBody(504, "Jina Reader request timed out"); + return { success: false, status: 504, error: body.error.message }; + } + const msg = + err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err)); + const body = buildErrorBody(502, msg); + return { success: false, status: 502, error: body.error.message }; + } finally { + clearTimeout(timeoutId); + } +} diff --git a/open-sse/executors/tavily-fetch.ts b/open-sse/executors/tavily-fetch.ts new file mode 100644 index 0000000000..9ca3437fa4 --- /dev/null +++ b/open-sse/executors/tavily-fetch.ts @@ -0,0 +1,103 @@ +/** + * Tavily Web Fetch Executor + * + * Fetches content from a URL using the Tavily Extract API. + * POST https://api.tavily.com/extract + * + * Free tier: included in Tavily plan. + * Docs: https://docs.tavily.com/documentation/api-reference/endpoint/post-extract + */ + +import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts"; +import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts"; + +const TAVILY_EXTRACT_URL = "https://api.tavily.com/extract"; +const TAVILY_TIMEOUT_MS = 30_000; + +interface TavilyFetchOptions { + url: string; + format: WebFetchFormat; + includeMetadata: boolean; + credentials: WebFetchCredentials; +} + +/** + * Execute a Tavily extract request. + * Tavily Extract returns the raw content of a given URL. + */ +export async function tavilyFetch(opts: TavilyFetchOptions): Promise { + const { url, includeMetadata, credentials } = opts; + + if (!credentials.apiKey) { + const body = buildErrorBody(401, "Tavily API key required"); + return { success: false, status: 401, error: body.error.message }; + } + + const requestBody: Record = { + api_key: credentials.apiKey, + urls: [url], + extract_depth: "basic", + }; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), TAVILY_TIMEOUT_MS); + + try { + const response = await fetch(TAVILY_EXTRACT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${credentials.apiKey}`, + }, + body: JSON.stringify(requestBody), + signal: controller.signal, + }); + + if (!response.ok) { + const rawError = await response.text().catch(() => `HTTP ${response.status}`); + const msg = sanitizeErrorMessage(`Tavily error ${response.status}: ${rawError}`); + const body = buildErrorBody(response.status, msg); + return { success: false, status: response.status, error: body.error.message }; + } + + const data = (await response.json()) as Record; + + const results = data.results as Array> | null; + const firstResult = results?.[0] ?? {}; + + const content = String(firstResult.raw_content ?? firstResult.content ?? ""); + + const rawLinks = firstResult.links; + const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : []; + + const metadata = includeMetadata + ? { + title: firstResult.title != null ? String(firstResult.title) : null, + description: null, + } + : null; + + return { + success: true, + data: { + provider: "tavily-search", + url, + content, + links, + metadata, + screenshot_url: null, + }, + }; + } catch (err: unknown) { + if (err instanceof Error && err.name === "AbortError") { + const body = buildErrorBody(504, "Tavily request timed out"); + return { success: false, status: 504, error: body.error.message }; + } + const msg = + err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err)); + const body = buildErrorBody(502, msg); + return { success: false, status: 502, error: body.error.message }; + } finally { + clearTimeout(timeoutId); + } +} diff --git a/open-sse/handlers/webFetch.ts b/open-sse/handlers/webFetch.ts new file mode 100644 index 0000000000..6e03f11b26 --- /dev/null +++ b/open-sse/handlers/webFetch.ts @@ -0,0 +1,121 @@ +/** + * Web Fetch Handler + * + * Handles POST /v1/web/fetch requests. + * Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, or Tavily). + * + * Request format: + * { + * "url": "https://example.com", + * "provider": "firecrawl" | "jina-reader" | "tavily-search", // optional + * "format": "markdown" | "html" | "links" | "screenshot", + * "depth": 0 | 1 | 2, + * "wait_for_selector": "main", + * "include_metadata": true + * } + */ + +import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; +import { firecrawlFetch } from "../executors/firecrawl-fetch.ts"; +import { jinaReaderFetch } from "../executors/jina-reader-fetch.ts"; +import { tavilyFetch } from "../executors/tavily-fetch.ts"; + +export type WebFetchFormat = "markdown" | "html" | "links" | "screenshot"; + +export interface WebFetchRequest { + url: string; + provider?: "firecrawl" | "jina-reader" | "tavily-search"; + format?: WebFetchFormat; + depth?: 0 | 1 | 2; + wait_for_selector?: string; + include_metadata?: boolean; +} + +export interface WebFetchResponse { + provider: string; + url: string; + content: string; + links: string[]; + metadata: { title: string | null; description: string | null } | null; + screenshot_url: string | null; +} + +export interface WebFetchResult { + success: boolean; + status?: number; + error?: string; + data?: WebFetchResponse; +} + +export interface WebFetchCredentials { + apiKey?: string; +} + +const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const; +type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]; + +/** + * Execute a web fetch request against the specified (or auto-selected) provider. + * + * @param req - Validated web fetch request body + * @param credentials - Provider API credentials (apiKey) + * @param resolvedProvider - Provider ID to use; if omitted auto-selects based on available creds + */ +export async function handleWebFetch( + req: WebFetchRequest, + credentials: WebFetchCredentials, + resolvedProvider?: WebFetchProviderId +): Promise { + const provider = resolvedProvider ?? req.provider ?? "firecrawl"; + + const format: WebFetchFormat = req.format ?? "markdown"; + const includeMetadata = req.include_metadata ?? false; + + try { + switch (provider) { + case "firecrawl": + return await firecrawlFetch({ + url: req.url, + format, + depth: req.depth ?? 0, + waitForSelector: req.wait_for_selector, + includeMetadata, + credentials, + }); + + case "jina-reader": + return await jinaReaderFetch({ + url: req.url, + format, + includeMetadata, + credentials, + }); + + case "tavily-search": + return await tavilyFetch({ + url: req.url, + format, + includeMetadata, + credentials, + }); + + default: { + const _exhaustive: never = provider; + return { + success: false, + status: 400, + error: `Unknown web fetch provider: ${_exhaustive}`, + }; + } + } + } catch (err: unknown) { + const msg = + err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err)); + const body = buildErrorBody(502, msg); + return { + success: false, + status: 502, + error: body.error.message, + }; + } +} diff --git a/open-sse/translator/helpers/schemaCoercion.ts b/open-sse/translator/helpers/schemaCoercion.ts index b87f4c7fde..eb47cb4483 100644 --- a/open-sse/translator/helpers/schemaCoercion.ts +++ b/open-sse/translator/helpers/schemaCoercion.ts @@ -1,4 +1,7 @@ -import { isDeepSeekReasoningModel, requiresReasoningReplay } from "../../services/reasoningCache.ts"; +import { + isDeepSeekReasoningModel, + requiresReasoningReplay, +} from "../../services/reasoningCache.ts"; /** * Shared sanitizers for tool payloads that arrive from IDEs/SDKs with diff --git a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx new file mode 100644 index 0000000000..2bbbb11df4 --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/MediaProviderPageClient.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { useTranslations } from "next-intl"; +import { Button } from "@/shared/components"; +import MediaProviderHeader from "../../components/MediaProviderHeader"; +import MediaProviderKindNav from "../../components/MediaProviderKindNav"; +import type { MediaKind } from "../../components/MediaProviderKindNav"; + +interface Connection { + id: string; + name?: string | null; + testStatus?: string | null; + isActive?: boolean; + lastErrorAt?: string | null; + lastError?: string | null; +} + +interface MediaProviderPageClientProps { + providerId: string; + providerName: string; + providerColor?: string; + kindLabel: string; + activeKind: MediaKind; + website?: string; + hasFree?: boolean; + freeNote?: string; +} + +export default function MediaProviderPageClient({ + providerId, + providerName, + providerColor, + kindLabel, + activeKind, + website, + hasFree, + freeNote, +}: MediaProviderPageClientProps) { + const t = useTranslations("media"); + const [connections, setConnections] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + const fetchConnections = async () => { + try { + const res = await fetch(`/api/providers?provider=${encodeURIComponent(providerId)}`); + if (res.ok) { + const data = await res.json(); + setConnections( + (data.connections ?? []).filter( + (c: Connection & { provider?: string }) => c.provider === providerId + ) + ); + } + } catch { + // ignore + } finally { + setLoading(false); + } + }; + void fetchConnections(); + }, [providerId]); + + const backHref = `/dashboard/media-providers/${activeKind}`; + + return ( +
+ + + + + {/* Connections */} +
+
+

+ {t("connections", { count: connections.length })} +

+ +
+ + {loading ? ( +
{t("loading")}
+ ) : connections.length === 0 ? ( +
+ key_off + {t("noConnections")} + +
+ ) : ( +
+ {connections.map((conn) => ( +
+ + {conn.name ?? conn.id} + {conn.isActive === false && ( + + disabled + + )} +
+ ))} +
+ )} +
+ + {/* Playground placeholder — Fase 4 */} +
+
+ labs +

Example (Playground inline — Fase 4)

+
+

+ Inline playground for this provider will be available in the next release. +

+
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/page.tsx b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/page.tsx new file mode 100644 index 0000000000..907ec38cc6 --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/[kind]/[id]/page.tsx @@ -0,0 +1,66 @@ +import { notFound } from "next/navigation"; +import { getTranslations } from "next-intl/server"; +import { AI_PROVIDERS } from "@/shared/constants/providers"; +import type { MediaKind } from "../../components/MediaProviderKindNav"; +import { MEDIA_KINDS } from "../../components/MediaProviderKindNav"; +import MediaProviderPageClient from "./MediaProviderPageClient"; + +interface PageProps { + params: Promise<{ kind: string; id: string }>; +} + +/** + * /dashboard/media-providers/[kind]/[id] + * + * Individual provider page for a media-service provider. + * Validates both kind and id; 404 if either is unknown or the provider + * does not declare the requested kind. + */ +export default async function MediaProviderDetailPage({ params }: PageProps) { + const { kind, id } = await params; + + // Validate kind + if (!MEDIA_KINDS.includes(kind as MediaKind)) { + notFound(); + } + + const validKind = kind as MediaKind; + + // Validate provider exists in AI_PROVIDERS + const provider = Object.values(AI_PROVIDERS).find((p) => p.id === id) as + | (Record & { + id: string; + name: string; + color?: string; + website?: string; + hasFree?: boolean; + freeNote?: string; + }) + | undefined; + + if (!provider) { + notFound(); + } + + // Validate that the provider declares this kind + const serviceKinds = provider.serviceKinds as string[] | undefined; + if (!Array.isArray(serviceKinds) || !serviceKinds.includes(validKind)) { + notFound(); + } + + const t = await getTranslations("media"); + const kindLabel = t(`kinds.${validKind}`); + + return ( + + ); +} diff --git a/src/app/(dashboard)/dashboard/media-providers/[kind]/page.tsx b/src/app/(dashboard)/dashboard/media-providers/[kind]/page.tsx new file mode 100644 index 0000000000..a9aa1c83bd --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/[kind]/page.tsx @@ -0,0 +1,91 @@ +import { notFound } from "next/navigation"; +import { getTranslations } from "next-intl/server"; +import Link from "next/link"; +import { AI_PROVIDERS } from "@/shared/constants/providers"; +import type { MediaKind } from "../components/MediaProviderKindNav"; +import MediaProviderKindNav, { MEDIA_KINDS } from "../components/MediaProviderKindNav"; +import ProviderIcon from "@/shared/components/ProviderIcon"; + +interface PageProps { + params: Promise<{ kind: string }>; +} + +/** + * /dashboard/media-providers/[kind] + * + * Lists all AI providers that declare the given serviceKind. + * Returns 404 for unknown kinds. + */ +export default async function MediaProviderKindPage({ params }: PageProps) { + const { kind } = await params; + + // Validate kind + if (!MEDIA_KINDS.includes(kind as MediaKind)) { + notFound(); + } + + const validKind = kind as MediaKind; + const t = await getTranslations("media"); + + // Filter providers that support this kind + const matchingProviders = Object.values(AI_PROVIDERS).filter((p) => { + const serviceKinds = (p as Record).serviceKinds as string[] | undefined; + return Array.isArray(serviceKinds) && serviceKinds.includes(validKind); + }); + + const kindLabel = t(`kinds.${validKind}`); + + return ( +
+
+

{kindLabel}

+

{t("noProviders")}

+
+ + + + {matchingProviders.length === 0 ? ( +
+ category +

{t("noProviders")}

+
+ ) : ( +
+ {matchingProviders.map((provider) => { + const p = provider as Record & { + id: string; + name: string; + color?: string; + hasFree?: boolean; + freeNote?: string; + }; + return ( + +
+
+
+ +
+ {p.name} +
+ {p.hasFree && ( + + Free + + )} +
+ + ); + })} +
+ )} +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderHeader.tsx b/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderHeader.tsx new file mode 100644 index 0000000000..a16146106d --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderHeader.tsx @@ -0,0 +1,75 @@ +"use client"; + +import Link from "next/link"; +import { useTranslations } from "next-intl"; +import ProviderIcon from "@/shared/components/ProviderIcon"; + +interface MediaProviderHeaderProps { + providerId: string; + providerName: string; + providerColor?: string; + kindLabel: string; + website?: string; + hasFree?: boolean; + freeNote?: string; + backHref: string; +} + +export default function MediaProviderHeader({ + providerId, + providerName, + providerColor, + kindLabel, + website, + hasFree, + freeNote, + backHref, +}: MediaProviderHeaderProps) { + const t = useTranslations("media"); + + return ( +
+ + arrow_back + {t("backToProviders")} + + +
+
+ +
+
+
+

{providerName}

+ + {kindLabel} + + {hasFree && ( + + Free + + )} +
+ {freeNote &&

{freeNote}

} + {website && ( + + open_in_new + {website} + + )} +
+
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderKindNav.tsx b/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderKindNav.tsx new file mode 100644 index 0000000000..56d9ff3ec0 --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/components/MediaProviderKindNav.tsx @@ -0,0 +1,58 @@ +"use client"; + +import Link from "next/link"; +import { useTranslations } from "next-intl"; + +export type MediaKind = + | "embedding" + | "image" + | "imageToText" + | "tts" + | "stt" + | "webSearch" + | "webFetch" + | "video" + | "music"; + +export const MEDIA_KINDS: MediaKind[] = [ + "embedding", + "image", + "imageToText", + "tts", + "stt", + "webSearch", + "webFetch", + "video", + "music", +]; + +interface MediaProviderKindNavProps { + activeKind: MediaKind; +} + +export default function MediaProviderKindNav({ activeKind }: MediaProviderKindNavProps) { + const t = useTranslations("media"); + + return ( +
+ {MEDIA_KINDS.map((kind) => { + const isActive = kind === activeKind; + const label = t(`kinds.${kind}`); + return ( + + {label} + + ); + })} +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/media-providers/page.tsx b/src/app/(dashboard)/dashboard/media-providers/page.tsx new file mode 100644 index 0000000000..d2ec5aa039 --- /dev/null +++ b/src/app/(dashboard)/dashboard/media-providers/page.tsx @@ -0,0 +1,9 @@ +import { redirect } from "next/navigation"; + +/** + * /dashboard/media-providers + * Default redirect to the embedding kind list. + */ +export default function MediaProvidersPage() { + redirect("/dashboard/media-providers/embedding"); +} diff --git a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx index fbd4f0605b..8cdd2d07ae 100644 --- a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx +++ b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx @@ -27,6 +27,19 @@ interface ProviderStats { codexFastActive?: boolean; } +const KIND_LABEL: Record = { + llm: "Chat", + embedding: "Embed", + image: "Image", + imageToText: "I→T", + tts: "TTS", + stt: "STT", + webSearch: "Search", + webFetch: "Fetch", + video: "Video", + music: "Music", +}; + interface ProviderCardProps { providerId: string; provider: { @@ -39,6 +52,7 @@ interface ProviderCardProps { hasFree?: boolean; freeNote?: string; subscriptionRisk?: boolean; + serviceKinds?: string[]; }; stats: ProviderStats; authType?: string; @@ -182,6 +196,18 @@ export default function ProviderCard({ )}
+ {provider.serviceKinds && provider.serviceKinds.length > 0 && ( +
+ {provider.serviceKinds.map((k) => ( + + {KIND_LABEL[k] ?? k} + + ))} +
+ )}

e.toggleAuthType === "oauth") .filter((e) => !IDE_PROVIDER_IDS.has(e.providerId)); + + // Web Fetch providers: filter across all entries by serviceKinds + const webFetchEntriesAll = dedupeProviderEntries( + [...staticProviderEntriesAll, ...compatibleProviderEntriesAll].filter((e) => { + const p = e.provider as DashboardProviderInfo & { serviceKinds?: string[] }; + return p.serviceKinds?.includes("webFetch") === true; + }) as DashboardProviderEntry[] + ); + const webFetchEntries = filterConfiguredProviderEntries( + webFetchEntriesAll, + effectiveShowConfiguredOnly, + searchQuery, + showFreeOnly + ); + const summaryStats = { all: countConfigured(dashboardProviderEntriesAll), free: countConfigured(freeSectionEntriesAll), @@ -664,6 +679,7 @@ export default function ProvidersPage() { upstreamproxy: countConfigured(upstreamProxyEntriesAll), cloudagent: countConfigured(cloudAgentProviderEntriesAll), ide: countConfigured(ideProviderEntriesAll), + webfetch: countConfigured(webFetchEntriesAll), }; if (loading) { @@ -815,6 +831,13 @@ export default function ProvidersPage() { label: "Search", stat: summaryStats.search, }, + { + key: "webfetch", + color: "bg-orange-500", + label: t("webFetch"), + stat: summaryStats.webfetch, + title: t("webFetchTooltip"), + }, { key: "audio", color: "bg-rose-500", @@ -1333,6 +1356,33 @@ export default function ProvidersPage() {

)} + {/* Web Fetch Providers */} + {showSection("webfetch") && webFetchEntries.length > 0 && ( +
+
+

+ {t("webFetchProvidersHeading")}{" "} + + +

+
+
+ {webFetchEntries.map( + ({ providerId, provider, stats, displayAuthType, toggleAuthType }) => ( + handleToggleProvider(providerId, toggleAuthType, active)} + /> + ) + )} +
+
+ )} + {/* Aggregators Gateways */} {showSection("apikey") && aggregatorProviderEntries.length > 0 && (
diff --git a/src/app/api/v1/web/fetch/route.ts b/src/app/api/v1/web/fetch/route.ts new file mode 100644 index 0000000000..e78eb8399f --- /dev/null +++ b/src/app/api/v1/web/fetch/route.ts @@ -0,0 +1,139 @@ +/** + * POST /v1/web/fetch + * + * Extract content from a URL using a configured web-fetch provider. + * Supports Firecrawl, Jina Reader, and Tavily Extract. + * + * Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? } + * Response: { provider, url, content, links, metadata, screenshot_url } + */ + +import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; +import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; +import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts"; +import * as log from "@/sse/utils/logger"; +import { extractApiKey, isValidApiKey, getProviderCredentials } from "@/sse/services/auth"; +import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; +import { v1WebFetchSchema } from "@/shared/validation/schemas"; +import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; + +const CORS_HEADERS = { + "Access-Control-Allow-Methods": "POST, OPTIONS", + "Access-Control-Allow-Headers": "*", +}; + +const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const; +type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number]; + +export async function OPTIONS() { + return new Response(null, { headers: CORS_HEADERS }); +} + +/** + * Resolve credentials for a web-fetch provider. Tries each known provider in + * priority order when no explicit provider is requested. + */ +async function resolveCredentials( + providerId: WebFetchProviderId +): Promise<{ apiKey?: string } | null> { + try { + const creds = await getProviderCredentials(providerId); + return creds ?? null; + } catch { + return null; + } +} + +export async function POST(request: Request) { + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + log.warn("WEB_FETCH", "Invalid JSON body"); + return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body"); + } + + const validation = validateBody(v1WebFetchSchema, rawBody); + if (isValidationFailure(validation)) { + return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message); + } + const body = validation.data; + + // Optional auth check + const apiKeyRaw = extractApiKey(request); + if (process.env.REQUIRE_API_KEY === "true" && !apiKeyRaw) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Authentication required"); + } + if (apiKeyRaw && !(await isValidApiKey(apiKeyRaw))) { + return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key"); + } + + // Enforce API key policies + const policy = await enforceApiKeyPolicy(request, "web-fetch"); + if (policy.rejection) return policy.rejection; + + // Resolve provider + credentials + let resolvedProvider: WebFetchProviderId | undefined; + let credentials: { apiKey?: string } = {}; + + if (body.provider) { + resolvedProvider = body.provider as WebFetchProviderId; + const creds = await resolveCredentials(resolvedProvider); + if (!creds) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `No credentials configured for web-fetch provider: ${resolvedProvider}. ` + + `Add an API key for "${resolvedProvider}" in the dashboard.` + ); + } + credentials = creds; + } else { + // Auto-select: try providers in priority order + for (const pid of WEB_FETCH_PROVIDERS) { + const creds = await resolveCredentials(pid); + if (creds) { + resolvedProvider = pid; + credentials = creds; + break; + } + } + if (!resolvedProvider) { + return errorResponse( + HTTP_STATUS.BAD_REQUEST, + `No credentials configured for any web-fetch provider. ` + + `Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.` + ); + } + } + + log.info("WEB_FETCH", `${resolvedProvider} | ${body.url} | format=${body.format}`); + + const result = await handleWebFetch( + { + url: body.url, + format: body.format, + depth: body.depth as 0 | 1 | 2, + wait_for_selector: body.wait_for_selector, + include_metadata: body.include_metadata, + }, + credentials, + resolvedProvider + ); + + if (!result.success) { + return new Response( + JSON.stringify({ + error: { message: result.error ?? "Web fetch failed", type: "web_fetch_error" }, + }), + { + status: result.status ?? 502, + headers: { "Content-Type": "application/json", ...CORS_HEADERS }, + } + ); + } + + return new Response(JSON.stringify(result.data), { + status: 200, + headers: { "Content-Type": "application/json", ...CORS_HEADERS }, + }); +} diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 0cc07bdafa..efe820ce83 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1507,7 +1507,24 @@ "result": "Result", "imageDescription": "Generate images from text prompts using OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI and more.", "videoDescription": "Create videos with AnimateDiff, Stable Video Diffusion via ComfyUI or SD WebUI.", - "musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI." + "musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI.", + "kinds": { + "embedding": "Embedding", + "image": "Image", + "imageToText": "Image to Text", + "tts": "Text to Speech", + "stt": "Speech to Text", + "webSearch": "Web Search", + "webFetch": "Web Fetch", + "video": "Video", + "music": "Music" + }, + "noProviders": "No providers configured for this kind yet.", + "addConnection": "Add Connection", + "backToProviders": "Back to Providers", + "connections": "{count} Connections", + "noConnections": "No connections yet — add one from the provider page.", + "loading": "Loading..." }, "search": { "searchQuery": "Search Query", @@ -3917,7 +3934,10 @@ "providerDetailValidClaudeCredentialsFile": "Valid Claude credentials file", "providerDetailPathAutoDetectedAllOs": "Path is auto-detected per OS (Linux/Mac/Windows).", "providerDetailMyClaudeAccountPlaceholder": "My Claude account", - "providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac)." + "providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac).", + "webFetch": "Web Fetch", + "webFetchTooltip": "Providers that extract content from web URLs (HTML → Markdown, scrape, screenshot)", + "webFetchProvidersHeading": "Web Fetch Providers" }, "settings": { "title": "Settings", diff --git a/src/i18n/messages/pt-BR.json b/src/i18n/messages/pt-BR.json index 07c7efacfb..4b1d6fe6b0 100644 --- a/src/i18n/messages/pt-BR.json +++ b/src/i18n/messages/pt-BR.json @@ -1505,7 +1505,24 @@ "result": "Resultado", "imageDescription": "Gere imagens a partir de prompts de texto usando OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI e mais.", "videoDescription": "Crie vídeos com AnimateDiff, Stable Video Diffusion via ComfyUI ou SD WebUI.", - "musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI." + "musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI.", + "kinds": { + "embedding": "Embedding", + "image": "Imagem", + "imageToText": "Imagem para Texto", + "tts": "Texto para Fala", + "stt": "Fala para Texto", + "webSearch": "Busca Web", + "webFetch": "Web Fetch", + "video": "Vídeo", + "music": "Música" + }, + "noProviders": "Nenhum provedor configurado para este tipo.", + "addConnection": "Adicionar Conexão", + "backToProviders": "Voltar para Provedores", + "connections": "{count} Conexões", + "noConnections": "Nenhuma conexão ainda — adicione uma na página do provedor.", + "loading": "Carregando..." }, "search": { "searchQuery": "Consulta de Pesquisa", @@ -3914,7 +3931,10 @@ "providerDetailValidClaudeCredentialsFile": "Arquivo de credenciais do Claude válido", "providerDetailPathAutoDetectedAllOs": "O caminho é detectado automaticamente por SO (Linux/Mac/Windows).", "providerDetailMyClaudeAccountPlaceholder": "Minha conta do Claude", - "providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac)." + "providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac).", + "webFetch": "Web Fetch", + "webFetchTooltip": "Provedores que extraem conteúdo de URLs (HTML → Markdown, scraping, screenshot)", + "webFetchProvidersHeading": "Provedores Web Fetch" }, "settings": { "title": "Configurações", diff --git a/src/shared/constants/providers.ts b/src/shared/constants/providers.ts index 9b6b727588..2768596dab 100644 --- a/src/shared/constants/providers.ts +++ b/src/shared/constants/providers.ts @@ -2,6 +2,22 @@ export type RiskNoticeVariant = "oauth" | "webCookie" | "deprecated"; +/** + * Service kind — declarative tag for what a provider can do beyond basic LLM chat. + * Affects UI filtering only; does not influence request routing. + */ +export type ServiceKind = + | "llm" + | "embedding" + | "image" + | "imageToText" + | "tts" + | "stt" + | "webSearch" + | "webFetch" + | "video" + | "music"; + export interface ProviderRiskNoticeFields { subscriptionRisk?: boolean; riskNoticeVariant?: RiskNoticeVariant; @@ -2118,6 +2134,52 @@ export const APIKEY_PROVIDERS = { passthroughModels: true, authHint: "Get API key at monsterapi.ai", }, + // ── Web Fetch Providers ───────────────────────────────────────────────────── + firecrawl: { + id: "firecrawl", + alias: "fc", + name: "Firecrawl", + icon: "language", + color: "#FB923C", + textIcon: "FC", + website: "https://firecrawl.dev", + hasFree: true, + notice: { + text: "Free tier: 500 fetches/month, no credit card needed.", + apiKeyUrl: "https://firecrawl.dev/app/api-keys", + }, + serviceKinds: ["webFetch"], + }, + "jina-reader": { + id: "jina-reader", + alias: "jr", + name: "Jina Reader", + icon: "menu_book", + color: "#0EA5E9", + textIcon: "JR", + website: "https://jina.ai/reader", + hasFree: true, + notice: { + text: "Free tier: 1M fetches/month.", + apiKeyUrl: "https://jina.ai/api-dashboard", + }, + serviceKinds: ["webFetch"], + }, + byteplus: { + id: "byteplus", + alias: "bpm", + name: "BytePlus ModelArk", + icon: "cloud", + color: "#2563EB", + textIcon: "BP", + website: "https://console.byteplus.com/ark", + hasFree: true, + notice: { + text: "Free credits for new accounts. Seed 2.0, Kimi K2 Thinking, GLM 4.7, GPT-OSS-120B available.", + apiKeyUrl: "https://console.byteplus.com/ark/region:ark+ap-southeast-1/apiKey", + }, + serviceKinds: ["llm"], + }, }; // Sub-categories within APIKEY_PROVIDERS (used by dashboard and catalog views). @@ -2355,6 +2417,7 @@ export const SEARCH_PROVIDERS = { website: "https://serper.dev", hasFree: true, authHint: "API key from serper.dev dashboard", + serviceKinds: ["webSearch"], }, "brave-search": { id: "brave-search", @@ -2377,6 +2440,7 @@ export const SEARCH_PROVIDERS = { website: "https://exa.ai", hasFree: true, authHint: "API key from dashboard.exa.ai", + serviceKinds: ["webSearch", "webFetch"], }, "tavily-search": { id: "tavily-search", @@ -2388,6 +2452,7 @@ export const SEARCH_PROVIDERS = { website: "https://tavily.com", hasFree: true, authHint: "API key from app.tavily.com (format: tvly-...)", + serviceKinds: ["webSearch", "webFetch"], }, "google-pse-search": { id: "google-pse-search", diff --git a/src/shared/validation/providerSchema.ts b/src/shared/validation/providerSchema.ts index 44a56419e9..882437a7d0 100644 --- a/src/shared/validation/providerSchema.ts +++ b/src/shared/validation/providerSchema.ts @@ -10,6 +10,19 @@ import { z } from "zod"; +const SERVICE_KIND_VALUES = [ + "llm", + "embedding", + "image", + "imageToText", + "tts", + "stt", + "webSearch", + "webFetch", + "video", + "music", +] as const; + export const ProviderSchema = z.object({ id: z.string().min(1), alias: z.string().min(1).optional(), @@ -27,6 +40,7 @@ export const ProviderSchema = z.object({ freeNote: z.string().optional(), authHint: z.string().optional(), apiHint: z.string().optional(), + serviceKinds: z.array(z.enum(SERVICE_KIND_VALUES)).optional(), }); export const ProvidersMapSchema = z.record(z.string(), ProviderSchema); diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index de087f5f73..ceec89a032 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -2147,3 +2147,14 @@ export const v1BatchCreateSchema = z.object({ }) .optional(), }); + +// ── Web Fetch ───────────────────────────────────────────────────────────────── + +export const v1WebFetchSchema = z.object({ + url: z.string().url("url must be a valid URL (http/https)"), + provider: z.enum(["firecrawl", "jina-reader", "tavily-search"]).optional(), + format: z.enum(["markdown", "html", "links", "screenshot"]).default("markdown"), + depth: z.union([z.literal(0), z.literal(1), z.literal(2)]).default(0), + wait_for_selector: z.string().max(256).optional(), + include_metadata: z.boolean().default(false), +}); diff --git a/tests/unit/firecrawl-executor.test.ts b/tests/unit/firecrawl-executor.test.ts new file mode 100644 index 0000000000..395a77c219 --- /dev/null +++ b/tests/unit/firecrawl-executor.test.ts @@ -0,0 +1,185 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { firecrawlFetch } = await import("../../open-sse/executors/firecrawl-fetch.ts"); + +// ── firecrawlFetch tests ────────────────────────────────────────────────────── + +test("firecrawlFetch calls api.firecrawl.dev/v1/scrape with Bearer auth", async () => { + const originalFetch = globalThis.fetch; + let captured: { url: string; headers: Record; body: Record } = { + url: "", + headers: {}, + body: {}, + }; + + globalThis.fetch = async (url, init = {}) => { + captured = { + url: String(url), + headers: (init as RequestInit).headers as Record, + body: JSON.parse(String((init as RequestInit).body ?? "{}")), + }; + return new Response( + JSON.stringify({ data: { markdown: "# Result", links: [], metadata: { title: "Test" } } }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: { apiKey: "fc-test-key" }, + }); + + assert.equal(result.success, true); + assert.equal(captured.url, "https://api.firecrawl.dev/v1/scrape"); + assert.equal(captured.headers["Authorization"], "Bearer fc-test-key"); + assert.deepEqual(captured.body.formats, ["markdown"]); + assert.equal(captured.body.url, "https://example.com"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("firecrawlFetch propagates 401 error without stack trace", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response(JSON.stringify({ error: "Unauthorized" }), { + status: 401, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: { apiKey: "bad-key" }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(result.error, "should have error"); + assert.ok(!result.error.includes("at /"), "error must not contain stack trace"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("firecrawlFetch returns 401 error when no API key", async () => { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(!result.error?.includes("at /")); +}); + +test("firecrawlFetch maps 'html' format correctly", async () => { + const originalFetch = globalThis.fetch; + let capturedFormats: unknown; + + globalThis.fetch = async (_url, init = {}) => { + const body = JSON.parse(String((init as RequestInit).body ?? "{}")); + capturedFormats = body.formats; + return new Response(JSON.stringify({ data: { html: "test", links: [] } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "html", + depth: 0, + includeMetadata: false, + credentials: { apiKey: "fc-key" }, + }); + + assert.equal(result.success, true); + assert.deepEqual(capturedFormats, ["html"]); + assert.ok(result.data?.content.includes(""), "should return html content"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("firecrawlFetch returns correct response shape", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response( + JSON.stringify({ + data: { + markdown: "# Page Title\nContent here.", + links: ["https://example.com/about", "https://example.com/contact"], + metadata: { title: "Page Title", description: "A test page" }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 0, + includeMetadata: true, + credentials: { apiKey: "fc-key" }, + }); + + assert.equal(result.success, true); + assert.ok(result.data, "should have data"); + assert.equal(result.data.provider, "firecrawl"); + assert.equal(result.data.url, "https://example.com"); + assert.ok(result.data.content.includes("Page Title"), "content should contain title"); + assert.ok(Array.isArray(result.data.links), "links should be array"); + assert.equal(result.data.links.length, 2); + assert.ok(result.data.metadata?.title === "Page Title"); + assert.equal(result.data.screenshot_url, null); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("firecrawlFetch forwards depth and wait_for_selector", async () => { + const originalFetch = globalThis.fetch; + let capturedBody: Record = {}; + + globalThis.fetch = async (_url, init = {}) => { + capturedBody = JSON.parse(String((init as RequestInit).body ?? "{}")); + return new Response(JSON.stringify({ data: { markdown: "" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + await firecrawlFetch({ + url: "https://example.com", + format: "markdown", + depth: 1, + waitForSelector: "article", + includeMetadata: false, + credentials: { apiKey: "fc-key" }, + }); + + assert.equal(capturedBody.maxDepth, 1, "should set maxDepth"); + assert.equal(capturedBody.waitFor, "article", "should set waitFor"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/jina-reader-executor.test.ts b/tests/unit/jina-reader-executor.test.ts new file mode 100644 index 0000000000..563063dd91 --- /dev/null +++ b/tests/unit/jina-reader-executor.test.ts @@ -0,0 +1,174 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { jinaReaderFetch } = await import("../../open-sse/executors/jina-reader-fetch.ts"); + +// ── jinaReaderFetch tests ───────────────────────────────────────────────────── + +test("jinaReaderFetch calls r.jina.ai/{url} with Bearer auth", async () => { + const originalFetch = globalThis.fetch; + let captured: { url: string; headers: Record } = { url: "", headers: {} }; + + globalThis.fetch = async (url, init = {}) => { + captured = { + url: String(url), + headers: (init as RequestInit).headers as Record, + }; + return new Response("# Hello from Jina", { + status: 200, + headers: { "content-type": "text/plain" }, + }); + }; + + try { + const result = await jinaReaderFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "jina-test-key" }, + }); + + assert.equal(result.success, true); + assert.ok( + captured.url.startsWith("https://r.jina.ai/"), + `URL should start with https://r.jina.ai/, got: ${captured.url}` + ); + assert.ok( + captured.url.includes(encodeURIComponent("https://example.com")), + "URL should include encoded target URL" + ); + assert.equal(captured.headers["Authorization"], "Bearer jina-test-key"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("jinaReaderFetch returns 401 error when no API key", async () => { + const result = await jinaReaderFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: {}, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace"); +}); + +test("jinaReaderFetch propagates non-200 status without stack trace", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response("Forbidden", { + status: 403, + headers: { "content-type": "text/plain" }, + }); + }; + + try { + const result = await jinaReaderFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "bad-key" }, + }); + + assert.equal(result.success, false); + assert.equal(result.status, 403); + assert.ok(result.error, "should have error message"); + assert.ok(!result.error.includes("at /"), "error must not contain stack trace"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("jinaReaderFetch parses JSON response with data.content", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response( + JSON.stringify({ + data: { + content: "# Parsed content", + title: "Title from JSON", + description: "A description", + links: ["https://example.com/a", "https://example.com/b"], + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await jinaReaderFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: true, + credentials: { apiKey: "jina-key" }, + }); + + assert.equal(result.success, true); + assert.ok(result.data, "should have data"); + assert.equal(result.data.provider, "jina-reader"); + assert.ok(result.data.content.includes("Parsed content"), "content should be parsed"); + assert.ok(result.data.metadata?.title === "Title from JSON", "metadata title should be set"); + assert.equal(result.data.screenshot_url, null); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("jinaReaderFetch falls back to plain text when response is not JSON", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response("Plain text content from Jina", { + status: 200, + headers: { "content-type": "text/plain; charset=utf-8" }, + }); + }; + + try { + const result = await jinaReaderFetch({ + url: "https://example.com", + format: "markdown", + includeMetadata: false, + credentials: { apiKey: "jina-key" }, + }); + + assert.equal(result.success, true); + assert.ok( + result.data?.content.includes("Plain text content"), + "should include plain text content" + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("jinaReaderFetch sets X-Return-Format header to html for html format", async () => { + const originalFetch = globalThis.fetch; + let capturedHeaders: Record = {}; + + globalThis.fetch = async (_url, init = {}) => { + capturedHeaders = (init as RequestInit).headers as Record; + return new Response("content", { + status: 200, + headers: { "content-type": "text/html" }, + }); + }; + + try { + await jinaReaderFetch({ + url: "https://example.com", + format: "html", + includeMetadata: false, + credentials: { apiKey: "jina-key" }, + }); + + assert.equal(capturedHeaders["X-Return-Format"], "html"); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/tests/unit/web-fetch-handler.test.ts b/tests/unit/web-fetch-handler.test.ts new file mode 100644 index 0000000000..ca026c9bd0 --- /dev/null +++ b/tests/unit/web-fetch-handler.test.ts @@ -0,0 +1,138 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { handleWebFetch } = await import("../../open-sse/handlers/webFetch.ts"); + +// ── handleWebFetch — basic routing ─────────────────────────────────────────── + +test("handleWebFetch routes to firecrawl when provider=firecrawl", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response( + JSON.stringify({ + data: { + markdown: "# Hello World", + links: ["https://example.com/page"], + metadata: { title: "Hello", description: "A test page" }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await handleWebFetch( + { url: "https://example.com", format: "markdown" }, + { apiKey: "test-key" }, + "firecrawl" + ); + + assert.equal(result.success, true, "should succeed"); + assert.ok(result.data, "should have data"); + assert.equal(result.data.provider, "firecrawl"); + assert.equal(result.data.url, "https://example.com"); + assert.ok(typeof result.data.content === "string", "content should be string"); + assert.ok(Array.isArray(result.data.links), "links should be array"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleWebFetch routes to jina-reader when provider=jina-reader", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + return new Response( + JSON.stringify({ + data: { + content: "# Jina content", + title: "Test", + description: "desc", + links: [], + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }; + + try { + const result = await handleWebFetch( + { url: "https://example.com", format: "markdown" }, + { apiKey: "jina-key" }, + "jina-reader" + ); + + assert.equal(result.success, true); + assert.equal(result.data?.provider, "jina-reader"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleWebFetch returns error 401 when no apiKey for firecrawl", async () => { + const result = await handleWebFetch({ url: "https://example.com" }, {}, "firecrawl"); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(result.error, "should have error message"); + // Error must not expose stack traces + assert.ok(!result.error.includes("at /"), "error must not contain stack trace paths"); +}); + +test("handleWebFetch returns error 401 when no apiKey for jina-reader", async () => { + const result = await handleWebFetch({ url: "https://example.com" }, {}, "jina-reader"); + + assert.equal(result.success, false); + assert.equal(result.status, 401); + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths"); +}); + +test("handleWebFetch wraps fetch errors via buildErrorBody (no raw stack)", async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => { + throw new Error("at /internal/path/executor.ts:42:10\nnetwork failure"); + }; + + try { + const result = await handleWebFetch( + { url: "https://example.com" }, + { apiKey: "test-key" }, + "firecrawl" + ); + + assert.equal(result.success, false); + assert.ok(result.status != null, "should have status"); + // Stack trace must be stripped + assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleWebFetch passes depth and wait_for_selector to firecrawl", async () => { + const originalFetch = globalThis.fetch; + let captured: { body: Record } = { body: {} }; + + globalThis.fetch = async (_url, init) => { + captured.body = JSON.parse(String((init as RequestInit).body ?? "{}")); + return new Response(JSON.stringify({ data: { markdown: "" } }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + try { + await handleWebFetch( + { url: "https://example.com", depth: 2, wait_for_selector: "main" }, + { apiKey: "test-key" }, + "firecrawl" + ); + + assert.equal(captured.body.maxDepth, 2, "should forward depth"); + assert.equal(captured.body.waitFor, "main", "should forward wait_for_selector"); + } finally { + globalThis.fetch = originalFetch; + } +});