feat(dashboard): media providers pages + Web Fetch category (#2645)

Integrated into release/v3.8.3
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-05-23 22:12:02 -03:00
committed by GitHub
parent 40cc555b68
commit 6fa7de0ba7
22 changed files with 1787 additions and 5 deletions

View File

@@ -0,0 +1,143 @@
/**
* Firecrawl Web Fetch Executor
*
* Fetches content from a URL using the Firecrawl scrape API.
* POST https://api.firecrawl.dev/v1/scrape
*
* Free tier: 500 fetches/month, no credit card required.
* Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape
*/
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1";
const FIRECRAWL_TIMEOUT_MS = 30_000;
function mapFormat(format: WebFetchFormat): string {
switch (format) {
case "html":
return "html";
case "links":
return "links";
case "screenshot":
return "screenshot";
case "markdown":
default:
return "markdown";
}
}
interface FirecrawlScrapeOptions {
url: string;
format: WebFetchFormat;
depth: number;
waitForSelector?: string;
includeMetadata: boolean;
credentials: WebFetchCredentials;
}
/**
* Execute a Firecrawl scrape request.
*/
export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebFetchResult> {
const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts;
if (!credentials.apiKey) {
const body = buildErrorBody(401, "Firecrawl API key required");
return { success: false, status: 401, error: body.error.message };
}
const formats = [mapFormat(format)];
const requestBody: Record<string, unknown> = {
url,
formats,
};
if (includeMetadata) {
requestBody.includeTags = ["title", "description", "og:title", "og:description"];
}
if (depth > 0) {
requestBody.maxDepth = depth;
}
if (waitForSelector) {
requestBody.waitFor = waitForSelector;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS);
try {
const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${credentials.apiKey}`,
},
body: JSON.stringify(requestBody),
signal: controller.signal,
});
if (!response.ok) {
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
const msg = sanitizeErrorMessage(`Firecrawl error ${response.status}: ${rawError}`);
const body = buildErrorBody(response.status, msg);
return { success: false, status: response.status, error: body.error.message };
}
const data = (await response.json()) as Record<string, unknown>;
const scraped = (data.data as Record<string, unknown> | null) ?? {};
const content =
format === "html"
? String(scraped.html ?? "")
: format === "links"
? JSON.stringify(scraped.links ?? [])
: String(scraped.markdown ?? scraped.content ?? "");
const rawLinks = scraped.links;
const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
const rawMeta = scraped.metadata as Record<string, unknown> | null | undefined;
const metadata = includeMetadata
? {
title: rawMeta?.title != null ? String(rawMeta.title) : null,
description: rawMeta?.description != null ? String(rawMeta.description) : null,
}
: null;
const screenshotUrl =
format === "screenshot"
? scraped.screenshot != null
? String(scraped.screenshot)
: null
: null;
return {
success: true,
data: {
provider: "firecrawl",
url,
content,
links,
metadata,
screenshot_url: screenshotUrl,
},
};
} catch (err: unknown) {
if (err instanceof Error && err.name === "AbortError") {
const body = buildErrorBody(504, "Firecrawl request timed out");
return { success: false, status: 504, error: body.error.message };
}
const msg =
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
const body = buildErrorBody(502, msg);
return { success: false, status: 502, error: body.error.message };
} finally {
clearTimeout(timeoutId);
}
}

View File

@@ -0,0 +1,119 @@
/**
* Jina Reader Web Fetch Executor
*
* Fetches content from a URL using the Jina Reader API.
* GET https://r.jina.ai/{url}
*
* Free tier: 1M fetches/month.
* Docs: https://jina.ai/reader/
*/
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
const JINA_READER_BASE = "https://r.jina.ai";
const JINA_TIMEOUT_MS = 30_000;
interface JinaReaderFetchOptions {
url: string;
format: WebFetchFormat;
includeMetadata: boolean;
credentials: WebFetchCredentials;
}
/**
* Execute a Jina Reader fetch request.
* Jina Reader uses a URL-based approach: GET https://r.jina.ai/<url>
*/
export async function jinaReaderFetch(opts: JinaReaderFetchOptions): Promise<WebFetchResult> {
const { url, format, includeMetadata, credentials } = opts;
if (!credentials.apiKey) {
const body = buildErrorBody(401, "Jina Reader API key required");
return { success: false, status: 401, error: body.error.message };
}
const headers: Record<string, string> = {
Authorization: `Bearer ${credentials.apiKey}`,
Accept: "application/json",
"X-Return-Format": format === "html" ? "html" : "markdown",
};
if (includeMetadata) {
headers["X-With-Generated-Alt"] = "true";
}
if (format === "links") {
headers["X-Gather-All-Links-At-The-End"] = "true";
}
const encodedUrl = encodeURIComponent(url);
const requestUrl = `${JINA_READER_BASE}/${encodedUrl}`;
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), JINA_TIMEOUT_MS);
try {
const response = await fetch(requestUrl, {
method: "GET",
headers,
signal: controller.signal,
});
if (!response.ok) {
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
const msg = sanitizeErrorMessage(`Jina Reader error ${response.status}: ${rawError}`);
const body = buildErrorBody(response.status, msg);
return { success: false, status: response.status, error: body.error.message };
}
// Jina Reader returns JSON with data.content or plain text
const contentType = response.headers.get("content-type") ?? "";
let content = "";
let links: string[] = [];
let metadata: { title: string | null; description: string | null } | null = null;
if (contentType.includes("application/json")) {
const json = (await response.json()) as Record<string, unknown>;
const responseData = (json.data as Record<string, unknown> | null) ?? {};
content = String(responseData.content ?? responseData.text ?? "");
if (includeMetadata) {
metadata = {
title: responseData.title != null ? String(responseData.title) : null,
description: responseData.description != null ? String(responseData.description) : null,
};
}
if (format === "links") {
const rawLinks = responseData.links;
links = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
}
} else {
content = await response.text();
}
return {
success: true,
data: {
provider: "jina-reader",
url,
content,
links,
metadata,
screenshot_url: null,
},
};
} catch (err: unknown) {
if (err instanceof Error && err.name === "AbortError") {
const body = buildErrorBody(504, "Jina Reader request timed out");
return { success: false, status: 504, error: body.error.message };
}
const msg =
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
const body = buildErrorBody(502, msg);
return { success: false, status: 502, error: body.error.message };
} finally {
clearTimeout(timeoutId);
}
}

View File

@@ -0,0 +1,103 @@
/**
* Tavily Web Fetch Executor
*
* Fetches content from a URL using the Tavily Extract API.
* POST https://api.tavily.com/extract
*
* Free tier: included in Tavily plan.
* Docs: https://docs.tavily.com/documentation/api-reference/endpoint/post-extract
*/
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
const TAVILY_EXTRACT_URL = "https://api.tavily.com/extract";
const TAVILY_TIMEOUT_MS = 30_000;
interface TavilyFetchOptions {
url: string;
format: WebFetchFormat;
includeMetadata: boolean;
credentials: WebFetchCredentials;
}
/**
* Execute a Tavily extract request.
* Tavily Extract returns the raw content of a given URL.
*/
export async function tavilyFetch(opts: TavilyFetchOptions): Promise<WebFetchResult> {
const { url, includeMetadata, credentials } = opts;
if (!credentials.apiKey) {
const body = buildErrorBody(401, "Tavily API key required");
return { success: false, status: 401, error: body.error.message };
}
const requestBody: Record<string, unknown> = {
api_key: credentials.apiKey,
urls: [url],
extract_depth: "basic",
};
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), TAVILY_TIMEOUT_MS);
try {
const response = await fetch(TAVILY_EXTRACT_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${credentials.apiKey}`,
},
body: JSON.stringify(requestBody),
signal: controller.signal,
});
if (!response.ok) {
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
const msg = sanitizeErrorMessage(`Tavily error ${response.status}: ${rawError}`);
const body = buildErrorBody(response.status, msg);
return { success: false, status: response.status, error: body.error.message };
}
const data = (await response.json()) as Record<string, unknown>;
const results = data.results as Array<Record<string, unknown>> | null;
const firstResult = results?.[0] ?? {};
const content = String(firstResult.raw_content ?? firstResult.content ?? "");
const rawLinks = firstResult.links;
const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
const metadata = includeMetadata
? {
title: firstResult.title != null ? String(firstResult.title) : null,
description: null,
}
: null;
return {
success: true,
data: {
provider: "tavily-search",
url,
content,
links,
metadata,
screenshot_url: null,
},
};
} catch (err: unknown) {
if (err instanceof Error && err.name === "AbortError") {
const body = buildErrorBody(504, "Tavily request timed out");
return { success: false, status: 504, error: body.error.message };
}
const msg =
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
const body = buildErrorBody(502, msg);
return { success: false, status: 502, error: body.error.message };
} finally {
clearTimeout(timeoutId);
}
}

View File

@@ -0,0 +1,121 @@
/**
* Web Fetch Handler
*
* Handles POST /v1/web/fetch requests.
* Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, or Tavily).
*
* Request format:
* {
* "url": "https://example.com",
* "provider": "firecrawl" | "jina-reader" | "tavily-search", // optional
* "format": "markdown" | "html" | "links" | "screenshot",
* "depth": 0 | 1 | 2,
* "wait_for_selector": "main",
* "include_metadata": true
* }
*/
import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts";
import { firecrawlFetch } from "../executors/firecrawl-fetch.ts";
import { jinaReaderFetch } from "../executors/jina-reader-fetch.ts";
import { tavilyFetch } from "../executors/tavily-fetch.ts";
export type WebFetchFormat = "markdown" | "html" | "links" | "screenshot";
export interface WebFetchRequest {
url: string;
provider?: "firecrawl" | "jina-reader" | "tavily-search";
format?: WebFetchFormat;
depth?: 0 | 1 | 2;
wait_for_selector?: string;
include_metadata?: boolean;
}
export interface WebFetchResponse {
provider: string;
url: string;
content: string;
links: string[];
metadata: { title: string | null; description: string | null } | null;
screenshot_url: string | null;
}
export interface WebFetchResult {
success: boolean;
status?: number;
error?: string;
data?: WebFetchResponse;
}
export interface WebFetchCredentials {
apiKey?: string;
}
const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const;
type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number];
/**
* Execute a web fetch request against the specified (or auto-selected) provider.
*
* @param req - Validated web fetch request body
* @param credentials - Provider API credentials (apiKey)
* @param resolvedProvider - Provider ID to use; if omitted auto-selects based on available creds
*/
export async function handleWebFetch(
req: WebFetchRequest,
credentials: WebFetchCredentials,
resolvedProvider?: WebFetchProviderId
): Promise<WebFetchResult> {
const provider = resolvedProvider ?? req.provider ?? "firecrawl";
const format: WebFetchFormat = req.format ?? "markdown";
const includeMetadata = req.include_metadata ?? false;
try {
switch (provider) {
case "firecrawl":
return await firecrawlFetch({
url: req.url,
format,
depth: req.depth ?? 0,
waitForSelector: req.wait_for_selector,
includeMetadata,
credentials,
});
case "jina-reader":
return await jinaReaderFetch({
url: req.url,
format,
includeMetadata,
credentials,
});
case "tavily-search":
return await tavilyFetch({
url: req.url,
format,
includeMetadata,
credentials,
});
default: {
const _exhaustive: never = provider;
return {
success: false,
status: 400,
error: `Unknown web fetch provider: ${_exhaustive}`,
};
}
}
} catch (err: unknown) {
const msg =
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
const body = buildErrorBody(502, msg);
return {
success: false,
status: 502,
error: body.error.message,
};
}
}

View File

@@ -1,4 +1,7 @@
import { isDeepSeekReasoningModel, requiresReasoningReplay } from "../../services/reasoningCache.ts";
import {
isDeepSeekReasoningModel,
requiresReasoningReplay,
} from "../../services/reasoningCache.ts";
/**
* Shared sanitizers for tool payloads that arrive from IDEs/SDKs with

View File

@@ -0,0 +1,152 @@
"use client";
import { useState, useEffect } from "react";
import { useTranslations } from "next-intl";
import { Button } from "@/shared/components";
import MediaProviderHeader from "../../components/MediaProviderHeader";
import MediaProviderKindNav from "../../components/MediaProviderKindNav";
import type { MediaKind } from "../../components/MediaProviderKindNav";
interface Connection {
id: string;
name?: string | null;
testStatus?: string | null;
isActive?: boolean;
lastErrorAt?: string | null;
lastError?: string | null;
}
interface MediaProviderPageClientProps {
providerId: string;
providerName: string;
providerColor?: string;
kindLabel: string;
activeKind: MediaKind;
website?: string;
hasFree?: boolean;
freeNote?: string;
}
export default function MediaProviderPageClient({
providerId,
providerName,
providerColor,
kindLabel,
activeKind,
website,
hasFree,
freeNote,
}: MediaProviderPageClientProps) {
const t = useTranslations("media");
const [connections, setConnections] = useState<Connection[]>([]);
const [loading, setLoading] = useState(true);
useEffect(() => {
const fetchConnections = async () => {
try {
const res = await fetch(`/api/providers?provider=${encodeURIComponent(providerId)}`);
if (res.ok) {
const data = await res.json();
setConnections(
(data.connections ?? []).filter(
(c: Connection & { provider?: string }) => c.provider === providerId
)
);
}
} catch {
// ignore
} finally {
setLoading(false);
}
};
void fetchConnections();
}, [providerId]);
const backHref = `/dashboard/media-providers/${activeKind}`;
return (
<div className="flex flex-col gap-6">
<MediaProviderHeader
providerId={providerId}
providerName={providerName}
providerColor={providerColor}
kindLabel={kindLabel}
website={website}
hasFree={hasFree}
freeNote={freeNote}
backHref={backHref}
/>
<MediaProviderKindNav activeKind={activeKind} />
{/* Connections */}
<div className="flex flex-col gap-3">
<div className="flex items-center justify-between">
<h2 className="text-base font-semibold">
{t("connections", { count: connections.length })}
</h2>
<Button
size="sm"
icon="add"
onClick={() => {
window.location.href = `/dashboard/providers/${providerId}`;
}}
>
{t("addConnection")}
</Button>
</div>
{loading ? (
<div className="py-8 text-center text-sm text-text-muted">{t("loading")}</div>
) : connections.length === 0 ? (
<div className="flex flex-col items-center justify-center gap-2 py-12 border border-dashed border-border rounded-xl text-text-muted text-sm">
<span className="material-symbols-outlined text-[28px]">key_off</span>
<span>{t("noConnections")}</span>
<Button
size="sm"
icon="add"
onClick={() => {
window.location.href = `/dashboard/providers/${providerId}`;
}}
>
{t("addConnection")}
</Button>
</div>
) : (
<div className="flex flex-col divide-y divide-border rounded-xl border border-border overflow-hidden">
{connections.map((conn) => (
<div key={conn.id} className="flex items-center gap-3 px-4 py-3 bg-bg-card">
<span
className={`size-2 rounded-full shrink-0 ${
conn.testStatus === "active" || conn.testStatus === "success"
? "bg-green-500"
: conn.testStatus === "error" || conn.testStatus === "unavailable"
? "bg-red-500"
: "bg-text-muted"
}`}
/>
<span className="text-sm font-medium flex-1 truncate">{conn.name ?? conn.id}</span>
{conn.isActive === false && (
<span className="text-[10px] px-1.5 py-0.5 rounded bg-bg-subtle border border-border text-text-muted">
disabled
</span>
)}
</div>
))}
</div>
)}
</div>
{/* Playground placeholder — Fase 4 */}
<div className="flex flex-col gap-2 border border-dashed border-border rounded-xl p-6">
<div className="flex items-center gap-2 text-text-muted">
<span className="material-symbols-outlined text-[20px]">labs</span>
<h3 className="text-sm font-medium">Example (Playground inline Fase 4)</h3>
</div>
<p className="text-xs text-text-muted">
Inline playground for this provider will be available in the next release.
</p>
</div>
</div>
);
}

View File

@@ -0,0 +1,66 @@
import { notFound } from "next/navigation";
import { getTranslations } from "next-intl/server";
import { AI_PROVIDERS } from "@/shared/constants/providers";
import type { MediaKind } from "../../components/MediaProviderKindNav";
import { MEDIA_KINDS } from "../../components/MediaProviderKindNav";
import MediaProviderPageClient from "./MediaProviderPageClient";
interface PageProps {
params: Promise<{ kind: string; id: string }>;
}
/**
* /dashboard/media-providers/[kind]/[id]
*
* Individual provider page for a media-service provider.
* Validates both kind and id; 404 if either is unknown or the provider
* does not declare the requested kind.
*/
export default async function MediaProviderDetailPage({ params }: PageProps) {
const { kind, id } = await params;
// Validate kind
if (!MEDIA_KINDS.includes(kind as MediaKind)) {
notFound();
}
const validKind = kind as MediaKind;
// Validate provider exists in AI_PROVIDERS
const provider = Object.values(AI_PROVIDERS).find((p) => p.id === id) as
| (Record<string, unknown> & {
id: string;
name: string;
color?: string;
website?: string;
hasFree?: boolean;
freeNote?: string;
})
| undefined;
if (!provider) {
notFound();
}
// Validate that the provider declares this kind
const serviceKinds = provider.serviceKinds as string[] | undefined;
if (!Array.isArray(serviceKinds) || !serviceKinds.includes(validKind)) {
notFound();
}
const t = await getTranslations("media");
const kindLabel = t(`kinds.${validKind}`);
return (
<MediaProviderPageClient
providerId={provider.id}
providerName={provider.name}
providerColor={provider.color}
kindLabel={kindLabel}
activeKind={validKind}
website={provider.website}
hasFree={provider.hasFree}
freeNote={provider.freeNote}
/>
);
}

View File

@@ -0,0 +1,91 @@
import { notFound } from "next/navigation";
import { getTranslations } from "next-intl/server";
import Link from "next/link";
import { AI_PROVIDERS } from "@/shared/constants/providers";
import type { MediaKind } from "../components/MediaProviderKindNav";
import MediaProviderKindNav, { MEDIA_KINDS } from "../components/MediaProviderKindNav";
import ProviderIcon from "@/shared/components/ProviderIcon";
interface PageProps {
params: Promise<{ kind: string }>;
}
/**
* /dashboard/media-providers/[kind]
*
* Lists all AI providers that declare the given serviceKind.
* Returns 404 for unknown kinds.
*/
export default async function MediaProviderKindPage({ params }: PageProps) {
const { kind } = await params;
// Validate kind
if (!MEDIA_KINDS.includes(kind as MediaKind)) {
notFound();
}
const validKind = kind as MediaKind;
const t = await getTranslations("media");
// Filter providers that support this kind
const matchingProviders = Object.values(AI_PROVIDERS).filter((p) => {
const serviceKinds = (p as Record<string, unknown>).serviceKinds as string[] | undefined;
return Array.isArray(serviceKinds) && serviceKinds.includes(validKind);
});
const kindLabel = t(`kinds.${validKind}`);
return (
<div className="flex flex-col gap-6">
<div className="flex flex-col gap-2">
<h1 className="text-2xl font-semibold">{kindLabel}</h1>
<p className="text-sm text-text-muted">{t("noProviders")}</p>
</div>
<MediaProviderKindNav activeKind={validKind} />
{matchingProviders.length === 0 ? (
<div className="flex flex-col items-center justify-center py-16 gap-3 border border-dashed border-border rounded-xl text-text-muted">
<span className="material-symbols-outlined text-[32px]">category</span>
<p className="text-sm">{t("noProviders")}</p>
</div>
) : (
<div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5 gap-3">
{matchingProviders.map((provider) => {
const p = provider as Record<string, unknown> & {
id: string;
name: string;
color?: string;
hasFree?: boolean;
freeNote?: string;
};
return (
<Link
key={p.id}
href={`/dashboard/media-providers/${validKind}/${p.id}`}
className="group"
>
<div className="rounded-xl border border-border bg-bg-card p-3 hover:bg-black/[0.01] dark:hover:bg-white/[0.01] transition-colors cursor-pointer h-full flex flex-col gap-2">
<div className="flex items-center gap-2">
<div
className="size-7 rounded-lg flex items-center justify-center shrink-0"
style={{ backgroundColor: `${p.color ?? "#64748b"}15` }}
>
<ProviderIcon providerId={p.id} size={20} type="color" />
</div>
<span className="text-sm font-medium truncate">{p.name}</span>
</div>
{p.hasFree && (
<span className="text-[10px] px-1.5 py-0.5 rounded-full bg-green-500/10 border border-green-500/20 text-green-600 dark:text-green-400 w-fit">
Free
</span>
)}
</div>
</Link>
);
})}
</div>
)}
</div>
);
}

View File

@@ -0,0 +1,75 @@
"use client";
import Link from "next/link";
import { useTranslations } from "next-intl";
import ProviderIcon from "@/shared/components/ProviderIcon";
interface MediaProviderHeaderProps {
providerId: string;
providerName: string;
providerColor?: string;
kindLabel: string;
website?: string;
hasFree?: boolean;
freeNote?: string;
backHref: string;
}
export default function MediaProviderHeader({
providerId,
providerName,
providerColor,
kindLabel,
website,
hasFree,
freeNote,
backHref,
}: MediaProviderHeaderProps) {
const t = useTranslations("media");
return (
<div className="flex flex-col gap-3">
<Link
href={backHref}
className="inline-flex items-center gap-1.5 text-sm text-text-muted hover:text-text-primary transition-colors w-fit"
>
<span className="material-symbols-outlined text-[16px]">arrow_back</span>
{t("backToProviders")}
</Link>
<div className="flex items-start gap-4">
<div
className="size-12 rounded-xl flex items-center justify-center shrink-0"
style={{ backgroundColor: `${providerColor ?? "#64748b"}20` }}
>
<ProviderIcon providerId={providerId} size={32} type="color" />
</div>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 flex-wrap">
<h1 className="text-xl font-semibold">{providerName}</h1>
<span className="text-xs px-2 py-0.5 rounded-full bg-bg-subtle border border-border text-text-muted">
{kindLabel}
</span>
{hasFree && (
<span className="text-xs px-2 py-0.5 rounded-full bg-green-500/10 border border-green-500/20 text-green-600 dark:text-green-400">
Free
</span>
)}
</div>
{freeNote && <p className="text-sm text-text-muted mt-1">{freeNote}</p>}
{website && (
<a
href={website}
target="_blank"
rel="noopener noreferrer"
className="inline-flex items-center gap-1 text-xs text-primary hover:underline mt-1"
>
<span className="material-symbols-outlined text-[13px]">open_in_new</span>
{website}
</a>
)}
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,58 @@
"use client";
import Link from "next/link";
import { useTranslations } from "next-intl";
export type MediaKind =
| "embedding"
| "image"
| "imageToText"
| "tts"
| "stt"
| "webSearch"
| "webFetch"
| "video"
| "music";
export const MEDIA_KINDS: MediaKind[] = [
"embedding",
"image",
"imageToText",
"tts",
"stt",
"webSearch",
"webFetch",
"video",
"music",
];
interface MediaProviderKindNavProps {
activeKind: MediaKind;
}
export default function MediaProviderKindNav({ activeKind }: MediaProviderKindNavProps) {
const t = useTranslations("media");
return (
<div className="flex flex-wrap gap-2 border-b border-border pb-3">
{MEDIA_KINDS.map((kind) => {
const isActive = kind === activeKind;
const label = t(`kinds.${kind}`);
return (
<Link
key={kind}
href={`/dashboard/media-providers/${kind}`}
className={`flex items-center px-3 py-1.5 rounded-full border text-xs font-medium transition-colors ${
isActive
? "bg-primary text-white border-primary"
: "bg-bg-subtle border-border text-text-muted hover:text-text-primary hover:border-primary/30"
}`}
aria-current={isActive ? "page" : undefined}
>
{label}
</Link>
);
})}
</div>
);
}

View File

@@ -0,0 +1,9 @@
import { redirect } from "next/navigation";
/**
* /dashboard/media-providers
* Default redirect to the embedding kind list.
*/
export default function MediaProvidersPage() {
redirect("/dashboard/media-providers/embedding");
}

View File

@@ -27,6 +27,19 @@ interface ProviderStats {
codexFastActive?: boolean;
}
const KIND_LABEL: Record<string, string> = {
llm: "Chat",
embedding: "Embed",
image: "Image",
imageToText: "I→T",
tts: "TTS",
stt: "STT",
webSearch: "Search",
webFetch: "Fetch",
video: "Video",
music: "Music",
};
interface ProviderCardProps {
providerId: string;
provider: {
@@ -39,6 +52,7 @@ interface ProviderCardProps {
hasFree?: boolean;
freeNote?: string;
subscriptionRisk?: boolean;
serviceKinds?: string[];
};
stats: ProviderStats;
authType?: string;
@@ -182,6 +196,18 @@ export default function ProviderCard({
)}
</div>
<div className="min-w-0">
{provider.serviceKinds && provider.serviceKinds.length > 0 && (
<div className="flex flex-wrap gap-1 mb-0.5">
{provider.serviceKinds.map((k) => (
<span
key={k}
className="text-[10px] px-1.5 py-0.5 rounded bg-bg-subtle border border-border text-text-muted leading-none"
>
{KIND_LABEL[k] ?? k}
</span>
))}
</div>
)}
<h3 className="text-sm font-semibold flex items-center gap-1 min-w-0">
<span
className={`truncate min-w-0 flex-1 ${provider.deprecated ? "line-through opacity-60" : ""}`}

View File

@@ -650,6 +650,21 @@ export default function ProvidersPage() {
const oauthOnlyEntriesAll = oauthProviderEntriesAll
.filter((e) => e.toggleAuthType === "oauth")
.filter((e) => !IDE_PROVIDER_IDS.has(e.providerId));
// Web Fetch providers: filter across all entries by serviceKinds
const webFetchEntriesAll = dedupeProviderEntries(
[...staticProviderEntriesAll, ...compatibleProviderEntriesAll].filter((e) => {
const p = e.provider as DashboardProviderInfo & { serviceKinds?: string[] };
return p.serviceKinds?.includes("webFetch") === true;
}) as DashboardProviderEntry[]
);
const webFetchEntries = filterConfiguredProviderEntries(
webFetchEntriesAll,
effectiveShowConfiguredOnly,
searchQuery,
showFreeOnly
);
const summaryStats = {
all: countConfigured(dashboardProviderEntriesAll),
free: countConfigured(freeSectionEntriesAll),
@@ -664,6 +679,7 @@ export default function ProvidersPage() {
upstreamproxy: countConfigured(upstreamProxyEntriesAll),
cloudagent: countConfigured(cloudAgentProviderEntriesAll),
ide: countConfigured(ideProviderEntriesAll),
webfetch: countConfigured(webFetchEntriesAll),
};
if (loading) {
@@ -815,6 +831,13 @@ export default function ProvidersPage() {
label: "Search",
stat: summaryStats.search,
},
{
key: "webfetch",
color: "bg-orange-500",
label: t("webFetch"),
stat: summaryStats.webfetch,
title: t("webFetchTooltip"),
},
{
key: "audio",
color: "bg-rose-500",
@@ -1333,6 +1356,33 @@ export default function ProvidersPage() {
</div>
)}
{/* Web Fetch Providers */}
{showSection("webfetch") && webFetchEntries.length > 0 && (
<div className="flex flex-col gap-4">
<div className="flex flex-wrap items-center gap-2">
<h2 className="text-xl font-semibold flex items-center gap-2 flex-1 min-w-0">
{t("webFetchProvidersHeading")}{" "}
<span className="size-2.5 rounded-full bg-orange-500" title={t("webFetchTooltip")} />
<ProviderCountBadge {...countConfigured(webFetchEntriesAll)} />
</h2>
</div>
<div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5 gap-3">
{webFetchEntries.map(
({ providerId, provider, stats, displayAuthType, toggleAuthType }) => (
<ProviderCard
key={`webfetch-${providerId}`}
providerId={providerId}
provider={provider}
stats={stats}
authType={displayAuthType}
onToggle={(active) => handleToggleProvider(providerId, toggleAuthType, active)}
/>
)
)}
</div>
</div>
)}
{/* Aggregators Gateways */}
{showSection("apikey") && aggregatorProviderEntries.length > 0 && (
<div className="flex flex-col gap-4">

View File

@@ -0,0 +1,139 @@
/**
* POST /v1/web/fetch
*
* Extract content from a URL using a configured web-fetch provider.
* Supports Firecrawl, Jina Reader, and Tavily Extract.
*
* Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? }
* Response: { provider, url, content, links, metadata, screenshot_url }
*/
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts";
import * as log from "@/sse/utils/logger";
import { extractApiKey, isValidApiKey, getProviderCredentials } from "@/sse/services/auth";
import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
import { v1WebFetchSchema } from "@/shared/validation/schemas";
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
const CORS_HEADERS = {
"Access-Control-Allow-Methods": "POST, OPTIONS",
"Access-Control-Allow-Headers": "*",
};
const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const;
type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number];
export async function OPTIONS() {
return new Response(null, { headers: CORS_HEADERS });
}
/**
* Resolve credentials for a web-fetch provider. Tries each known provider in
* priority order when no explicit provider is requested.
*/
async function resolveCredentials(
providerId: WebFetchProviderId
): Promise<{ apiKey?: string } | null> {
try {
const creds = await getProviderCredentials(providerId);
return creds ?? null;
} catch {
return null;
}
}
export async function POST(request: Request) {
let rawBody: unknown;
try {
rawBody = await request.json();
} catch {
log.warn("WEB_FETCH", "Invalid JSON body");
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
}
const validation = validateBody(v1WebFetchSchema, rawBody);
if (isValidationFailure(validation)) {
return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message);
}
const body = validation.data;
// Optional auth check
const apiKeyRaw = extractApiKey(request);
if (process.env.REQUIRE_API_KEY === "true" && !apiKeyRaw) {
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Authentication required");
}
if (apiKeyRaw && !(await isValidApiKey(apiKeyRaw))) {
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
}
// Enforce API key policies
const policy = await enforceApiKeyPolicy(request, "web-fetch");
if (policy.rejection) return policy.rejection;
// Resolve provider + credentials
let resolvedProvider: WebFetchProviderId | undefined;
let credentials: { apiKey?: string } = {};
if (body.provider) {
resolvedProvider = body.provider as WebFetchProviderId;
const creds = await resolveCredentials(resolvedProvider);
if (!creds) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
`No credentials configured for web-fetch provider: ${resolvedProvider}. ` +
`Add an API key for "${resolvedProvider}" in the dashboard.`
);
}
credentials = creds;
} else {
// Auto-select: try providers in priority order
for (const pid of WEB_FETCH_PROVIDERS) {
const creds = await resolveCredentials(pid);
if (creds) {
resolvedProvider = pid;
credentials = creds;
break;
}
}
if (!resolvedProvider) {
return errorResponse(
HTTP_STATUS.BAD_REQUEST,
`No credentials configured for any web-fetch provider. ` +
`Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.`
);
}
}
log.info("WEB_FETCH", `${resolvedProvider} | ${body.url} | format=${body.format}`);
const result = await handleWebFetch(
{
url: body.url,
format: body.format,
depth: body.depth as 0 | 1 | 2,
wait_for_selector: body.wait_for_selector,
include_metadata: body.include_metadata,
},
credentials,
resolvedProvider
);
if (!result.success) {
return new Response(
JSON.stringify({
error: { message: result.error ?? "Web fetch failed", type: "web_fetch_error" },
}),
{
status: result.status ?? 502,
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
}
);
}
return new Response(JSON.stringify(result.data), {
status: 200,
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
});
}

View File

@@ -1507,7 +1507,24 @@
"result": "Result",
"imageDescription": "Generate images from text prompts using OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI and more.",
"videoDescription": "Create videos with AnimateDiff, Stable Video Diffusion via ComfyUI or SD WebUI.",
"musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI."
"musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI.",
"kinds": {
"embedding": "Embedding",
"image": "Image",
"imageToText": "Image to Text",
"tts": "Text to Speech",
"stt": "Speech to Text",
"webSearch": "Web Search",
"webFetch": "Web Fetch",
"video": "Video",
"music": "Music"
},
"noProviders": "No providers configured for this kind yet.",
"addConnection": "Add Connection",
"backToProviders": "Back to Providers",
"connections": "{count} Connections",
"noConnections": "No connections yet — add one from the provider page.",
"loading": "Loading..."
},
"search": {
"searchQuery": "Search Query",
@@ -3917,7 +3934,10 @@
"providerDetailValidClaudeCredentialsFile": "Valid Claude credentials file",
"providerDetailPathAutoDetectedAllOs": "Path is auto-detected per OS (Linux/Mac/Windows).",
"providerDetailMyClaudeAccountPlaceholder": "My Claude account",
"providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac)."
"providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac).",
"webFetch": "Web Fetch",
"webFetchTooltip": "Providers that extract content from web URLs (HTML → Markdown, scrape, screenshot)",
"webFetchProvidersHeading": "Web Fetch Providers"
},
"settings": {
"title": "Settings",

View File

@@ -1505,7 +1505,24 @@
"result": "Resultado",
"imageDescription": "Gere imagens a partir de prompts de texto usando OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI e mais.",
"videoDescription": "Crie vídeos com AnimateDiff, Stable Video Diffusion via ComfyUI ou SD WebUI.",
"musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI."
"musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI.",
"kinds": {
"embedding": "Embedding",
"image": "Imagem",
"imageToText": "Imagem para Texto",
"tts": "Texto para Fala",
"stt": "Fala para Texto",
"webSearch": "Busca Web",
"webFetch": "Web Fetch",
"video": "Vídeo",
"music": "Música"
},
"noProviders": "Nenhum provedor configurado para este tipo.",
"addConnection": "Adicionar Conexão",
"backToProviders": "Voltar para Provedores",
"connections": "{count} Conexões",
"noConnections": "Nenhuma conexão ainda — adicione uma na página do provedor.",
"loading": "Carregando..."
},
"search": {
"searchQuery": "Consulta de Pesquisa",
@@ -3914,7 +3931,10 @@
"providerDetailValidClaudeCredentialsFile": "Arquivo de credenciais do Claude válido",
"providerDetailPathAutoDetectedAllOs": "O caminho é detectado automaticamente por SO (Linux/Mac/Windows).",
"providerDetailMyClaudeAccountPlaceholder": "Minha conta do Claude",
"providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac)."
"providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac).",
"webFetch": "Web Fetch",
"webFetchTooltip": "Provedores que extraem conteúdo de URLs (HTML → Markdown, scraping, screenshot)",
"webFetchProvidersHeading": "Provedores Web Fetch"
},
"settings": {
"title": "Configurações",

View File

@@ -2,6 +2,22 @@
export type RiskNoticeVariant = "oauth" | "webCookie" | "deprecated";
/**
* Service kind — declarative tag for what a provider can do beyond basic LLM chat.
* Affects UI filtering only; does not influence request routing.
*/
export type ServiceKind =
| "llm"
| "embedding"
| "image"
| "imageToText"
| "tts"
| "stt"
| "webSearch"
| "webFetch"
| "video"
| "music";
export interface ProviderRiskNoticeFields {
subscriptionRisk?: boolean;
riskNoticeVariant?: RiskNoticeVariant;
@@ -2118,6 +2134,52 @@ export const APIKEY_PROVIDERS = {
passthroughModels: true,
authHint: "Get API key at monsterapi.ai",
},
// ── Web Fetch Providers ─────────────────────────────────────────────────────
firecrawl: {
id: "firecrawl",
alias: "fc",
name: "Firecrawl",
icon: "language",
color: "#FB923C",
textIcon: "FC",
website: "https://firecrawl.dev",
hasFree: true,
notice: {
text: "Free tier: 500 fetches/month, no credit card needed.",
apiKeyUrl: "https://firecrawl.dev/app/api-keys",
},
serviceKinds: ["webFetch"],
},
"jina-reader": {
id: "jina-reader",
alias: "jr",
name: "Jina Reader",
icon: "menu_book",
color: "#0EA5E9",
textIcon: "JR",
website: "https://jina.ai/reader",
hasFree: true,
notice: {
text: "Free tier: 1M fetches/month.",
apiKeyUrl: "https://jina.ai/api-dashboard",
},
serviceKinds: ["webFetch"],
},
byteplus: {
id: "byteplus",
alias: "bpm",
name: "BytePlus ModelArk",
icon: "cloud",
color: "#2563EB",
textIcon: "BP",
website: "https://console.byteplus.com/ark",
hasFree: true,
notice: {
text: "Free credits for new accounts. Seed 2.0, Kimi K2 Thinking, GLM 4.7, GPT-OSS-120B available.",
apiKeyUrl: "https://console.byteplus.com/ark/region:ark+ap-southeast-1/apiKey",
},
serviceKinds: ["llm"],
},
};
// Sub-categories within APIKEY_PROVIDERS (used by dashboard and catalog views).
@@ -2355,6 +2417,7 @@ export const SEARCH_PROVIDERS = {
website: "https://serper.dev",
hasFree: true,
authHint: "API key from serper.dev dashboard",
serviceKinds: ["webSearch"],
},
"brave-search": {
id: "brave-search",
@@ -2377,6 +2440,7 @@ export const SEARCH_PROVIDERS = {
website: "https://exa.ai",
hasFree: true,
authHint: "API key from dashboard.exa.ai",
serviceKinds: ["webSearch", "webFetch"],
},
"tavily-search": {
id: "tavily-search",
@@ -2388,6 +2452,7 @@ export const SEARCH_PROVIDERS = {
website: "https://tavily.com",
hasFree: true,
authHint: "API key from app.tavily.com (format: tvly-...)",
serviceKinds: ["webSearch", "webFetch"],
},
"google-pse-search": {
id: "google-pse-search",

View File

@@ -10,6 +10,19 @@
import { z } from "zod";
const SERVICE_KIND_VALUES = [
"llm",
"embedding",
"image",
"imageToText",
"tts",
"stt",
"webSearch",
"webFetch",
"video",
"music",
] as const;
export const ProviderSchema = z.object({
id: z.string().min(1),
alias: z.string().min(1).optional(),
@@ -27,6 +40,7 @@ export const ProviderSchema = z.object({
freeNote: z.string().optional(),
authHint: z.string().optional(),
apiHint: z.string().optional(),
serviceKinds: z.array(z.enum(SERVICE_KIND_VALUES)).optional(),
});
export const ProvidersMapSchema = z.record(z.string(), ProviderSchema);

View File

@@ -2147,3 +2147,14 @@ export const v1BatchCreateSchema = z.object({
})
.optional(),
});
// ── Web Fetch ─────────────────────────────────────────────────────────────────
export const v1WebFetchSchema = z.object({
url: z.string().url("url must be a valid URL (http/https)"),
provider: z.enum(["firecrawl", "jina-reader", "tavily-search"]).optional(),
format: z.enum(["markdown", "html", "links", "screenshot"]).default("markdown"),
depth: z.union([z.literal(0), z.literal(1), z.literal(2)]).default(0),
wait_for_selector: z.string().max(256).optional(),
include_metadata: z.boolean().default(false),
});

View File

@@ -0,0 +1,185 @@
import test from "node:test";
import assert from "node:assert/strict";
const { firecrawlFetch } = await import("../../open-sse/executors/firecrawl-fetch.ts");
// ── firecrawlFetch tests ──────────────────────────────────────────────────────
test("firecrawlFetch calls api.firecrawl.dev/v1/scrape with Bearer auth", async () => {
const originalFetch = globalThis.fetch;
let captured: { url: string; headers: Record<string, string>; body: Record<string, unknown> } = {
url: "",
headers: {},
body: {},
};
globalThis.fetch = async (url, init = {}) => {
captured = {
url: String(url),
headers: (init as RequestInit).headers as Record<string, string>,
body: JSON.parse(String((init as RequestInit).body ?? "{}")),
};
return new Response(
JSON.stringify({ data: { markdown: "# Result", links: [], metadata: { title: "Test" } } }),
{ status: 200, headers: { "content-type": "application/json" } }
);
};
try {
const result = await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 0,
includeMetadata: false,
credentials: { apiKey: "fc-test-key" },
});
assert.equal(result.success, true);
assert.equal(captured.url, "https://api.firecrawl.dev/v1/scrape");
assert.equal(captured.headers["Authorization"], "Bearer fc-test-key");
assert.deepEqual(captured.body.formats, ["markdown"]);
assert.equal(captured.body.url, "https://example.com");
} finally {
globalThis.fetch = originalFetch;
}
});
test("firecrawlFetch propagates 401 error without stack trace", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response(JSON.stringify({ error: "Unauthorized" }), {
status: 401,
headers: { "content-type": "application/json" },
});
};
try {
const result = await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 0,
includeMetadata: false,
credentials: { apiKey: "bad-key" },
});
assert.equal(result.success, false);
assert.equal(result.status, 401);
assert.ok(result.error, "should have error");
assert.ok(!result.error.includes("at /"), "error must not contain stack trace");
} finally {
globalThis.fetch = originalFetch;
}
});
test("firecrawlFetch returns 401 error when no API key", async () => {
const result = await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 0,
includeMetadata: false,
credentials: {},
});
assert.equal(result.success, false);
assert.equal(result.status, 401);
assert.ok(!result.error?.includes("at /"));
});
test("firecrawlFetch maps 'html' format correctly", async () => {
const originalFetch = globalThis.fetch;
let capturedFormats: unknown;
globalThis.fetch = async (_url, init = {}) => {
const body = JSON.parse(String((init as RequestInit).body ?? "{}"));
capturedFormats = body.formats;
return new Response(JSON.stringify({ data: { html: "<html>test</html>", links: [] } }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
try {
const result = await firecrawlFetch({
url: "https://example.com",
format: "html",
depth: 0,
includeMetadata: false,
credentials: { apiKey: "fc-key" },
});
assert.equal(result.success, true);
assert.deepEqual(capturedFormats, ["html"]);
assert.ok(result.data?.content.includes("<html>"), "should return html content");
} finally {
globalThis.fetch = originalFetch;
}
});
test("firecrawlFetch returns correct response shape", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response(
JSON.stringify({
data: {
markdown: "# Page Title\nContent here.",
links: ["https://example.com/about", "https://example.com/contact"],
metadata: { title: "Page Title", description: "A test page" },
},
}),
{ status: 200, headers: { "content-type": "application/json" } }
);
};
try {
const result = await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 0,
includeMetadata: true,
credentials: { apiKey: "fc-key" },
});
assert.equal(result.success, true);
assert.ok(result.data, "should have data");
assert.equal(result.data.provider, "firecrawl");
assert.equal(result.data.url, "https://example.com");
assert.ok(result.data.content.includes("Page Title"), "content should contain title");
assert.ok(Array.isArray(result.data.links), "links should be array");
assert.equal(result.data.links.length, 2);
assert.ok(result.data.metadata?.title === "Page Title");
assert.equal(result.data.screenshot_url, null);
} finally {
globalThis.fetch = originalFetch;
}
});
test("firecrawlFetch forwards depth and wait_for_selector", async () => {
const originalFetch = globalThis.fetch;
let capturedBody: Record<string, unknown> = {};
globalThis.fetch = async (_url, init = {}) => {
capturedBody = JSON.parse(String((init as RequestInit).body ?? "{}"));
return new Response(JSON.stringify({ data: { markdown: "" } }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
try {
await firecrawlFetch({
url: "https://example.com",
format: "markdown",
depth: 1,
waitForSelector: "article",
includeMetadata: false,
credentials: { apiKey: "fc-key" },
});
assert.equal(capturedBody.maxDepth, 1, "should set maxDepth");
assert.equal(capturedBody.waitFor, "article", "should set waitFor");
} finally {
globalThis.fetch = originalFetch;
}
});

View File

@@ -0,0 +1,174 @@
import test from "node:test";
import assert from "node:assert/strict";
const { jinaReaderFetch } = await import("../../open-sse/executors/jina-reader-fetch.ts");
// ── jinaReaderFetch tests ─────────────────────────────────────────────────────
test("jinaReaderFetch calls r.jina.ai/{url} with Bearer auth", async () => {
const originalFetch = globalThis.fetch;
let captured: { url: string; headers: Record<string, string> } = { url: "", headers: {} };
globalThis.fetch = async (url, init = {}) => {
captured = {
url: String(url),
headers: (init as RequestInit).headers as Record<string, string>,
};
return new Response("# Hello from Jina", {
status: 200,
headers: { "content-type": "text/plain" },
});
};
try {
const result = await jinaReaderFetch({
url: "https://example.com",
format: "markdown",
includeMetadata: false,
credentials: { apiKey: "jina-test-key" },
});
assert.equal(result.success, true);
assert.ok(
captured.url.startsWith("https://r.jina.ai/"),
`URL should start with https://r.jina.ai/, got: ${captured.url}`
);
assert.ok(
captured.url.includes(encodeURIComponent("https://example.com")),
"URL should include encoded target URL"
);
assert.equal(captured.headers["Authorization"], "Bearer jina-test-key");
} finally {
globalThis.fetch = originalFetch;
}
});
test("jinaReaderFetch returns 401 error when no API key", async () => {
const result = await jinaReaderFetch({
url: "https://example.com",
format: "markdown",
includeMetadata: false,
credentials: {},
});
assert.equal(result.success, false);
assert.equal(result.status, 401);
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace");
});
test("jinaReaderFetch propagates non-200 status without stack trace", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response("Forbidden", {
status: 403,
headers: { "content-type": "text/plain" },
});
};
try {
const result = await jinaReaderFetch({
url: "https://example.com",
format: "markdown",
includeMetadata: false,
credentials: { apiKey: "bad-key" },
});
assert.equal(result.success, false);
assert.equal(result.status, 403);
assert.ok(result.error, "should have error message");
assert.ok(!result.error.includes("at /"), "error must not contain stack trace");
} finally {
globalThis.fetch = originalFetch;
}
});
test("jinaReaderFetch parses JSON response with data.content", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response(
JSON.stringify({
data: {
content: "# Parsed content",
title: "Title from JSON",
description: "A description",
links: ["https://example.com/a", "https://example.com/b"],
},
}),
{ status: 200, headers: { "content-type": "application/json" } }
);
};
try {
const result = await jinaReaderFetch({
url: "https://example.com",
format: "markdown",
includeMetadata: true,
credentials: { apiKey: "jina-key" },
});
assert.equal(result.success, true);
assert.ok(result.data, "should have data");
assert.equal(result.data.provider, "jina-reader");
assert.ok(result.data.content.includes("Parsed content"), "content should be parsed");
assert.ok(result.data.metadata?.title === "Title from JSON", "metadata title should be set");
assert.equal(result.data.screenshot_url, null);
} finally {
globalThis.fetch = originalFetch;
}
});
test("jinaReaderFetch falls back to plain text when response is not JSON", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response("Plain text content from Jina", {
status: 200,
headers: { "content-type": "text/plain; charset=utf-8" },
});
};
try {
const result = await jinaReaderFetch({
url: "https://example.com",
format: "markdown",
includeMetadata: false,
credentials: { apiKey: "jina-key" },
});
assert.equal(result.success, true);
assert.ok(
result.data?.content.includes("Plain text content"),
"should include plain text content"
);
} finally {
globalThis.fetch = originalFetch;
}
});
test("jinaReaderFetch sets X-Return-Format header to html for html format", async () => {
const originalFetch = globalThis.fetch;
let capturedHeaders: Record<string, string> = {};
globalThis.fetch = async (_url, init = {}) => {
capturedHeaders = (init as RequestInit).headers as Record<string, string>;
return new Response("<html>content</html>", {
status: 200,
headers: { "content-type": "text/html" },
});
};
try {
await jinaReaderFetch({
url: "https://example.com",
format: "html",
includeMetadata: false,
credentials: { apiKey: "jina-key" },
});
assert.equal(capturedHeaders["X-Return-Format"], "html");
} finally {
globalThis.fetch = originalFetch;
}
});

View File

@@ -0,0 +1,138 @@
import test from "node:test";
import assert from "node:assert/strict";
const { handleWebFetch } = await import("../../open-sse/handlers/webFetch.ts");
// ── handleWebFetch — basic routing ───────────────────────────────────────────
test("handleWebFetch routes to firecrawl when provider=firecrawl", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response(
JSON.stringify({
data: {
markdown: "# Hello World",
links: ["https://example.com/page"],
metadata: { title: "Hello", description: "A test page" },
},
}),
{ status: 200, headers: { "content-type": "application/json" } }
);
};
try {
const result = await handleWebFetch(
{ url: "https://example.com", format: "markdown" },
{ apiKey: "test-key" },
"firecrawl"
);
assert.equal(result.success, true, "should succeed");
assert.ok(result.data, "should have data");
assert.equal(result.data.provider, "firecrawl");
assert.equal(result.data.url, "https://example.com");
assert.ok(typeof result.data.content === "string", "content should be string");
assert.ok(Array.isArray(result.data.links), "links should be array");
} finally {
globalThis.fetch = originalFetch;
}
});
test("handleWebFetch routes to jina-reader when provider=jina-reader", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
return new Response(
JSON.stringify({
data: {
content: "# Jina content",
title: "Test",
description: "desc",
links: [],
},
}),
{ status: 200, headers: { "content-type": "application/json" } }
);
};
try {
const result = await handleWebFetch(
{ url: "https://example.com", format: "markdown" },
{ apiKey: "jina-key" },
"jina-reader"
);
assert.equal(result.success, true);
assert.equal(result.data?.provider, "jina-reader");
} finally {
globalThis.fetch = originalFetch;
}
});
test("handleWebFetch returns error 401 when no apiKey for firecrawl", async () => {
const result = await handleWebFetch({ url: "https://example.com" }, {}, "firecrawl");
assert.equal(result.success, false);
assert.equal(result.status, 401);
assert.ok(result.error, "should have error message");
// Error must not expose stack traces
assert.ok(!result.error.includes("at /"), "error must not contain stack trace paths");
});
test("handleWebFetch returns error 401 when no apiKey for jina-reader", async () => {
const result = await handleWebFetch({ url: "https://example.com" }, {}, "jina-reader");
assert.equal(result.success, false);
assert.equal(result.status, 401);
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths");
});
test("handleWebFetch wraps fetch errors via buildErrorBody (no raw stack)", async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => {
throw new Error("at /internal/path/executor.ts:42:10\nnetwork failure");
};
try {
const result = await handleWebFetch(
{ url: "https://example.com" },
{ apiKey: "test-key" },
"firecrawl"
);
assert.equal(result.success, false);
assert.ok(result.status != null, "should have status");
// Stack trace must be stripped
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths");
} finally {
globalThis.fetch = originalFetch;
}
});
test("handleWebFetch passes depth and wait_for_selector to firecrawl", async () => {
const originalFetch = globalThis.fetch;
let captured: { body: Record<string, unknown> } = { body: {} };
globalThis.fetch = async (_url, init) => {
captured.body = JSON.parse(String((init as RequestInit).body ?? "{}"));
return new Response(JSON.stringify({ data: { markdown: "" } }), {
status: 200,
headers: { "content-type": "application/json" },
});
};
try {
await handleWebFetch(
{ url: "https://example.com", depth: 2, wait_for_selector: "main" },
{ apiKey: "test-key" },
"firecrawl"
);
assert.equal(captured.body.maxDepth, 2, "should forward depth");
assert.equal(captured.body.waitFor, "main", "should forward wait_for_selector");
} finally {
globalThis.fetch = originalFetch;
}
});