mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-04 22:32:12 +03:00
feat(dashboard): media providers pages + Web Fetch category (#2645)
Integrated into release/v3.8.3
This commit is contained in:
committed by
GitHub
parent
40cc555b68
commit
6fa7de0ba7
143
open-sse/executors/firecrawl-fetch.ts
Normal file
143
open-sse/executors/firecrawl-fetch.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* Firecrawl Web Fetch Executor
|
||||
*
|
||||
* Fetches content from a URL using the Firecrawl scrape API.
|
||||
* POST https://api.firecrawl.dev/v1/scrape
|
||||
*
|
||||
* Free tier: 500 fetches/month, no credit card required.
|
||||
* Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape
|
||||
*/
|
||||
|
||||
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
|
||||
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
|
||||
|
||||
const FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v1";
|
||||
const FIRECRAWL_TIMEOUT_MS = 30_000;
|
||||
|
||||
function mapFormat(format: WebFetchFormat): string {
|
||||
switch (format) {
|
||||
case "html":
|
||||
return "html";
|
||||
case "links":
|
||||
return "links";
|
||||
case "screenshot":
|
||||
return "screenshot";
|
||||
case "markdown":
|
||||
default:
|
||||
return "markdown";
|
||||
}
|
||||
}
|
||||
|
||||
interface FirecrawlScrapeOptions {
|
||||
url: string;
|
||||
format: WebFetchFormat;
|
||||
depth: number;
|
||||
waitForSelector?: string;
|
||||
includeMetadata: boolean;
|
||||
credentials: WebFetchCredentials;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a Firecrawl scrape request.
|
||||
*/
|
||||
export async function firecrawlFetch(opts: FirecrawlScrapeOptions): Promise<WebFetchResult> {
|
||||
const { url, format, depth, waitForSelector, includeMetadata, credentials } = opts;
|
||||
|
||||
if (!credentials.apiKey) {
|
||||
const body = buildErrorBody(401, "Firecrawl API key required");
|
||||
return { success: false, status: 401, error: body.error.message };
|
||||
}
|
||||
|
||||
const formats = [mapFormat(format)];
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
url,
|
||||
formats,
|
||||
};
|
||||
|
||||
if (includeMetadata) {
|
||||
requestBody.includeTags = ["title", "description", "og:title", "og:description"];
|
||||
}
|
||||
|
||||
if (depth > 0) {
|
||||
requestBody.maxDepth = depth;
|
||||
}
|
||||
|
||||
if (waitForSelector) {
|
||||
requestBody.waitFor = waitForSelector;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), FIRECRAWL_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${FIRECRAWL_API_BASE}/scrape`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${credentials.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
|
||||
const msg = sanitizeErrorMessage(`Firecrawl error ${response.status}: ${rawError}`);
|
||||
const body = buildErrorBody(response.status, msg);
|
||||
return { success: false, status: response.status, error: body.error.message };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as Record<string, unknown>;
|
||||
|
||||
const scraped = (data.data as Record<string, unknown> | null) ?? {};
|
||||
|
||||
const content =
|
||||
format === "html"
|
||||
? String(scraped.html ?? "")
|
||||
: format === "links"
|
||||
? JSON.stringify(scraped.links ?? [])
|
||||
: String(scraped.markdown ?? scraped.content ?? "");
|
||||
|
||||
const rawLinks = scraped.links;
|
||||
const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
|
||||
|
||||
const rawMeta = scraped.metadata as Record<string, unknown> | null | undefined;
|
||||
const metadata = includeMetadata
|
||||
? {
|
||||
title: rawMeta?.title != null ? String(rawMeta.title) : null,
|
||||
description: rawMeta?.description != null ? String(rawMeta.description) : null,
|
||||
}
|
||||
: null;
|
||||
|
||||
const screenshotUrl =
|
||||
format === "screenshot"
|
||||
? scraped.screenshot != null
|
||||
? String(scraped.screenshot)
|
||||
: null
|
||||
: null;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
provider: "firecrawl",
|
||||
url,
|
||||
content,
|
||||
links,
|
||||
metadata,
|
||||
screenshot_url: screenshotUrl,
|
||||
},
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
const body = buildErrorBody(504, "Firecrawl request timed out");
|
||||
return { success: false, status: 504, error: body.error.message };
|
||||
}
|
||||
const msg =
|
||||
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
|
||||
const body = buildErrorBody(502, msg);
|
||||
return { success: false, status: 502, error: body.error.message };
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
119
open-sse/executors/jina-reader-fetch.ts
Normal file
119
open-sse/executors/jina-reader-fetch.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Jina Reader Web Fetch Executor
|
||||
*
|
||||
* Fetches content from a URL using the Jina Reader API.
|
||||
* GET https://r.jina.ai/{url}
|
||||
*
|
||||
* Free tier: 1M fetches/month.
|
||||
* Docs: https://jina.ai/reader/
|
||||
*/
|
||||
|
||||
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
|
||||
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
|
||||
|
||||
const JINA_READER_BASE = "https://r.jina.ai";
|
||||
const JINA_TIMEOUT_MS = 30_000;
|
||||
|
||||
interface JinaReaderFetchOptions {
|
||||
url: string;
|
||||
format: WebFetchFormat;
|
||||
includeMetadata: boolean;
|
||||
credentials: WebFetchCredentials;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a Jina Reader fetch request.
|
||||
* Jina Reader uses a URL-based approach: GET https://r.jina.ai/<url>
|
||||
*/
|
||||
export async function jinaReaderFetch(opts: JinaReaderFetchOptions): Promise<WebFetchResult> {
|
||||
const { url, format, includeMetadata, credentials } = opts;
|
||||
|
||||
if (!credentials.apiKey) {
|
||||
const body = buildErrorBody(401, "Jina Reader API key required");
|
||||
return { success: false, status: 401, error: body.error.message };
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Authorization: `Bearer ${credentials.apiKey}`,
|
||||
Accept: "application/json",
|
||||
"X-Return-Format": format === "html" ? "html" : "markdown",
|
||||
};
|
||||
|
||||
if (includeMetadata) {
|
||||
headers["X-With-Generated-Alt"] = "true";
|
||||
}
|
||||
|
||||
if (format === "links") {
|
||||
headers["X-Gather-All-Links-At-The-End"] = "true";
|
||||
}
|
||||
|
||||
const encodedUrl = encodeURIComponent(url);
|
||||
const requestUrl = `${JINA_READER_BASE}/${encodedUrl}`;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), JINA_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(requestUrl, {
|
||||
method: "GET",
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
|
||||
const msg = sanitizeErrorMessage(`Jina Reader error ${response.status}: ${rawError}`);
|
||||
const body = buildErrorBody(response.status, msg);
|
||||
return { success: false, status: response.status, error: body.error.message };
|
||||
}
|
||||
|
||||
// Jina Reader returns JSON with data.content or plain text
|
||||
const contentType = response.headers.get("content-type") ?? "";
|
||||
let content = "";
|
||||
let links: string[] = [];
|
||||
let metadata: { title: string | null; description: string | null } | null = null;
|
||||
|
||||
if (contentType.includes("application/json")) {
|
||||
const json = (await response.json()) as Record<string, unknown>;
|
||||
const responseData = (json.data as Record<string, unknown> | null) ?? {};
|
||||
content = String(responseData.content ?? responseData.text ?? "");
|
||||
|
||||
if (includeMetadata) {
|
||||
metadata = {
|
||||
title: responseData.title != null ? String(responseData.title) : null,
|
||||
description: responseData.description != null ? String(responseData.description) : null,
|
||||
};
|
||||
}
|
||||
|
||||
if (format === "links") {
|
||||
const rawLinks = responseData.links;
|
||||
links = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
|
||||
}
|
||||
} else {
|
||||
content = await response.text();
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
provider: "jina-reader",
|
||||
url,
|
||||
content,
|
||||
links,
|
||||
metadata,
|
||||
screenshot_url: null,
|
||||
},
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
const body = buildErrorBody(504, "Jina Reader request timed out");
|
||||
return { success: false, status: 504, error: body.error.message };
|
||||
}
|
||||
const msg =
|
||||
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
|
||||
const body = buildErrorBody(502, msg);
|
||||
return { success: false, status: 502, error: body.error.message };
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
103
open-sse/executors/tavily-fetch.ts
Normal file
103
open-sse/executors/tavily-fetch.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Tavily Web Fetch Executor
|
||||
*
|
||||
* Fetches content from a URL using the Tavily Extract API.
|
||||
* POST https://api.tavily.com/extract
|
||||
*
|
||||
* Free tier: included in Tavily plan.
|
||||
* Docs: https://docs.tavily.com/documentation/api-reference/endpoint/post-extract
|
||||
*/
|
||||
|
||||
import { sanitizeErrorMessage, buildErrorBody } from "../utils/error.ts";
|
||||
import type { WebFetchResult, WebFetchFormat, WebFetchCredentials } from "../handlers/webFetch.ts";
|
||||
|
||||
const TAVILY_EXTRACT_URL = "https://api.tavily.com/extract";
|
||||
const TAVILY_TIMEOUT_MS = 30_000;
|
||||
|
||||
interface TavilyFetchOptions {
|
||||
url: string;
|
||||
format: WebFetchFormat;
|
||||
includeMetadata: boolean;
|
||||
credentials: WebFetchCredentials;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a Tavily extract request.
|
||||
* Tavily Extract returns the raw content of a given URL.
|
||||
*/
|
||||
export async function tavilyFetch(opts: TavilyFetchOptions): Promise<WebFetchResult> {
|
||||
const { url, includeMetadata, credentials } = opts;
|
||||
|
||||
if (!credentials.apiKey) {
|
||||
const body = buildErrorBody(401, "Tavily API key required");
|
||||
return { success: false, status: 401, error: body.error.message };
|
||||
}
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
api_key: credentials.apiKey,
|
||||
urls: [url],
|
||||
extract_depth: "basic",
|
||||
};
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), TAVILY_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(TAVILY_EXTRACT_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${credentials.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const rawError = await response.text().catch(() => `HTTP ${response.status}`);
|
||||
const msg = sanitizeErrorMessage(`Tavily error ${response.status}: ${rawError}`);
|
||||
const body = buildErrorBody(response.status, msg);
|
||||
return { success: false, status: response.status, error: body.error.message };
|
||||
}
|
||||
|
||||
const data = (await response.json()) as Record<string, unknown>;
|
||||
|
||||
const results = data.results as Array<Record<string, unknown>> | null;
|
||||
const firstResult = results?.[0] ?? {};
|
||||
|
||||
const content = String(firstResult.raw_content ?? firstResult.content ?? "");
|
||||
|
||||
const rawLinks = firstResult.links;
|
||||
const links: string[] = Array.isArray(rawLinks) ? rawLinks.map((l) => String(l)) : [];
|
||||
|
||||
const metadata = includeMetadata
|
||||
? {
|
||||
title: firstResult.title != null ? String(firstResult.title) : null,
|
||||
description: null,
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
data: {
|
||||
provider: "tavily-search",
|
||||
url,
|
||||
content,
|
||||
links,
|
||||
metadata,
|
||||
screenshot_url: null,
|
||||
},
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof Error && err.name === "AbortError") {
|
||||
const body = buildErrorBody(504, "Tavily request timed out");
|
||||
return { success: false, status: 504, error: body.error.message };
|
||||
}
|
||||
const msg =
|
||||
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
|
||||
const body = buildErrorBody(502, msg);
|
||||
return { success: false, status: 502, error: body.error.message };
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
121
open-sse/handlers/webFetch.ts
Normal file
121
open-sse/handlers/webFetch.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Web Fetch Handler
|
||||
*
|
||||
* Handles POST /v1/web/fetch requests.
|
||||
* Dispatches to a web-fetch provider executor (Firecrawl, Jina Reader, or Tavily).
|
||||
*
|
||||
* Request format:
|
||||
* {
|
||||
* "url": "https://example.com",
|
||||
* "provider": "firecrawl" | "jina-reader" | "tavily-search", // optional
|
||||
* "format": "markdown" | "html" | "links" | "screenshot",
|
||||
* "depth": 0 | 1 | 2,
|
||||
* "wait_for_selector": "main",
|
||||
* "include_metadata": true
|
||||
* }
|
||||
*/
|
||||
|
||||
import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts";
|
||||
import { firecrawlFetch } from "../executors/firecrawl-fetch.ts";
|
||||
import { jinaReaderFetch } from "../executors/jina-reader-fetch.ts";
|
||||
import { tavilyFetch } from "../executors/tavily-fetch.ts";
|
||||
|
||||
export type WebFetchFormat = "markdown" | "html" | "links" | "screenshot";
|
||||
|
||||
export interface WebFetchRequest {
|
||||
url: string;
|
||||
provider?: "firecrawl" | "jina-reader" | "tavily-search";
|
||||
format?: WebFetchFormat;
|
||||
depth?: 0 | 1 | 2;
|
||||
wait_for_selector?: string;
|
||||
include_metadata?: boolean;
|
||||
}
|
||||
|
||||
export interface WebFetchResponse {
|
||||
provider: string;
|
||||
url: string;
|
||||
content: string;
|
||||
links: string[];
|
||||
metadata: { title: string | null; description: string | null } | null;
|
||||
screenshot_url: string | null;
|
||||
}
|
||||
|
||||
export interface WebFetchResult {
|
||||
success: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
data?: WebFetchResponse;
|
||||
}
|
||||
|
||||
export interface WebFetchCredentials {
|
||||
apiKey?: string;
|
||||
}
|
||||
|
||||
const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const;
|
||||
type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number];
|
||||
|
||||
/**
|
||||
* Execute a web fetch request against the specified (or auto-selected) provider.
|
||||
*
|
||||
* @param req - Validated web fetch request body
|
||||
* @param credentials - Provider API credentials (apiKey)
|
||||
* @param resolvedProvider - Provider ID to use; if omitted auto-selects based on available creds
|
||||
*/
|
||||
export async function handleWebFetch(
|
||||
req: WebFetchRequest,
|
||||
credentials: WebFetchCredentials,
|
||||
resolvedProvider?: WebFetchProviderId
|
||||
): Promise<WebFetchResult> {
|
||||
const provider = resolvedProvider ?? req.provider ?? "firecrawl";
|
||||
|
||||
const format: WebFetchFormat = req.format ?? "markdown";
|
||||
const includeMetadata = req.include_metadata ?? false;
|
||||
|
||||
try {
|
||||
switch (provider) {
|
||||
case "firecrawl":
|
||||
return await firecrawlFetch({
|
||||
url: req.url,
|
||||
format,
|
||||
depth: req.depth ?? 0,
|
||||
waitForSelector: req.wait_for_selector,
|
||||
includeMetadata,
|
||||
credentials,
|
||||
});
|
||||
|
||||
case "jina-reader":
|
||||
return await jinaReaderFetch({
|
||||
url: req.url,
|
||||
format,
|
||||
includeMetadata,
|
||||
credentials,
|
||||
});
|
||||
|
||||
case "tavily-search":
|
||||
return await tavilyFetch({
|
||||
url: req.url,
|
||||
format,
|
||||
includeMetadata,
|
||||
credentials,
|
||||
});
|
||||
|
||||
default: {
|
||||
const _exhaustive: never = provider;
|
||||
return {
|
||||
success: false,
|
||||
status: 400,
|
||||
error: `Unknown web fetch provider: ${_exhaustive}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const msg =
|
||||
err instanceof Error ? sanitizeErrorMessage(err.message) : sanitizeErrorMessage(String(err));
|
||||
const body = buildErrorBody(502, msg);
|
||||
return {
|
||||
success: false,
|
||||
status: 502,
|
||||
error: body.error.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
import { isDeepSeekReasoningModel, requiresReasoningReplay } from "../../services/reasoningCache.ts";
|
||||
import {
|
||||
isDeepSeekReasoningModel,
|
||||
requiresReasoningReplay,
|
||||
} from "../../services/reasoningCache.ts";
|
||||
|
||||
/**
|
||||
* Shared sanitizers for tool payloads that arrive from IDEs/SDKs with
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import { Button } from "@/shared/components";
|
||||
import MediaProviderHeader from "../../components/MediaProviderHeader";
|
||||
import MediaProviderKindNav from "../../components/MediaProviderKindNav";
|
||||
import type { MediaKind } from "../../components/MediaProviderKindNav";
|
||||
|
||||
interface Connection {
|
||||
id: string;
|
||||
name?: string | null;
|
||||
testStatus?: string | null;
|
||||
isActive?: boolean;
|
||||
lastErrorAt?: string | null;
|
||||
lastError?: string | null;
|
||||
}
|
||||
|
||||
interface MediaProviderPageClientProps {
|
||||
providerId: string;
|
||||
providerName: string;
|
||||
providerColor?: string;
|
||||
kindLabel: string;
|
||||
activeKind: MediaKind;
|
||||
website?: string;
|
||||
hasFree?: boolean;
|
||||
freeNote?: string;
|
||||
}
|
||||
|
||||
export default function MediaProviderPageClient({
|
||||
providerId,
|
||||
providerName,
|
||||
providerColor,
|
||||
kindLabel,
|
||||
activeKind,
|
||||
website,
|
||||
hasFree,
|
||||
freeNote,
|
||||
}: MediaProviderPageClientProps) {
|
||||
const t = useTranslations("media");
|
||||
const [connections, setConnections] = useState<Connection[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchConnections = async () => {
|
||||
try {
|
||||
const res = await fetch(`/api/providers?provider=${encodeURIComponent(providerId)}`);
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
setConnections(
|
||||
(data.connections ?? []).filter(
|
||||
(c: Connection & { provider?: string }) => c.provider === providerId
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
void fetchConnections();
|
||||
}, [providerId]);
|
||||
|
||||
const backHref = `/dashboard/media-providers/${activeKind}`;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-6">
|
||||
<MediaProviderHeader
|
||||
providerId={providerId}
|
||||
providerName={providerName}
|
||||
providerColor={providerColor}
|
||||
kindLabel={kindLabel}
|
||||
website={website}
|
||||
hasFree={hasFree}
|
||||
freeNote={freeNote}
|
||||
backHref={backHref}
|
||||
/>
|
||||
|
||||
<MediaProviderKindNav activeKind={activeKind} />
|
||||
|
||||
{/* Connections */}
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-base font-semibold">
|
||||
{t("connections", { count: connections.length })}
|
||||
</h2>
|
||||
<Button
|
||||
size="sm"
|
||||
icon="add"
|
||||
onClick={() => {
|
||||
window.location.href = `/dashboard/providers/${providerId}`;
|
||||
}}
|
||||
>
|
||||
{t("addConnection")}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{loading ? (
|
||||
<div className="py-8 text-center text-sm text-text-muted">{t("loading")}</div>
|
||||
) : connections.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center gap-2 py-12 border border-dashed border-border rounded-xl text-text-muted text-sm">
|
||||
<span className="material-symbols-outlined text-[28px]">key_off</span>
|
||||
<span>{t("noConnections")}</span>
|
||||
<Button
|
||||
size="sm"
|
||||
icon="add"
|
||||
onClick={() => {
|
||||
window.location.href = `/dashboard/providers/${providerId}`;
|
||||
}}
|
||||
>
|
||||
{t("addConnection")}
|
||||
</Button>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col divide-y divide-border rounded-xl border border-border overflow-hidden">
|
||||
{connections.map((conn) => (
|
||||
<div key={conn.id} className="flex items-center gap-3 px-4 py-3 bg-bg-card">
|
||||
<span
|
||||
className={`size-2 rounded-full shrink-0 ${
|
||||
conn.testStatus === "active" || conn.testStatus === "success"
|
||||
? "bg-green-500"
|
||||
: conn.testStatus === "error" || conn.testStatus === "unavailable"
|
||||
? "bg-red-500"
|
||||
: "bg-text-muted"
|
||||
}`}
|
||||
/>
|
||||
<span className="text-sm font-medium flex-1 truncate">{conn.name ?? conn.id}</span>
|
||||
{conn.isActive === false && (
|
||||
<span className="text-[10px] px-1.5 py-0.5 rounded bg-bg-subtle border border-border text-text-muted">
|
||||
disabled
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Playground placeholder — Fase 4 */}
|
||||
<div className="flex flex-col gap-2 border border-dashed border-border rounded-xl p-6">
|
||||
<div className="flex items-center gap-2 text-text-muted">
|
||||
<span className="material-symbols-outlined text-[20px]">labs</span>
|
||||
<h3 className="text-sm font-medium">Example (Playground inline — Fase 4)</h3>
|
||||
</div>
|
||||
<p className="text-xs text-text-muted">
|
||||
Inline playground for this provider will be available in the next release.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { notFound } from "next/navigation";
|
||||
import { getTranslations } from "next-intl/server";
|
||||
import { AI_PROVIDERS } from "@/shared/constants/providers";
|
||||
import type { MediaKind } from "../../components/MediaProviderKindNav";
|
||||
import { MEDIA_KINDS } from "../../components/MediaProviderKindNav";
|
||||
import MediaProviderPageClient from "./MediaProviderPageClient";
|
||||
|
||||
interface PageProps {
|
||||
params: Promise<{ kind: string; id: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* /dashboard/media-providers/[kind]/[id]
|
||||
*
|
||||
* Individual provider page for a media-service provider.
|
||||
* Validates both kind and id; 404 if either is unknown or the provider
|
||||
* does not declare the requested kind.
|
||||
*/
|
||||
export default async function MediaProviderDetailPage({ params }: PageProps) {
|
||||
const { kind, id } = await params;
|
||||
|
||||
// Validate kind
|
||||
if (!MEDIA_KINDS.includes(kind as MediaKind)) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const validKind = kind as MediaKind;
|
||||
|
||||
// Validate provider exists in AI_PROVIDERS
|
||||
const provider = Object.values(AI_PROVIDERS).find((p) => p.id === id) as
|
||||
| (Record<string, unknown> & {
|
||||
id: string;
|
||||
name: string;
|
||||
color?: string;
|
||||
website?: string;
|
||||
hasFree?: boolean;
|
||||
freeNote?: string;
|
||||
})
|
||||
| undefined;
|
||||
|
||||
if (!provider) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
// Validate that the provider declares this kind
|
||||
const serviceKinds = provider.serviceKinds as string[] | undefined;
|
||||
if (!Array.isArray(serviceKinds) || !serviceKinds.includes(validKind)) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const t = await getTranslations("media");
|
||||
const kindLabel = t(`kinds.${validKind}`);
|
||||
|
||||
return (
|
||||
<MediaProviderPageClient
|
||||
providerId={provider.id}
|
||||
providerName={provider.name}
|
||||
providerColor={provider.color}
|
||||
kindLabel={kindLabel}
|
||||
activeKind={validKind}
|
||||
website={provider.website}
|
||||
hasFree={provider.hasFree}
|
||||
freeNote={provider.freeNote}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { notFound } from "next/navigation";
|
||||
import { getTranslations } from "next-intl/server";
|
||||
import Link from "next/link";
|
||||
import { AI_PROVIDERS } from "@/shared/constants/providers";
|
||||
import type { MediaKind } from "../components/MediaProviderKindNav";
|
||||
import MediaProviderKindNav, { MEDIA_KINDS } from "../components/MediaProviderKindNav";
|
||||
import ProviderIcon from "@/shared/components/ProviderIcon";
|
||||
|
||||
interface PageProps {
|
||||
params: Promise<{ kind: string }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* /dashboard/media-providers/[kind]
|
||||
*
|
||||
* Lists all AI providers that declare the given serviceKind.
|
||||
* Returns 404 for unknown kinds.
|
||||
*/
|
||||
export default async function MediaProviderKindPage({ params }: PageProps) {
|
||||
const { kind } = await params;
|
||||
|
||||
// Validate kind
|
||||
if (!MEDIA_KINDS.includes(kind as MediaKind)) {
|
||||
notFound();
|
||||
}
|
||||
|
||||
const validKind = kind as MediaKind;
|
||||
const t = await getTranslations("media");
|
||||
|
||||
// Filter providers that support this kind
|
||||
const matchingProviders = Object.values(AI_PROVIDERS).filter((p) => {
|
||||
const serviceKinds = (p as Record<string, unknown>).serviceKinds as string[] | undefined;
|
||||
return Array.isArray(serviceKinds) && serviceKinds.includes(validKind);
|
||||
});
|
||||
|
||||
const kindLabel = t(`kinds.${validKind}`);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-6">
|
||||
<div className="flex flex-col gap-2">
|
||||
<h1 className="text-2xl font-semibold">{kindLabel}</h1>
|
||||
<p className="text-sm text-text-muted">{t("noProviders")}</p>
|
||||
</div>
|
||||
|
||||
<MediaProviderKindNav activeKind={validKind} />
|
||||
|
||||
{matchingProviders.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center py-16 gap-3 border border-dashed border-border rounded-xl text-text-muted">
|
||||
<span className="material-symbols-outlined text-[32px]">category</span>
|
||||
<p className="text-sm">{t("noProviders")}</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5 gap-3">
|
||||
{matchingProviders.map((provider) => {
|
||||
const p = provider as Record<string, unknown> & {
|
||||
id: string;
|
||||
name: string;
|
||||
color?: string;
|
||||
hasFree?: boolean;
|
||||
freeNote?: string;
|
||||
};
|
||||
return (
|
||||
<Link
|
||||
key={p.id}
|
||||
href={`/dashboard/media-providers/${validKind}/${p.id}`}
|
||||
className="group"
|
||||
>
|
||||
<div className="rounded-xl border border-border bg-bg-card p-3 hover:bg-black/[0.01] dark:hover:bg-white/[0.01] transition-colors cursor-pointer h-full flex flex-col gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<div
|
||||
className="size-7 rounded-lg flex items-center justify-center shrink-0"
|
||||
style={{ backgroundColor: `${p.color ?? "#64748b"}15` }}
|
||||
>
|
||||
<ProviderIcon providerId={p.id} size={20} type="color" />
|
||||
</div>
|
||||
<span className="text-sm font-medium truncate">{p.name}</span>
|
||||
</div>
|
||||
{p.hasFree && (
|
||||
<span className="text-[10px] px-1.5 py-0.5 rounded-full bg-green-500/10 border border-green-500/20 text-green-600 dark:text-green-400 w-fit">
|
||||
Free
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</Link>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
import { useTranslations } from "next-intl";
|
||||
import ProviderIcon from "@/shared/components/ProviderIcon";
|
||||
|
||||
interface MediaProviderHeaderProps {
|
||||
providerId: string;
|
||||
providerName: string;
|
||||
providerColor?: string;
|
||||
kindLabel: string;
|
||||
website?: string;
|
||||
hasFree?: boolean;
|
||||
freeNote?: string;
|
||||
backHref: string;
|
||||
}
|
||||
|
||||
export default function MediaProviderHeader({
|
||||
providerId,
|
||||
providerName,
|
||||
providerColor,
|
||||
kindLabel,
|
||||
website,
|
||||
hasFree,
|
||||
freeNote,
|
||||
backHref,
|
||||
}: MediaProviderHeaderProps) {
|
||||
const t = useTranslations("media");
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-3">
|
||||
<Link
|
||||
href={backHref}
|
||||
className="inline-flex items-center gap-1.5 text-sm text-text-muted hover:text-text-primary transition-colors w-fit"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[16px]">arrow_back</span>
|
||||
{t("backToProviders")}
|
||||
</Link>
|
||||
|
||||
<div className="flex items-start gap-4">
|
||||
<div
|
||||
className="size-12 rounded-xl flex items-center justify-center shrink-0"
|
||||
style={{ backgroundColor: `${providerColor ?? "#64748b"}20` }}
|
||||
>
|
||||
<ProviderIcon providerId={providerId} size={32} type="color" />
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<h1 className="text-xl font-semibold">{providerName}</h1>
|
||||
<span className="text-xs px-2 py-0.5 rounded-full bg-bg-subtle border border-border text-text-muted">
|
||||
{kindLabel}
|
||||
</span>
|
||||
{hasFree && (
|
||||
<span className="text-xs px-2 py-0.5 rounded-full bg-green-500/10 border border-green-500/20 text-green-600 dark:text-green-400">
|
||||
Free
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{freeNote && <p className="text-sm text-text-muted mt-1">{freeNote}</p>}
|
||||
{website && (
|
||||
<a
|
||||
href={website}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 text-xs text-primary hover:underline mt-1"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[13px]">open_in_new</span>
|
||||
{website}
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
export type MediaKind =
|
||||
| "embedding"
|
||||
| "image"
|
||||
| "imageToText"
|
||||
| "tts"
|
||||
| "stt"
|
||||
| "webSearch"
|
||||
| "webFetch"
|
||||
| "video"
|
||||
| "music";
|
||||
|
||||
export const MEDIA_KINDS: MediaKind[] = [
|
||||
"embedding",
|
||||
"image",
|
||||
"imageToText",
|
||||
"tts",
|
||||
"stt",
|
||||
"webSearch",
|
||||
"webFetch",
|
||||
"video",
|
||||
"music",
|
||||
];
|
||||
|
||||
interface MediaProviderKindNavProps {
|
||||
activeKind: MediaKind;
|
||||
}
|
||||
|
||||
export default function MediaProviderKindNav({ activeKind }: MediaProviderKindNavProps) {
|
||||
const t = useTranslations("media");
|
||||
|
||||
return (
|
||||
<div className="flex flex-wrap gap-2 border-b border-border pb-3">
|
||||
{MEDIA_KINDS.map((kind) => {
|
||||
const isActive = kind === activeKind;
|
||||
const label = t(`kinds.${kind}`);
|
||||
return (
|
||||
<Link
|
||||
key={kind}
|
||||
href={`/dashboard/media-providers/${kind}`}
|
||||
className={`flex items-center px-3 py-1.5 rounded-full border text-xs font-medium transition-colors ${
|
||||
isActive
|
||||
? "bg-primary text-white border-primary"
|
||||
: "bg-bg-subtle border-border text-text-muted hover:text-text-primary hover:border-primary/30"
|
||||
}`}
|
||||
aria-current={isActive ? "page" : undefined}
|
||||
>
|
||||
{label}
|
||||
</Link>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
9
src/app/(dashboard)/dashboard/media-providers/page.tsx
Normal file
9
src/app/(dashboard)/dashboard/media-providers/page.tsx
Normal file
@@ -0,0 +1,9 @@
|
||||
import { redirect } from "next/navigation";
|
||||
|
||||
/**
|
||||
* /dashboard/media-providers
|
||||
* Default redirect to the embedding kind list.
|
||||
*/
|
||||
export default function MediaProvidersPage() {
|
||||
redirect("/dashboard/media-providers/embedding");
|
||||
}
|
||||
@@ -27,6 +27,19 @@ interface ProviderStats {
|
||||
codexFastActive?: boolean;
|
||||
}
|
||||
|
||||
const KIND_LABEL: Record<string, string> = {
|
||||
llm: "Chat",
|
||||
embedding: "Embed",
|
||||
image: "Image",
|
||||
imageToText: "I→T",
|
||||
tts: "TTS",
|
||||
stt: "STT",
|
||||
webSearch: "Search",
|
||||
webFetch: "Fetch",
|
||||
video: "Video",
|
||||
music: "Music",
|
||||
};
|
||||
|
||||
interface ProviderCardProps {
|
||||
providerId: string;
|
||||
provider: {
|
||||
@@ -39,6 +52,7 @@ interface ProviderCardProps {
|
||||
hasFree?: boolean;
|
||||
freeNote?: string;
|
||||
subscriptionRisk?: boolean;
|
||||
serviceKinds?: string[];
|
||||
};
|
||||
stats: ProviderStats;
|
||||
authType?: string;
|
||||
@@ -182,6 +196,18 @@ export default function ProviderCard({
|
||||
)}
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
{provider.serviceKinds && provider.serviceKinds.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1 mb-0.5">
|
||||
{provider.serviceKinds.map((k) => (
|
||||
<span
|
||||
key={k}
|
||||
className="text-[10px] px-1.5 py-0.5 rounded bg-bg-subtle border border-border text-text-muted leading-none"
|
||||
>
|
||||
{KIND_LABEL[k] ?? k}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<h3 className="text-sm font-semibold flex items-center gap-1 min-w-0">
|
||||
<span
|
||||
className={`truncate min-w-0 flex-1 ${provider.deprecated ? "line-through opacity-60" : ""}`}
|
||||
|
||||
@@ -650,6 +650,21 @@ export default function ProvidersPage() {
|
||||
const oauthOnlyEntriesAll = oauthProviderEntriesAll
|
||||
.filter((e) => e.toggleAuthType === "oauth")
|
||||
.filter((e) => !IDE_PROVIDER_IDS.has(e.providerId));
|
||||
|
||||
// Web Fetch providers: filter across all entries by serviceKinds
|
||||
const webFetchEntriesAll = dedupeProviderEntries(
|
||||
[...staticProviderEntriesAll, ...compatibleProviderEntriesAll].filter((e) => {
|
||||
const p = e.provider as DashboardProviderInfo & { serviceKinds?: string[] };
|
||||
return p.serviceKinds?.includes("webFetch") === true;
|
||||
}) as DashboardProviderEntry[]
|
||||
);
|
||||
const webFetchEntries = filterConfiguredProviderEntries(
|
||||
webFetchEntriesAll,
|
||||
effectiveShowConfiguredOnly,
|
||||
searchQuery,
|
||||
showFreeOnly
|
||||
);
|
||||
|
||||
const summaryStats = {
|
||||
all: countConfigured(dashboardProviderEntriesAll),
|
||||
free: countConfigured(freeSectionEntriesAll),
|
||||
@@ -664,6 +679,7 @@ export default function ProvidersPage() {
|
||||
upstreamproxy: countConfigured(upstreamProxyEntriesAll),
|
||||
cloudagent: countConfigured(cloudAgentProviderEntriesAll),
|
||||
ide: countConfigured(ideProviderEntriesAll),
|
||||
webfetch: countConfigured(webFetchEntriesAll),
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
@@ -815,6 +831,13 @@ export default function ProvidersPage() {
|
||||
label: "Search",
|
||||
stat: summaryStats.search,
|
||||
},
|
||||
{
|
||||
key: "webfetch",
|
||||
color: "bg-orange-500",
|
||||
label: t("webFetch"),
|
||||
stat: summaryStats.webfetch,
|
||||
title: t("webFetchTooltip"),
|
||||
},
|
||||
{
|
||||
key: "audio",
|
||||
color: "bg-rose-500",
|
||||
@@ -1333,6 +1356,33 @@ export default function ProvidersPage() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Web Fetch Providers */}
|
||||
{showSection("webfetch") && webFetchEntries.length > 0 && (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<h2 className="text-xl font-semibold flex items-center gap-2 flex-1 min-w-0">
|
||||
{t("webFetchProvidersHeading")}{" "}
|
||||
<span className="size-2.5 rounded-full bg-orange-500" title={t("webFetchTooltip")} />
|
||||
<ProviderCountBadge {...countConfigured(webFetchEntriesAll)} />
|
||||
</h2>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5 gap-3">
|
||||
{webFetchEntries.map(
|
||||
({ providerId, provider, stats, displayAuthType, toggleAuthType }) => (
|
||||
<ProviderCard
|
||||
key={`webfetch-${providerId}`}
|
||||
providerId={providerId}
|
||||
provider={provider}
|
||||
stats={stats}
|
||||
authType={displayAuthType}
|
||||
onToggle={(active) => handleToggleProvider(providerId, toggleAuthType, active)}
|
||||
/>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Aggregators Gateways */}
|
||||
{showSection("apikey") && aggregatorProviderEntries.length > 0 && (
|
||||
<div className="flex flex-col gap-4">
|
||||
|
||||
139
src/app/api/v1/web/fetch/route.ts
Normal file
139
src/app/api/v1/web/fetch/route.ts
Normal file
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* POST /v1/web/fetch
|
||||
*
|
||||
* Extract content from a URL using a configured web-fetch provider.
|
||||
* Supports Firecrawl, Jina Reader, and Tavily Extract.
|
||||
*
|
||||
* Request: { url, provider?, format?, depth?, wait_for_selector?, include_metadata? }
|
||||
* Response: { provider, url, content, links, metadata, screenshot_url }
|
||||
*/
|
||||
|
||||
import { errorResponse } from "@omniroute/open-sse/utils/error.ts";
|
||||
import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts";
|
||||
import { handleWebFetch } from "@omniroute/open-sse/handlers/webFetch.ts";
|
||||
import * as log from "@/sse/utils/logger";
|
||||
import { extractApiKey, isValidApiKey, getProviderCredentials } from "@/sse/services/auth";
|
||||
import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy";
|
||||
import { v1WebFetchSchema } from "@/shared/validation/schemas";
|
||||
import { isValidationFailure, validateBody } from "@/shared/validation/helpers";
|
||||
|
||||
const CORS_HEADERS = {
|
||||
"Access-Control-Allow-Methods": "POST, OPTIONS",
|
||||
"Access-Control-Allow-Headers": "*",
|
||||
};
|
||||
|
||||
const WEB_FETCH_PROVIDERS = ["firecrawl", "jina-reader", "tavily-search"] as const;
|
||||
type WebFetchProviderId = (typeof WEB_FETCH_PROVIDERS)[number];
|
||||
|
||||
export async function OPTIONS() {
|
||||
return new Response(null, { headers: CORS_HEADERS });
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve credentials for a web-fetch provider. Tries each known provider in
|
||||
* priority order when no explicit provider is requested.
|
||||
*/
|
||||
async function resolveCredentials(
|
||||
providerId: WebFetchProviderId
|
||||
): Promise<{ apiKey?: string } | null> {
|
||||
try {
|
||||
const creds = await getProviderCredentials(providerId);
|
||||
return creds ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
let rawBody: unknown;
|
||||
try {
|
||||
rawBody = await request.json();
|
||||
} catch {
|
||||
log.warn("WEB_FETCH", "Invalid JSON body");
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, "Invalid JSON body");
|
||||
}
|
||||
|
||||
const validation = validateBody(v1WebFetchSchema, rawBody);
|
||||
if (isValidationFailure(validation)) {
|
||||
return errorResponse(HTTP_STATUS.BAD_REQUEST, validation.error.message);
|
||||
}
|
||||
const body = validation.data;
|
||||
|
||||
// Optional auth check
|
||||
const apiKeyRaw = extractApiKey(request);
|
||||
if (process.env.REQUIRE_API_KEY === "true" && !apiKeyRaw) {
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Authentication required");
|
||||
}
|
||||
if (apiKeyRaw && !(await isValidApiKey(apiKeyRaw))) {
|
||||
return errorResponse(HTTP_STATUS.UNAUTHORIZED, "Invalid API key");
|
||||
}
|
||||
|
||||
// Enforce API key policies
|
||||
const policy = await enforceApiKeyPolicy(request, "web-fetch");
|
||||
if (policy.rejection) return policy.rejection;
|
||||
|
||||
// Resolve provider + credentials
|
||||
let resolvedProvider: WebFetchProviderId | undefined;
|
||||
let credentials: { apiKey?: string } = {};
|
||||
|
||||
if (body.provider) {
|
||||
resolvedProvider = body.provider as WebFetchProviderId;
|
||||
const creds = await resolveCredentials(resolvedProvider);
|
||||
if (!creds) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
`No credentials configured for web-fetch provider: ${resolvedProvider}. ` +
|
||||
`Add an API key for "${resolvedProvider}" in the dashboard.`
|
||||
);
|
||||
}
|
||||
credentials = creds;
|
||||
} else {
|
||||
// Auto-select: try providers in priority order
|
||||
for (const pid of WEB_FETCH_PROVIDERS) {
|
||||
const creds = await resolveCredentials(pid);
|
||||
if (creds) {
|
||||
resolvedProvider = pid;
|
||||
credentials = creds;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!resolvedProvider) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
`No credentials configured for any web-fetch provider. ` +
|
||||
`Add an API key for one of: ${WEB_FETCH_PROVIDERS.join(", ")}.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
log.info("WEB_FETCH", `${resolvedProvider} | ${body.url} | format=${body.format}`);
|
||||
|
||||
const result = await handleWebFetch(
|
||||
{
|
||||
url: body.url,
|
||||
format: body.format,
|
||||
depth: body.depth as 0 | 1 | 2,
|
||||
wait_for_selector: body.wait_for_selector,
|
||||
include_metadata: body.include_metadata,
|
||||
},
|
||||
credentials,
|
||||
resolvedProvider
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: { message: result.error ?? "Web fetch failed", type: "web_fetch_error" },
|
||||
}),
|
||||
{
|
||||
status: result.status ?? 502,
|
||||
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return new Response(JSON.stringify(result.data), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json", ...CORS_HEADERS },
|
||||
});
|
||||
}
|
||||
@@ -1507,7 +1507,24 @@
|
||||
"result": "Result",
|
||||
"imageDescription": "Generate images from text prompts using OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI and more.",
|
||||
"videoDescription": "Create videos with AnimateDiff, Stable Video Diffusion via ComfyUI or SD WebUI.",
|
||||
"musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI."
|
||||
"musicDescription": "Compose music using Stable Audio Open or MusicGen via ComfyUI.",
|
||||
"kinds": {
|
||||
"embedding": "Embedding",
|
||||
"image": "Image",
|
||||
"imageToText": "Image to Text",
|
||||
"tts": "Text to Speech",
|
||||
"stt": "Speech to Text",
|
||||
"webSearch": "Web Search",
|
||||
"webFetch": "Web Fetch",
|
||||
"video": "Video",
|
||||
"music": "Music"
|
||||
},
|
||||
"noProviders": "No providers configured for this kind yet.",
|
||||
"addConnection": "Add Connection",
|
||||
"backToProviders": "Back to Providers",
|
||||
"connections": "{count} Connections",
|
||||
"noConnections": "No connections yet — add one from the provider page.",
|
||||
"loading": "Loading..."
|
||||
},
|
||||
"search": {
|
||||
"searchQuery": "Search Query",
|
||||
@@ -3917,7 +3934,10 @@
|
||||
"providerDetailValidClaudeCredentialsFile": "Valid Claude credentials file",
|
||||
"providerDetailPathAutoDetectedAllOs": "Path is auto-detected per OS (Linux/Mac/Windows).",
|
||||
"providerDetailMyClaudeAccountPlaceholder": "My Claude account",
|
||||
"providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac)."
|
||||
"providerDetailPathAutoDetected": "Path is auto-detected per OS (Linux/Mac).",
|
||||
"webFetch": "Web Fetch",
|
||||
"webFetchTooltip": "Providers that extract content from web URLs (HTML → Markdown, scrape, screenshot)",
|
||||
"webFetchProvidersHeading": "Web Fetch Providers"
|
||||
},
|
||||
"settings": {
|
||||
"title": "Settings",
|
||||
|
||||
@@ -1505,7 +1505,24 @@
|
||||
"result": "Resultado",
|
||||
"imageDescription": "Gere imagens a partir de prompts de texto usando OpenAI, xAI, Together, Hyperbolic, SD WebUI, ComfyUI e mais.",
|
||||
"videoDescription": "Crie vídeos com AnimateDiff, Stable Video Diffusion via ComfyUI ou SD WebUI.",
|
||||
"musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI."
|
||||
"musicDescription": "Componha músicas usando Stable Audio Open ou MusicGen via ComfyUI.",
|
||||
"kinds": {
|
||||
"embedding": "Embedding",
|
||||
"image": "Imagem",
|
||||
"imageToText": "Imagem para Texto",
|
||||
"tts": "Texto para Fala",
|
||||
"stt": "Fala para Texto",
|
||||
"webSearch": "Busca Web",
|
||||
"webFetch": "Web Fetch",
|
||||
"video": "Vídeo",
|
||||
"music": "Música"
|
||||
},
|
||||
"noProviders": "Nenhum provedor configurado para este tipo.",
|
||||
"addConnection": "Adicionar Conexão",
|
||||
"backToProviders": "Voltar para Provedores",
|
||||
"connections": "{count} Conexões",
|
||||
"noConnections": "Nenhuma conexão ainda — adicione uma na página do provedor.",
|
||||
"loading": "Carregando..."
|
||||
},
|
||||
"search": {
|
||||
"searchQuery": "Consulta de Pesquisa",
|
||||
@@ -3914,7 +3931,10 @@
|
||||
"providerDetailValidClaudeCredentialsFile": "Arquivo de credenciais do Claude válido",
|
||||
"providerDetailPathAutoDetectedAllOs": "O caminho é detectado automaticamente por SO (Linux/Mac/Windows).",
|
||||
"providerDetailMyClaudeAccountPlaceholder": "Minha conta do Claude",
|
||||
"providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac)."
|
||||
"providerDetailPathAutoDetected": "O caminho é detectado automaticamente por SO (Linux/Mac).",
|
||||
"webFetch": "Web Fetch",
|
||||
"webFetchTooltip": "Provedores que extraem conteúdo de URLs (HTML → Markdown, scraping, screenshot)",
|
||||
"webFetchProvidersHeading": "Provedores Web Fetch"
|
||||
},
|
||||
"settings": {
|
||||
"title": "Configurações",
|
||||
|
||||
@@ -2,6 +2,22 @@
|
||||
|
||||
export type RiskNoticeVariant = "oauth" | "webCookie" | "deprecated";
|
||||
|
||||
/**
|
||||
* Service kind — declarative tag for what a provider can do beyond basic LLM chat.
|
||||
* Affects UI filtering only; does not influence request routing.
|
||||
*/
|
||||
export type ServiceKind =
|
||||
| "llm"
|
||||
| "embedding"
|
||||
| "image"
|
||||
| "imageToText"
|
||||
| "tts"
|
||||
| "stt"
|
||||
| "webSearch"
|
||||
| "webFetch"
|
||||
| "video"
|
||||
| "music";
|
||||
|
||||
export interface ProviderRiskNoticeFields {
|
||||
subscriptionRisk?: boolean;
|
||||
riskNoticeVariant?: RiskNoticeVariant;
|
||||
@@ -2118,6 +2134,52 @@ export const APIKEY_PROVIDERS = {
|
||||
passthroughModels: true,
|
||||
authHint: "Get API key at monsterapi.ai",
|
||||
},
|
||||
// ── Web Fetch Providers ─────────────────────────────────────────────────────
|
||||
firecrawl: {
|
||||
id: "firecrawl",
|
||||
alias: "fc",
|
||||
name: "Firecrawl",
|
||||
icon: "language",
|
||||
color: "#FB923C",
|
||||
textIcon: "FC",
|
||||
website: "https://firecrawl.dev",
|
||||
hasFree: true,
|
||||
notice: {
|
||||
text: "Free tier: 500 fetches/month, no credit card needed.",
|
||||
apiKeyUrl: "https://firecrawl.dev/app/api-keys",
|
||||
},
|
||||
serviceKinds: ["webFetch"],
|
||||
},
|
||||
"jina-reader": {
|
||||
id: "jina-reader",
|
||||
alias: "jr",
|
||||
name: "Jina Reader",
|
||||
icon: "menu_book",
|
||||
color: "#0EA5E9",
|
||||
textIcon: "JR",
|
||||
website: "https://jina.ai/reader",
|
||||
hasFree: true,
|
||||
notice: {
|
||||
text: "Free tier: 1M fetches/month.",
|
||||
apiKeyUrl: "https://jina.ai/api-dashboard",
|
||||
},
|
||||
serviceKinds: ["webFetch"],
|
||||
},
|
||||
byteplus: {
|
||||
id: "byteplus",
|
||||
alias: "bpm",
|
||||
name: "BytePlus ModelArk",
|
||||
icon: "cloud",
|
||||
color: "#2563EB",
|
||||
textIcon: "BP",
|
||||
website: "https://console.byteplus.com/ark",
|
||||
hasFree: true,
|
||||
notice: {
|
||||
text: "Free credits for new accounts. Seed 2.0, Kimi K2 Thinking, GLM 4.7, GPT-OSS-120B available.",
|
||||
apiKeyUrl: "https://console.byteplus.com/ark/region:ark+ap-southeast-1/apiKey",
|
||||
},
|
||||
serviceKinds: ["llm"],
|
||||
},
|
||||
};
|
||||
|
||||
// Sub-categories within APIKEY_PROVIDERS (used by dashboard and catalog views).
|
||||
@@ -2355,6 +2417,7 @@ export const SEARCH_PROVIDERS = {
|
||||
website: "https://serper.dev",
|
||||
hasFree: true,
|
||||
authHint: "API key from serper.dev dashboard",
|
||||
serviceKinds: ["webSearch"],
|
||||
},
|
||||
"brave-search": {
|
||||
id: "brave-search",
|
||||
@@ -2377,6 +2440,7 @@ export const SEARCH_PROVIDERS = {
|
||||
website: "https://exa.ai",
|
||||
hasFree: true,
|
||||
authHint: "API key from dashboard.exa.ai",
|
||||
serviceKinds: ["webSearch", "webFetch"],
|
||||
},
|
||||
"tavily-search": {
|
||||
id: "tavily-search",
|
||||
@@ -2388,6 +2452,7 @@ export const SEARCH_PROVIDERS = {
|
||||
website: "https://tavily.com",
|
||||
hasFree: true,
|
||||
authHint: "API key from app.tavily.com (format: tvly-...)",
|
||||
serviceKinds: ["webSearch", "webFetch"],
|
||||
},
|
||||
"google-pse-search": {
|
||||
id: "google-pse-search",
|
||||
|
||||
@@ -10,6 +10,19 @@
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
const SERVICE_KIND_VALUES = [
|
||||
"llm",
|
||||
"embedding",
|
||||
"image",
|
||||
"imageToText",
|
||||
"tts",
|
||||
"stt",
|
||||
"webSearch",
|
||||
"webFetch",
|
||||
"video",
|
||||
"music",
|
||||
] as const;
|
||||
|
||||
export const ProviderSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
alias: z.string().min(1).optional(),
|
||||
@@ -27,6 +40,7 @@ export const ProviderSchema = z.object({
|
||||
freeNote: z.string().optional(),
|
||||
authHint: z.string().optional(),
|
||||
apiHint: z.string().optional(),
|
||||
serviceKinds: z.array(z.enum(SERVICE_KIND_VALUES)).optional(),
|
||||
});
|
||||
|
||||
export const ProvidersMapSchema = z.record(z.string(), ProviderSchema);
|
||||
|
||||
@@ -2147,3 +2147,14 @@ export const v1BatchCreateSchema = z.object({
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
// ── Web Fetch ─────────────────────────────────────────────────────────────────
|
||||
|
||||
export const v1WebFetchSchema = z.object({
|
||||
url: z.string().url("url must be a valid URL (http/https)"),
|
||||
provider: z.enum(["firecrawl", "jina-reader", "tavily-search"]).optional(),
|
||||
format: z.enum(["markdown", "html", "links", "screenshot"]).default("markdown"),
|
||||
depth: z.union([z.literal(0), z.literal(1), z.literal(2)]).default(0),
|
||||
wait_for_selector: z.string().max(256).optional(),
|
||||
include_metadata: z.boolean().default(false),
|
||||
});
|
||||
|
||||
185
tests/unit/firecrawl-executor.test.ts
Normal file
185
tests/unit/firecrawl-executor.test.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { firecrawlFetch } = await import("../../open-sse/executors/firecrawl-fetch.ts");
|
||||
|
||||
// ── firecrawlFetch tests ──────────────────────────────────────────────────────
|
||||
|
||||
test("firecrawlFetch calls api.firecrawl.dev/v1/scrape with Bearer auth", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let captured: { url: string; headers: Record<string, string>; body: Record<string, unknown> } = {
|
||||
url: "",
|
||||
headers: {},
|
||||
body: {},
|
||||
};
|
||||
|
||||
globalThis.fetch = async (url, init = {}) => {
|
||||
captured = {
|
||||
url: String(url),
|
||||
headers: (init as RequestInit).headers as Record<string, string>,
|
||||
body: JSON.parse(String((init as RequestInit).body ?? "{}")),
|
||||
};
|
||||
return new Response(
|
||||
JSON.stringify({ data: { markdown: "# Result", links: [], metadata: { title: "Test" } } }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 0,
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "fc-test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(captured.url, "https://api.firecrawl.dev/v1/scrape");
|
||||
assert.equal(captured.headers["Authorization"], "Bearer fc-test-key");
|
||||
assert.deepEqual(captured.body.formats, ["markdown"]);
|
||||
assert.equal(captured.body.url, "https://example.com");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("firecrawlFetch propagates 401 error without stack trace", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response(JSON.stringify({ error: "Unauthorized" }), {
|
||||
status: 401,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 0,
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "bad-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 401);
|
||||
assert.ok(result.error, "should have error");
|
||||
assert.ok(!result.error.includes("at /"), "error must not contain stack trace");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("firecrawlFetch returns 401 error when no API key", async () => {
|
||||
const result = await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 0,
|
||||
includeMetadata: false,
|
||||
credentials: {},
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 401);
|
||||
assert.ok(!result.error?.includes("at /"));
|
||||
});
|
||||
|
||||
test("firecrawlFetch maps 'html' format correctly", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let capturedFormats: unknown;
|
||||
|
||||
globalThis.fetch = async (_url, init = {}) => {
|
||||
const body = JSON.parse(String((init as RequestInit).body ?? "{}"));
|
||||
capturedFormats = body.formats;
|
||||
return new Response(JSON.stringify({ data: { html: "<html>test</html>", links: [] } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "html",
|
||||
depth: 0,
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "fc-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.deepEqual(capturedFormats, ["html"]);
|
||||
assert.ok(result.data?.content.includes("<html>"), "should return html content");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("firecrawlFetch returns correct response shape", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
markdown: "# Page Title\nContent here.",
|
||||
links: ["https://example.com/about", "https://example.com/contact"],
|
||||
metadata: { title: "Page Title", description: "A test page" },
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 0,
|
||||
includeMetadata: true,
|
||||
credentials: { apiKey: "fc-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.ok(result.data, "should have data");
|
||||
assert.equal(result.data.provider, "firecrawl");
|
||||
assert.equal(result.data.url, "https://example.com");
|
||||
assert.ok(result.data.content.includes("Page Title"), "content should contain title");
|
||||
assert.ok(Array.isArray(result.data.links), "links should be array");
|
||||
assert.equal(result.data.links.length, 2);
|
||||
assert.ok(result.data.metadata?.title === "Page Title");
|
||||
assert.equal(result.data.screenshot_url, null);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("firecrawlFetch forwards depth and wait_for_selector", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let capturedBody: Record<string, unknown> = {};
|
||||
|
||||
globalThis.fetch = async (_url, init = {}) => {
|
||||
capturedBody = JSON.parse(String((init as RequestInit).body ?? "{}"));
|
||||
return new Response(JSON.stringify({ data: { markdown: "" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await firecrawlFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
depth: 1,
|
||||
waitForSelector: "article",
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "fc-key" },
|
||||
});
|
||||
|
||||
assert.equal(capturedBody.maxDepth, 1, "should set maxDepth");
|
||||
assert.equal(capturedBody.waitFor, "article", "should set waitFor");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
174
tests/unit/jina-reader-executor.test.ts
Normal file
174
tests/unit/jina-reader-executor.test.ts
Normal file
@@ -0,0 +1,174 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { jinaReaderFetch } = await import("../../open-sse/executors/jina-reader-fetch.ts");
|
||||
|
||||
// ── jinaReaderFetch tests ─────────────────────────────────────────────────────
|
||||
|
||||
test("jinaReaderFetch calls r.jina.ai/{url} with Bearer auth", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let captured: { url: string; headers: Record<string, string> } = { url: "", headers: {} };
|
||||
|
||||
globalThis.fetch = async (url, init = {}) => {
|
||||
captured = {
|
||||
url: String(url),
|
||||
headers: (init as RequestInit).headers as Record<string, string>,
|
||||
};
|
||||
return new Response("# Hello from Jina", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/plain" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "jina-test-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.ok(
|
||||
captured.url.startsWith("https://r.jina.ai/"),
|
||||
`URL should start with https://r.jina.ai/, got: ${captured.url}`
|
||||
);
|
||||
assert.ok(
|
||||
captured.url.includes(encodeURIComponent("https://example.com")),
|
||||
"URL should include encoded target URL"
|
||||
);
|
||||
assert.equal(captured.headers["Authorization"], "Bearer jina-test-key");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("jinaReaderFetch returns 401 error when no API key", async () => {
|
||||
const result = await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
includeMetadata: false,
|
||||
credentials: {},
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 401);
|
||||
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace");
|
||||
});
|
||||
|
||||
test("jinaReaderFetch propagates non-200 status without stack trace", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response("Forbidden", {
|
||||
status: 403,
|
||||
headers: { "content-type": "text/plain" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "bad-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 403);
|
||||
assert.ok(result.error, "should have error message");
|
||||
assert.ok(!result.error.includes("at /"), "error must not contain stack trace");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("jinaReaderFetch parses JSON response with data.content", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
content: "# Parsed content",
|
||||
title: "Title from JSON",
|
||||
description: "A description",
|
||||
links: ["https://example.com/a", "https://example.com/b"],
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
includeMetadata: true,
|
||||
credentials: { apiKey: "jina-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.ok(result.data, "should have data");
|
||||
assert.equal(result.data.provider, "jina-reader");
|
||||
assert.ok(result.data.content.includes("Parsed content"), "content should be parsed");
|
||||
assert.ok(result.data.metadata?.title === "Title from JSON", "metadata title should be set");
|
||||
assert.equal(result.data.screenshot_url, null);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("jinaReaderFetch falls back to plain text when response is not JSON", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response("Plain text content from Jina", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/plain; charset=utf-8" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "markdown",
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "jina-key" },
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.ok(
|
||||
result.data?.content.includes("Plain text content"),
|
||||
"should include plain text content"
|
||||
);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("jinaReaderFetch sets X-Return-Format header to html for html format", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let capturedHeaders: Record<string, string> = {};
|
||||
|
||||
globalThis.fetch = async (_url, init = {}) => {
|
||||
capturedHeaders = (init as RequestInit).headers as Record<string, string>;
|
||||
return new Response("<html>content</html>", {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/html" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await jinaReaderFetch({
|
||||
url: "https://example.com",
|
||||
format: "html",
|
||||
includeMetadata: false,
|
||||
credentials: { apiKey: "jina-key" },
|
||||
});
|
||||
|
||||
assert.equal(capturedHeaders["X-Return-Format"], "html");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
138
tests/unit/web-fetch-handler.test.ts
Normal file
138
tests/unit/web-fetch-handler.test.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { handleWebFetch } = await import("../../open-sse/handlers/webFetch.ts");
|
||||
|
||||
// ── handleWebFetch — basic routing ───────────────────────────────────────────
|
||||
|
||||
test("handleWebFetch routes to firecrawl when provider=firecrawl", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
markdown: "# Hello World",
|
||||
links: ["https://example.com/page"],
|
||||
metadata: { title: "Hello", description: "A test page" },
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleWebFetch(
|
||||
{ url: "https://example.com", format: "markdown" },
|
||||
{ apiKey: "test-key" },
|
||||
"firecrawl"
|
||||
);
|
||||
|
||||
assert.equal(result.success, true, "should succeed");
|
||||
assert.ok(result.data, "should have data");
|
||||
assert.equal(result.data.provider, "firecrawl");
|
||||
assert.equal(result.data.url, "https://example.com");
|
||||
assert.ok(typeof result.data.content === "string", "content should be string");
|
||||
assert.ok(Array.isArray(result.data.links), "links should be array");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleWebFetch routes to jina-reader when provider=jina-reader", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
data: {
|
||||
content: "# Jina content",
|
||||
title: "Test",
|
||||
description: "desc",
|
||||
links: [],
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } }
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleWebFetch(
|
||||
{ url: "https://example.com", format: "markdown" },
|
||||
{ apiKey: "jina-key" },
|
||||
"jina-reader"
|
||||
);
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(result.data?.provider, "jina-reader");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleWebFetch returns error 401 when no apiKey for firecrawl", async () => {
|
||||
const result = await handleWebFetch({ url: "https://example.com" }, {}, "firecrawl");
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 401);
|
||||
assert.ok(result.error, "should have error message");
|
||||
// Error must not expose stack traces
|
||||
assert.ok(!result.error.includes("at /"), "error must not contain stack trace paths");
|
||||
});
|
||||
|
||||
test("handleWebFetch returns error 401 when no apiKey for jina-reader", async () => {
|
||||
const result = await handleWebFetch({ url: "https://example.com" }, {}, "jina-reader");
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.status, 401);
|
||||
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths");
|
||||
});
|
||||
|
||||
test("handleWebFetch wraps fetch errors via buildErrorBody (no raw stack)", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async () => {
|
||||
throw new Error("at /internal/path/executor.ts:42:10\nnetwork failure");
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await handleWebFetch(
|
||||
{ url: "https://example.com" },
|
||||
{ apiKey: "test-key" },
|
||||
"firecrawl"
|
||||
);
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.ok(result.status != null, "should have status");
|
||||
// Stack trace must be stripped
|
||||
assert.ok(!result.error?.includes("at /"), "error must not contain stack trace paths");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test("handleWebFetch passes depth and wait_for_selector to firecrawl", async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let captured: { body: Record<string, unknown> } = { body: {} };
|
||||
|
||||
globalThis.fetch = async (_url, init) => {
|
||||
captured.body = JSON.parse(String((init as RequestInit).body ?? "{}"));
|
||||
return new Response(JSON.stringify({ data: { markdown: "" } }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await handleWebFetch(
|
||||
{ url: "https://example.com", depth: 2, wait_for_selector: "main" },
|
||||
{ apiKey: "test-key" },
|
||||
"firecrawl"
|
||||
);
|
||||
|
||||
assert.equal(captured.body.maxDepth, 2, "should forward depth");
|
||||
assert.equal(captured.body.waitFor, "main", "should forward wait_for_selector");
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user