feat(mcp): add omniroute_web_fetch tool for URL content extraction (#4510)

Adds an MCP tool to extract URL content via the existing /v1/web/fetch endpoint (Firecrawl/Jina/Tavily). Mirrors omniroute_web_search; scope execute:search; mcp_audit logged.

Integrated into release/v3.8.33.

open-sse/mcp-server/schemas/tools.ts

@@ -459,6 +459,65 @@ export const webSearchTool: McpToolDefinition<typeof webSearchInput, typeof webS
   sourceEndpoints: ["/v1/search"],
 };
 
+// --- Tool 10: omniroute_web_fetch ---
+export const webFetchInput = z.object({
+  url: z
+    .string()
+    .min(1, "URL is required")
+    .describe("The URL to fetch content from"),
+  provider: z
+    .enum(["firecrawl", "jina-reader", "tavily-search"])
+    .optional()
+    .describe("Specific fetch provider to use (default: first available)"),
+  format: z
+    .enum(["markdown", "html", "links", "screenshot"])
+    .optional()
+    .default("markdown")
+    .describe("Output format for the fetched content"),
+  include_metadata: z
+    .boolean()
+    .optional()
+    .default(false)
+    .describe("Include page metadata (title, description) in the response"),
+  depth: z
+    .number()
+    .int()
+    .min(0)
+    .max(2)
+    .optional()
+    .describe("Crawl depth for Firecrawl (0 = single page, max 2)"),
+  wait_for_selector: z
+    .string()
+    .optional()
+    .describe("CSS selector to wait for before extracting content (Firecrawl only)"),
+});
+
+export const webFetchOutput = z.object({
+  provider: z.string(),
+  url: z.string(),
+  content: z.string(),
+  links: z.array(z.string()),
+  metadata: z
+    .object({
+      title: z.string().nullable(),
+      description: z.string().nullable(),
+    })
+    .nullable(),
+  screenshot_url: z.string().nullable(),
+});
+
+export const webFetchTool: McpToolDefinition<typeof webFetchInput, typeof webFetchOutput> = {
+  name: "omniroute_web_fetch",
+  description:
+    "Fetches and extracts content from a URL using OmniRoute's web fetch gateway. Supports multiple providers (Firecrawl, Jina Reader, Tavily) with automatic failover. Returns the page content as markdown, HTML, links, or screenshot, along with metadata.",
+  inputSchema: webFetchInput,
+  outputSchema: webFetchOutput,
+  scopes: ["execute:search"],
+  auditLevel: "basic",
+  phase: 1,
+  sourceEndpoints: ["/v1/web/fetch"],
+};
+
 // ============ Phase 2: Advanced Tools (8) ============
 
 // --- Tool 9: omniroute_simulate_route ---
@@ -1397,6 +1456,7 @@ export const MCP_TOOLS = [
   costReportTool,
   listModelsCatalogTool,
   webSearchTool,
+  webFetchTool,
   simulateRouteTool,
   setBudgetGuardTool,
   setRoutingStrategyTool,

open-sse/mcp-server/server.ts

@@ -29,6 +29,7 @@ import {
   costReportInput,
   listModelsCatalogInput,
   webSearchInput,
+  webFetchInput,
   simulateRouteInput,
   setBudgetGuardInput,
   setRoutingStrategyInput,
@@ -788,6 +789,39 @@ async function handleWebSearch(args: {
   }
 }
 
+async function handleWebFetch(args: {
+  url: string;
+  provider?: "firecrawl" | "jina-reader" | "tavily-search";
+  format?: "markdown" | "html" | "links" | "screenshot";
+  include_metadata?: boolean;
+  depth?: number;
+  wait_for_selector?: string;
+}) {
+  const start = Date.now();
+  try {
+    const body: Record<string, unknown> = {
+      url: args.url,
+      format: args.format ?? "markdown",
+      include_metadata: args.include_metadata ?? false,
+    };
+    if (args.provider) body.provider = args.provider;
+    if (args.depth !== undefined) body.depth = args.depth;
+    if (args.wait_for_selector) body.wait_for_selector = args.wait_for_selector;
+
+    const result = await omniRouteFetch("/v1/web/fetch", {
+      method: "POST",
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(60000),
+    });
+    await logToolCall("omniroute_web_fetch", args, result, Date.now() - start, true);
+    return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    await logToolCall("omniroute_web_fetch", args, null, Date.now() - start, false, msg);
+    return { content: [{ type: "text" as const, text: `Error: ${msg}` }], isError: true };
+  }
+}
+
 // ============ MCP Server Setup ============
 
 /**
@@ -1102,6 +1136,18 @@ export function createMcpServer(): McpServer {
     )
   );
 
+  server.registerTool(
+    "omniroute_web_fetch",
+    {
+      description:
+        "Fetches and extracts content from a URL using OmniRoute's web fetch gateway. Supports multiple providers (Firecrawl, Jina Reader, Tavily) with automatic failover. Returns the page content as markdown, HTML, links, or screenshot, along with metadata.",
+      inputSchema: webFetchInput,
+    },
+    withScopeEnforcement("omniroute_web_fetch", (args) =>
+      handleWebFetch(webFetchInput.parse(args))
+    )
+  );
+
   server.registerTool(
     "omniroute_cache_stats",
     {

src/shared/constants/mcpScopes.ts

@@ -41,6 +41,7 @@ export const MCP_TOOL_SCOPES: Record<string, readonly McpScope[]> = {
   omniroute_check_quota: ["read:quota"],
   omniroute_route_request: ["execute:completions"],
   omniroute_web_search: ["execute:search"],
+  omniroute_web_fetch: ["execute:search"],
   omniroute_cost_report: ["read:usage"],
   omniroute_list_models_catalog: ["read:models"],
 

tests/unit/mcp-web-fetch-tool.test.ts

@@ -0,0 +1,146 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  webFetchInput,
+  webFetchOutput,
+  webFetchTool,
+  MCP_TOOLS,
+  MCP_TOOL_MAP,
+} from "../../open-sse/mcp-server/schemas/tools.ts";
+import { MCP_TOOL_SCOPES } from "../../src/shared/constants/mcpScopes.ts";
+
+// ── Tool definition shape ──
+
+test("webFetchTool has the required McpToolDefinition shape", () => {
+  assert.equal(webFetchTool.name, "omniroute_web_fetch");
+  assert.equal(typeof webFetchTool.description, "string");
+  assert.ok(webFetchTool.description.length > 0);
+  assert.ok(webFetchTool.inputSchema != null);
+  assert.ok(webFetchTool.outputSchema != null);
+  assert.equal(typeof webFetchTool.inputSchema.parse, "function");
+  assert.deepEqual(webFetchTool.scopes, ["execute:search"]);
+  assert.equal(webFetchTool.auditLevel, "basic");
+  assert.equal(webFetchTool.phase, 1);
+  assert.ok(webFetchTool.sourceEndpoints.includes("/v1/web/fetch"));
+});
+
+test("webFetchTool is registered in MCP_TOOLS and MCP_TOOL_MAP", () => {
+  const toolNames = MCP_TOOLS.map((t) => t.name);
+  assert.ok(
+    toolNames.includes("omniroute_web_fetch"),
+    "webFetchTool must be in MCP_TOOLS array"
+  );
+  assert.ok(
+    "omniroute_web_fetch" in MCP_TOOL_MAP,
+    "webFetchTool must be in MCP_TOOL_MAP"
+  );
+});
+
+// ── Scope mapping ──
+
+test("omniroute_web_fetch is mapped in MCP_TOOL_SCOPES with execute:search", () => {
+  const scopes = MCP_TOOL_SCOPES["omniroute_web_fetch"];
+  assert.ok(scopes != null, "omniroute_web_fetch must have a scope mapping");
+  assert.ok(
+    scopes.includes("execute:search"),
+    "omniroute_web_fetch must require execute:search scope"
+  );
+});
+
+// ── Input schema validation ──
+
+test("webFetchInput accepts a valid minimal request (URL only)", () => {
+  const parsed = webFetchInput.parse({ url: "https://example.com" });
+  assert.equal(parsed.url, "https://example.com");
+  assert.equal(parsed.format, "markdown"); // default
+  assert.equal(parsed.include_metadata, false); // default
+});
+
+test("webFetchInput accepts all optional fields", () => {
+  const parsed = webFetchInput.parse({
+    url: "https://example.com",
+    provider: "firecrawl",
+    format: "html",
+    include_metadata: true,
+    depth: 1,
+    wait_for_selector: "#content",
+  });
+  assert.equal(parsed.provider, "firecrawl");
+  assert.equal(parsed.format, "html");
+  assert.equal(parsed.include_metadata, true);
+  assert.equal(parsed.depth, 1);
+  assert.equal(parsed.wait_for_selector, "#content");
+});
+
+test("webFetchInput rejects missing URL", () => {
+  assert.throws(
+    () => webFetchInput.parse({}),
+    /URL is required/,
+    "Missing url should fail validation"
+  );
+});
+
+test("webFetchInput rejects empty URL", () => {
+  assert.throws(
+    () => webFetchInput.parse({ url: "" }),
+    /URL is required/,
+    "Empty url should fail validation"
+  );
+});
+
+test("webFetchInput rejects depth > 2 (matches WebFetchRequest type constraint)", () => {
+  assert.throws(
+    () => webFetchInput.parse({ url: "https://example.com", depth: 3 }),
+    "depth > 2 should fail validation to match the 0 | 1 | 2 type in WebFetchRequest"
+  );
+});
+
+test("webFetchInput accepts depth values 0, 1, 2", () => {
+  for (const depth of [0, 1, 2]) {
+    const parsed = webFetchInput.parse({ url: "https://example.com", depth });
+    assert.equal(parsed.depth, depth);
+  }
+});
+
+test("webFetchInput rejects invalid provider", () => {
+  assert.throws(
+    () => webFetchInput.parse({ url: "https://example.com", provider: "unknown-provider" }),
+    "Unknown provider should fail validation"
+  );
+});
+
+test("webFetchInput rejects invalid format", () => {
+  assert.throws(
+    () => webFetchInput.parse({ url: "https://example.com", format: "xml" }),
+    "Invalid format should fail validation"
+  );
+});
+
+// ── Output schema validation ──
+
+test("webFetchOutput validates a typical scrape response", () => {
+  const result = webFetchOutput.parse({
+    provider: "firecrawl",
+    url: "https://example.com",
+    content: "# Example Domain\n\nThis domain is for use in documentation examples.",
+    links: ["https://iana.org/domains/example"],
+    metadata: { title: "Example Domain", description: "Example site" },
+    screenshot_url: null,
+  });
+  assert.equal(result.provider, "firecrawl");
+  assert.equal(result.links.length, 1);
+  assert.equal(result.metadata?.title, "Example Domain");
+});
+
+test("webFetchOutput validates a response with null metadata", () => {
+  const result = webFetchOutput.parse({
+    provider: "jina-reader",
+    url: "https://example.com",
+    content: "Some content",
+    links: [],
+    metadata: null,
+    screenshot_url: null,
+  });
+  assert.equal(result.metadata, null);
+});