diff --git a/changelog.d/features/7286-geminiweb-tool-calling.md b/changelog.d/features/7286-geminiweb-tool-calling.md new file mode 100644 index 0000000000..47ad9f5a52 --- /dev/null +++ b/changelog.d/features/7286-geminiweb-tool-calling.md @@ -0,0 +1 @@ +- **feat(sse):** `gemini-web` now honors the OpenAI `tools` array by wiring the existing `webTools.ts` prompt-emulation shim (already used by 11 other web-cookie executors) — the tool contract is serialized into the prompt typed into the Gemini web UI, and `{...}` blocks in the response are parsed back into `tool_calls` (malformed JSON degrades to ordinary chat content, never an error), including for streaming requests; `docs/reference/PROVIDER_REFERENCE.md` gained a "Tool calling" column (`native`/`emulated`/`none`) for providers with confirmed ground truth. `claude-web` still silently drops `tools` pending a separate emulate-vs-native design decision (#7286). diff --git a/docs/reference/PROVIDER_REFERENCE.md b/docs/reference/PROVIDER_REFERENCE.md index 008bae9117..e51d8e1bdc 100644 --- a/docs/reference/PROVIDER_REFERENCE.md +++ b/docs/reference/PROVIDER_REFERENCE.md @@ -27,6 +27,8 @@ Total providers: **268**. See category breakdown below. Additional tags: `image`, `video`, `aggregator`, `enterprise`, `embed/rerank`, `self-hosted`. +`Tool calling` (where shown): `native` — real function-calling API; `emulated` — the `tools` array is prompt-emulated via `webTools.ts` (regex-parsed `{...}` blocks); `none` — `tools` is currently silently dropped. See #7286. + Use the dashboard at `/dashboard/providers` to enable, configure, and test each provider. --- @@ -61,35 +63,35 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each ## Web Cookie Providers (27) -| ID | Alias | Name | Tags | Website | Notes | -|----|-------|------|------|---------|-------| -| `adapta-web` | `adp-web` | Adapta.org (Adapta One Web) | Web cookie | [link](https://agent.adapta.one) | Paste your __client cookie value from .clerk.agent.adapta.one (DevTools → Application → Cookies) | -| `blackbox-web` | `bb-web` | Blackbox Web (Subscription) | Web cookie | [link](https://app.blackbox.ai) | Paste your __Secure-authjs.session-token value or full cookie header from app.blackbox.ai | -| `chatgpt-web` | `cgpt-web` | ChatGPT Web (Plus/Pro) | Web cookie | [link](https://chatgpt.com) | Paste your __Secure-next-auth.session-token cookie value from chatgpt.com | -| `claude-web` | `cw` | Claude Web | Web cookie | [link](https://claude.ai) | Paste your session cookie from claude.ai | -| `copilot-m365-web` | `m365copilot` | Microsoft 365 Copilot (BizChat) | Web cookie | [link](https://m365.cloud.microsoft/chat) | Sign in at m365.cloud.microsoft/chat, then open DevTools → Network → filter 'WS' → click the Chathub WebSocket connection. Copy both the access_token query parameter AND the account-specific Chathub path segment from its request URL (wss://…/Chathub/?…&access_token=…). It is NOT an Authorization: Bearer header on an XHR/Fetch request. The token is short-lived; this is an unofficial integration. | -| `copilot-web` | `copilot` | Microsoft Copilot Web | Web cookie | [link](https://copilot.microsoft.com) | Paste your access_token from copilot.microsoft.com (or export a .har file from DevTools while logged in) | -| `deepseek-web` | `ds-web` | DeepSeek Web | Web cookie | [link](https://chat.deepseek.com) | Paste your userToken from chat.deepseek.com — DevTools → Application → Local Storage → userToken | -| `doubao-web` | `db` | Dola Web (ByteDance) | Web cookie | [link](https://www.dola.com) | Paste the full Cookie header from www.dola.com. It should include sessionid, ttwid, and s_v_web_id. If s_v_web_id is unavailable, fp=verify_... from a chat/completion request URL can be used as a fallback. | -| `gemini-business` | `gembiz` | Gemini Business (Enterprise) | Web cookie | [link](https://business.gemini.google) | From your enterprise account: open business.gemini.google/home/cid/{your-cid}, then copy __Secure-1PSID and __Secure-1PSIDTS cookies from DevTools → Application → Cookies. Paste as a cookie header below. | -| `gemini-web` | `gweb` | Gemini Web (Free) | Web cookie | [link](https://gemini.google.com) | Paste your __Secure-1PSID cookie value from gemini.google.com. Optionally add __Secure-1PSIDTS separated by semicolon. | -| `grok-web` | `gw` | Grok Web (Subscription) | Web cookie | [link](https://grok.com) | Paste the full grok.com cookie line from DevTools → Application → Cookies. Include both `sso` and `sso-rw` (e.g. `sso=...; sso-rw=...`) — Grok's anti-bot rejects `sso` on its own. | -| `huggingchat` | `huggingchat` | HuggingChat (Free) | Web cookie | [link](https://huggingface.co/chat) | Paste the full Cookie header from huggingface.co/chat (DevTools → Network → /chat/conversation → Request Headers → Cookie). It should include hf-chat and may also include token / aws-waf-token. | -| `inner-ai` | `in-ai` | Inner.ai (Subscription) | Web cookie | [link](https://app.innerai.com) | Paste your token cookie and email separated by a space: open DevTools → Application → Cookies → .innerai.com, copy the token value, then append a space and your Inner.ai login email. Example: eyJhbG... user@example.com | -| `kimi-web` | `kimi-web` | Kimi Web | Web cookie | [link](https://www.kimi.com) | Paste access_token from www.kimi.com DevTools → Application → Local Storage. A legacy kimi-auth cookie is also accepted. | -| `lmarena` | `lma` | Arena (Free) | Web cookie | [link](https://arena.ai) | Paste the full Cookie header from arena.ai (DevTools → Network → request → Cookie). Include arena-auth-prod-v1.0/.1… and cf_clearance/__cf_bm when present. OmniRoute uses Chrome TLS impersonation; if Arena still 403s, set providerSpecificData.recaptchaV3Token from a live browser session. | -| `microsoft-designer-web` | `msdesigner` | Microsoft Designer (Image Generation) | Web cookie | [link](https://designer.microsoft.com) | Sign in at designer.microsoft.com, then open DevTools → Network, generate an image, and find the request to DallE.ashx?action=GetDallEImagesCogSci. Copy the value of its Authorization: Bearer header (the access_token — no 'Bearer ' prefix). The token is short-lived; this is an unofficial, reverse-engineered integration. | -| `muse-spark-web` | `ms-web` | Muse Spark Web (Meta AI) | Web cookie | [link](https://www.meta.ai) | Paste your ecto_1_sess value or full cookie header from meta.ai | -| `notion-web` | `nw` | Notion AI Web (Unofficial/Experimental) | Web cookie | [link](https://www.notion.so) | Paste your token_v2 cookie value from notion.so (DevTools → Application → Cookies). Optionally append `; space_id=...` and/or `; notion_browser_id=...` if your workspace requires them. | -| `perplexity-web` | `pplx-web` | Perplexity Web (Pro/Max) | Web cookie | [link](https://www.perplexity.ai) | Paste your __Secure-next-auth.session-token cookie value from perplexity.ai | -| `poe-web` | `poe` | Poe Web (Subscription) | Web cookie | [link](https://poe.com) | Paste your p-b cookie value from poe.com (DevTools → Application → Cookies → p-b) | -| `qwen-web` | `qwen-web` | Qwen Web (Free) | Web cookie | [link](https://chat.qwen.ai) | Open chat.qwen.ai, log in, then open DevTools → Application → Local Storage → copy the "token" value (or use tongyi_sso_ticket cookie as Bearer token). | -| `t3-web` | `t3chat` | t3.chat (Pro/Free) | Web cookie | [link](https://t3.chat) | Open t3.chat in your browser, log in, then open DevTools → Application → Local Storage → https://t3.chat. Copy the value of 'convex-session-id'. Also open DevTools → Network, copy the Cookie header from any request. Paste both values here. See provider setup docs for a step-by-step guide. | -| `v0-vercel-web` | `v0-vercel-web` | v0 Vercel Web (Code Gen) | Web cookie | [link](https://v0.dev) | Paste your session cookie from v0.dev (DevTools → Application → Cookies) | -| `venice-web` | `ven` | Venice Web (Privacy) | Web cookie | [link](https://venice.ai) | Paste your session cookie from venice.ai (DevTools → Application → Cookies) | -| `yuanbao-web` | `ybw` | Tencent Yuanbao (Free) | Web cookie | [link](https://yuanbao.tencent.com) | Log in to yuanbao.tencent.com, then paste the full Cookie header (DevTools → Network → any /api request → Request Headers → Cookie). It must contain hy_user and hy_token. | -| `zai-web` | `zw` | Z.ai Web (Free) | Web cookie | [link](https://chat.z.ai) | Paste the full Cookie header from chat.z.ai (must include the token= cookie) | -| `zenmux-free` | `zmf` | ZenMux Free (Web) | Web cookie | [link](https://zenmux.ai) | Login at zenmux.ai, then export all cookies using EditThisCookie or Cookie-Editor and paste the full Cookie header string here. Refresh every ~30 days. | +| ID | Alias | Name | Tags | Website | Notes | Tool calling | +|----|-------|------|------|---------|-------|--------------| +| `adapta-web` | `adp-web` | Adapta.org (Adapta One Web) | Web cookie | [link](https://agent.adapta.one) | Paste your __client cookie value from .clerk.agent.adapta.one (DevTools → Application → Cookies) | emulated | +| `blackbox-web` | `bb-web` | Blackbox Web (Subscription) | Web cookie | [link](https://app.blackbox.ai) | Paste your __Secure-authjs.session-token value or full cookie header from app.blackbox.ai | emulated | +| `chatgpt-web` | `cgpt-web` | ChatGPT Web (Plus/Pro) | Web cookie | [link](https://chatgpt.com) | Paste your __Secure-next-auth.session-token cookie value from chatgpt.com | emulated | +| `claude-web` | `cw` | Claude Web | Web cookie | [link](https://claude.ai) | Paste your session cookie from claude.ai | none | +| `copilot-m365-web` | `m365copilot` | Microsoft 365 Copilot (BizChat) | Web cookie | [link](https://m365.cloud.microsoft/chat) | Sign in at m365.cloud.microsoft/chat, then open DevTools → Network → filter 'WS' → click the Chathub WebSocket connection. Copy both the access_token query parameter AND the account-specific Chathub path segment from its request URL (wss://…/Chathub/?…&access_token=…). It is NOT an Authorization: Bearer header on an XHR/Fetch request. The token is short-lived; this is an unofficial integration. | — | +| `copilot-web` | `copilot` | Microsoft Copilot Web | Web cookie | [link](https://copilot.microsoft.com) | Paste your access_token from copilot.microsoft.com (or export a .har file from DevTools while logged in) | — | +| `deepseek-web` | `ds-web` | DeepSeek Web | Web cookie | [link](https://chat.deepseek.com) | Paste your userToken from chat.deepseek.com — DevTools → Application → Local Storage → userToken | emulated | +| `doubao-web` | `db` | Dola Web (ByteDance) | Web cookie | [link](https://www.dola.com) | Paste the full Cookie header from www.dola.com. It should include sessionid, ttwid, and s_v_web_id. If s_v_web_id is unavailable, fp=verify_... from a chat/completion request URL can be used as a fallback. | — | +| `gemini-business` | `gembiz` | Gemini Business (Enterprise) | Web cookie | [link](https://business.gemini.google) | From your enterprise account: open business.gemini.google/home/cid/{your-cid}, then copy __Secure-1PSID and __Secure-1PSIDTS cookies from DevTools → Application → Cookies. Paste as a cookie header below. | — | +| `gemini-web` | `gweb` | Gemini Web (Free) | Web cookie | [link](https://gemini.google.com) | Paste your __Secure-1PSID cookie value from gemini.google.com. Optionally add __Secure-1PSIDTS separated by semicolon. | emulated | +| `grok-web` | `gw` | Grok Web (Subscription) | Web cookie | [link](https://grok.com) | Paste the full grok.com cookie line from DevTools → Application → Cookies. Include both `sso` and `sso-rw` (e.g. `sso=...; sso-rw=...`) — Grok's anti-bot rejects `sso` on its own. | — | +| `huggingchat` | `huggingchat` | HuggingChat (Free) | Web cookie | [link](https://huggingface.co/chat) | Paste the full Cookie header from huggingface.co/chat (DevTools → Network → /chat/conversation → Request Headers → Cookie). It should include hf-chat and may also include token / aws-waf-token. | — | +| `inner-ai` | `in-ai` | Inner.ai (Subscription) | Web cookie | [link](https://app.innerai.com) | Paste your token cookie and email separated by a space: open DevTools → Application → Cookies → .innerai.com, copy the token value, then append a space and your Inner.ai login email. Example: eyJhbG... user@example.com | emulated | +| `kimi-web` | `kimi-web` | Kimi Web | Web cookie | [link](https://www.kimi.com) | Paste access_token from www.kimi.com DevTools → Application → Local Storage. A legacy kimi-auth cookie is also accepted. | — | +| `lmarena` | `lma` | Arena (Free) | Web cookie | [link](https://arena.ai) | Paste the full Cookie header from arena.ai (DevTools → Network → request → Cookie). Include arena-auth-prod-v1.0/.1… and cf_clearance/__cf_bm when present. OmniRoute uses Chrome TLS impersonation; if Arena still 403s, set providerSpecificData.recaptchaV3Token from a live browser session. | — | +| `microsoft-designer-web` | `msdesigner` | Microsoft Designer (Image Generation) | Web cookie | [link](https://designer.microsoft.com) | Sign in at designer.microsoft.com, then open DevTools → Network, generate an image, and find the request to DallE.ashx?action=GetDallEImagesCogSci. Copy the value of its Authorization: Bearer header (the access_token — no 'Bearer ' prefix). The token is short-lived; this is an unofficial, reverse-engineered integration. | — | +| `muse-spark-web` | `ms-web` | Muse Spark Web (Meta AI) | Web cookie | [link](https://www.meta.ai) | Paste your ecto_1_sess value or full cookie header from meta.ai | emulated | +| `notion-web` | `nw` | Notion AI Web (Unofficial/Experimental) | Web cookie | [link](https://www.notion.so) | Paste your token_v2 cookie value from notion.so (DevTools → Application → Cookies). Include `; space_id=` so live model discovery (getAvailableModels) can list GPT/Claude/Gemini/etc. Optionally also `; notion_browser_id=...` / `; notion_user_id=...`. | — | +| `perplexity-web` | `pplx-web` | Perplexity Web (Pro/Max) | Web cookie | [link](https://www.perplexity.ai) | Paste your __Secure-next-auth.session-token cookie value from perplexity.ai | emulated | +| `poe-web` | `poe` | Poe Web (Subscription) | Web cookie | [link](https://poe.com) | Paste your p-b cookie value from poe.com (DevTools → Application → Cookies → p-b) | — | +| `qwen-web` | `qwen-web` | Qwen Web (Free) | Web cookie | [link](https://chat.qwen.ai) | Open chat.qwen.ai, log in, then open DevTools → Application → Local Storage → copy the "token" value (or use tongyi_sso_ticket cookie as Bearer token). | emulated | +| `t3-web` | `t3chat` | t3.chat (Pro/Free) | Web cookie | [link](https://t3.chat) | Open t3.chat in your browser, log in, then open DevTools → Application → Local Storage → https://t3.chat. Copy the value of 'convex-session-id'. Also open DevTools → Network, copy the Cookie header from any request. Paste both values here. See provider setup docs for a step-by-step guide. | emulated | +| `v0-vercel-web` | `v0-vercel-web` | v0 Vercel Web (Code Gen) | Web cookie | [link](https://v0.dev) | Paste your session cookie from v0.dev (DevTools → Application → Cookies) | — | +| `venice-web` | `ven` | Venice Web (Privacy) | Web cookie | [link](https://venice.ai) | Paste your session cookie from venice.ai (DevTools → Application → Cookies) | — | +| `yuanbao-web` | `ybw` | Tencent Yuanbao (Free) | Web cookie | [link](https://yuanbao.tencent.com) | Log in to yuanbao.tencent.com, then paste the full Cookie header (DevTools → Network → any /api request → Request Headers → Cookie). It must contain hy_user and hy_token. | — | +| `zai-web` | `zw` | Z.ai Web (Free) | Web cookie | [link](https://chat.z.ai) | Paste the full Cookie header from chat.z.ai (must include the token= cookie) | — | +| `zenmux-free` | `zmf` | ZenMux Free (Web) | Web cookie | [link](https://zenmux.ai) | Login at zenmux.ai, then export all cookies using EditThisCookie or Cookie-Editor and paste the full Cookie header string here. Refresh every ~30 days. | — | ## API Key Providers (paid / paid-with-free-credits) (179) @@ -351,22 +353,6 @@ Use the dashboard at `/dashboard/providers` to enable, configure, and test each - Executors: [`open-sse/executors/`](../../open-sse/executors/) (31 files) - Translators: [`open-sse/translator/`](../../open-sse/translator/) -## Provider-specific quota tracking notes - -- **`nvidia` (NVIDIA NIM)** — NVIDIA does not publish a usage/quota API and sends - no rate-limit response headers (confirmed via NVIDIA staff forum posts and an - open, unresolved NVIDIA feature request for a usage API — see #6846). OmniRoute - ships **Phase 1 client-side quota tracking** for this provider: a static local - RPM budget (default 40/min, matching the documented "~40 RPM" free-tier note), - per-model 429 lockout (a throttled model does not cool down the whole - connection — already covered by #6773's `passthroughModels` flag), and a - per-connection concurrency cap (default 6) so parallel bursts queue instead of - 429-storming. All three are operator-overridable via - `ResilienceSettings.providerQuotaOverrides.nvidia` (`{ rpm, concurrency }`). - These numbers are **estimated locally** — NVIDIA does not confirm or expose the - real per-model ceiling. Adaptive per-model ceiling learning (AIMD) and a - dashboard quota card are deferred to Phase 2/3 follow-up issues. - ## See Also - [FREE_TIERS.md](./FREE_TIERS.md) — curated free-tier guide diff --git a/open-sse/executors/gemini-web.ts b/open-sse/executors/gemini-web.ts index c60df8eb35..de018e77b0 100644 --- a/open-sse/executors/gemini-web.ts +++ b/open-sse/executors/gemini-web.ts @@ -15,6 +15,8 @@ import { BaseExecutor, type ExecuteInput } from "./base.ts"; import { sanitizeErrorMessage } from "../utils/error.ts"; +import { prepareToolMessages } from "../translator/webTools.ts"; +import { buildToolModeResponse } from "./chatgptWebTools.ts"; // ─── Constants ────────────────────────────────────────────────────────────── @@ -71,6 +73,52 @@ function formatStreamChunk(content: string, model: string, finishReason: string }; } +/** + * Build the plain-text prompt typed into the Gemini web UI when a tool + * contract is active — the synthetic system message injected by + * `prepareToolMessages()` prepended to the last user message. gemini-web + * only ever sends a single flat string (no native message array), so the + * tool contract and the user's ask are concatenated (#7286). + */ +export function buildGeminiToolPrompt( + effectiveMessages: Array<{ role: string; content: unknown }> +): string { + const toolSystemMsg = effectiveMessages.find((m) => m.role === "system"); + const lastUserMsg = [...effectiveMessages].reverse().find((m) => m.role === "user"); + const userText = typeof lastUserMsg?.content === "string" ? lastUserMsg.content : ""; + const toolPrompt = typeof toolSystemMsg?.content === "string" ? toolSystemMsg.content : ""; + return toolPrompt ? `${toolPrompt}\n\n${userText}` : userText; +} + +/** + * Tool mode: wrap the buffered Gemini response text in the standard OpenAI + * completion shape, then delegate to the shared `buildToolModeResponse()` + * (`chatgptWebTools.ts`) — parses `{...}` blocks out of the + * text into `tool_calls` (malformed JSON degrades to ordinary `content`, + * never throws) and replays either buffered JSON or a terminal SSE chunk + * depending on `stream` (#7286). Exported standalone so the branching logic + * is testable without a full Playwright mock. + */ +export async function buildGeminiToolResponse( + responseText: string, + requestedTools: unknown, + stream: boolean, + model: string, + cid: string, + created: number +): Promise { + const bufferedJson = new Response(JSON.stringify(formatChatCompletion(responseText, model)), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + return buildToolModeResponse(bufferedJson, requestedTools, stream, { + cid, + created, + model, + idSeed: "gwe", + }); +} + /** * Parse cookie string, stripping attributes (Path, Domain, Expires, etc.) * Input: full browser cookie string or just "name=value; name2=value2" @@ -284,8 +332,16 @@ export class GeminiWebExecutor extends BaseExecutor { } const messages = requestBody.messages || []; - const lastUserMsg = messages.filter((m) => m.role === "user").pop(); - const prompt = lastUserMsg?.content || ""; + const { hasTools, requestedTools, effectiveMessages } = prepareToolMessages( + body as Record, + messages + ); + + // hasTools === false: byte-for-byte the original prompt derivation + // (regression guard — #7286 must not touch the no-tools path). + const prompt = hasTools + ? buildGeminiToolPrompt(effectiveMessages) + : messages.filter((m) => m.role === "user").pop()?.content || ""; if (!prompt) { return { @@ -382,6 +438,20 @@ export class GeminiWebExecutor extends BaseExecutor { const modelId = model || "gemini-2.5-pro"; + if (hasTools) { + const cid = `chatcmpl-gwe-${crypto.randomUUID().slice(0, 12)}`; + const created = Math.floor(Date.now() / 1000); + const toolResponse = await buildGeminiToolResponse( + responseText, + requestedTools, + Boolean(stream), + modelId, + cid, + created + ); + return { response: toolResponse, url: GEMINI_URL, headers: {}, transformedBody: body }; + } + if (stream) { // Pseudo-streaming: send complete response as single SSE chunk // Gemini's StreamGenerate returns complete responses, not chunked streams diff --git a/scripts/docs/gen-provider-reference.ts b/scripts/docs/gen-provider-reference.ts index 0b6c5ec9b6..118a7f9bc1 100644 --- a/scripts/docs/gen-provider-reference.ts +++ b/scripts/docs/gen-provider-reference.ts @@ -41,6 +41,12 @@ type ProviderRecord = { hasFree?: boolean; deprecated?: boolean; deprecationReason?: string; + /** + * #7286: native function calling / prompt-emulated via webTools.ts / silently dropped. + * Loosely typed `string` (not a literal union) — the source provider-catalog object + * literals infer widened `string` for this field and are passed in as-is. + */ + toolCalling?: string; [k: string]: unknown; }; @@ -54,13 +60,19 @@ function escapeCell(value: string | undefined): string { return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, " "); } -function row(p: ProviderRecord, category: string): string { +function row(p: ProviderRecord, category: string, includeToolCalling: boolean): string { const alias = p.alias ? `\`${p.alias}\`` : "—"; const hint = p.deprecated ? `⚠️ **DEPRECATED.** ${escapeCell(p.deprecationReason)}` : escapeCell(p.authHint || p.freeNote); const link = p.website ? `[link](${p.website})` : "—"; - return `| \`${p.id}\` | ${alias} | ${escapeCell(p.name)} | ${category} | ${link} | ${hint} |`; + const base = `| \`${p.id}\` | ${alias} | ${escapeCell(p.name)} | ${category} | ${link} | ${hint} |`; + return includeToolCalling ? `${base} ${escapeCell(p.toolCalling)} |` : base; +} + +/** #7286: only render the "Tool calling" column when at least one record in the section sets it. */ +function hasToolCallingColumn(rows: ProviderRecord[]): boolean { + return rows.some((p) => typeof p.toolCalling === "string"); } function categoryTags(id: string): string[] { @@ -82,11 +94,20 @@ function buildSection(title: string, rows: ProviderRecord[], category: string): if (rows.length === 0) return ""; const lines: string[] = []; lines.push(`## ${title} (${rows.length})\n`); - lines.push("| ID | Alias | Name | Tags | Website | Notes |"); - lines.push("|----|-------|------|------|---------|-------|"); + const includeToolCalling = hasToolCallingColumn(rows); + lines.push( + includeToolCalling + ? "| ID | Alias | Name | Tags | Website | Notes | Tool calling |" + : "| ID | Alias | Name | Tags | Website | Notes |" + ); + lines.push( + includeToolCalling + ? "|----|-------|------|------|---------|-------|--------------|" + : "|----|-------|------|------|---------|-------|" + ); for (const p of sortById(rows)) { const tags = [category, ...categoryTags(p.id)].join(", "); - lines.push(row(p, tags)); + lines.push(row(p, tags, includeToolCalling)); } lines.push(""); return lines.join("\n"); @@ -127,6 +148,10 @@ function buildHeader(total: number): string { "", "Additional tags: `image`, `video`, `aggregator`, `enterprise`, `embed/rerank`, `self-hosted`.", "", + "`Tool calling` (where shown): `native` — real function-calling API; `emulated` — the " + + "`tools` array is prompt-emulated via `webTools.ts` (regex-parsed `{...}` " + + "blocks); `none` — `tools` is currently silently dropped. See #7286.", + "", "Use the dashboard at `/dashboard/providers` to enable, configure, and test each provider.", "", "---", diff --git a/src/shared/constants/providers/noauth.ts b/src/shared/constants/providers/noauth.ts index e959971eaa..2b754a30d9 100644 --- a/src/shared/constants/providers/noauth.ts +++ b/src/shared/constants/providers/noauth.ts @@ -34,6 +34,8 @@ export const NOAUTH_PROVIDERS = { serviceKinds: ["llm"], freeNote: "Free — anonymous access to multiple AI models via DuckDuckGo.", authHint: "No credentials required — DuckDuckGo AI Chat is anonymous and free.", + // #7286: tools[] is prompt-emulated via webTools.ts (parseToolCallsFromText). + toolCalling: "emulated", }, "felo-web": { id: "felo-web", diff --git a/src/shared/constants/providers/web-cookie.ts b/src/shared/constants/providers/web-cookie.ts index ba46837004..aa3da130f6 100644 --- a/src/shared/constants/providers/web-cookie.ts +++ b/src/shared/constants/providers/web-cookie.ts @@ -14,6 +14,7 @@ export const WEB_COOKIE_PROVIDERS = { authHint: "Paste your __Secure-next-auth.session-token cookie value from chatgpt.com", subscriptionRisk: true, riskNoticeVariant: "webCookie", + toolCalling: "emulated", }, "grok-web": { id: "grok-web", @@ -40,6 +41,8 @@ export const WEB_COOKIE_PROVIDERS = { "Paste your __Secure-1PSID cookie value from gemini.google.com. Optionally add __Secure-1PSIDTS separated by semicolon.", subscriptionRisk: true, riskNoticeVariant: "webCookie", + // #7286 Level 2: tools[] is prompt-emulated via webTools.ts (parseToolCallsFromText). + toolCalling: "emulated", }, "perplexity-web": { id: "perplexity-web", @@ -52,6 +55,7 @@ export const WEB_COOKIE_PROVIDERS = { authHint: "Paste your __Secure-next-auth.session-token cookie value from perplexity.ai", subscriptionRisk: true, riskNoticeVariant: "webCookie", + toolCalling: "emulated", }, "blackbox-web": { id: "blackbox-web", @@ -65,6 +69,7 @@ export const WEB_COOKIE_PROVIDERS = { "Paste your __Secure-authjs.session-token value or full cookie header from app.blackbox.ai", subscriptionRisk: true, riskNoticeVariant: "webCookie", + toolCalling: "emulated", }, "muse-spark-web": { id: "muse-spark-web", @@ -77,6 +82,7 @@ export const WEB_COOKIE_PROVIDERS = { hasFree: true, freeNote: "Free with login — Meta AI platform with Llama models.", authHint: "Paste your ecto_1_sess value or full cookie header from meta.ai", + toolCalling: "emulated", }, "claude-web": { id: "claude-web", @@ -89,6 +95,9 @@ export const WEB_COOKIE_PROVIDERS = { authHint: "Paste your session cookie from claude.ai", subscriptionRisk: true, riskNoticeVariant: "webCookie", + // #7286 Level 3 (deferred): still silently drops tools[] — getDefaultTools() + // is a fixed Claude.ai backend contract; needs an emulate-vs-native decision. + toolCalling: "none", }, "deepseek-web": { id: "deepseek-web", @@ -102,6 +111,7 @@ export const WEB_COOKIE_PROVIDERS = { "Paste your userToken from chat.deepseek.com — DevTools → Application → Local Storage → userToken", subscriptionRisk: true, riskNoticeVariant: "webCookie", + toolCalling: "emulated", }, "copilot-web": { id: "copilot-web", @@ -156,6 +166,7 @@ export const WEB_COOKIE_PROVIDERS = { "Open t3.chat in your browser, log in, then open DevTools → Application → Local Storage → https://t3.chat. " + "Copy the value of 'convex-session-id'. Also open DevTools → Network, copy the Cookie header from any request. " + "Paste both values here. See provider setup docs for a step-by-step guide.", + toolCalling: "emulated", }, "inner-ai": { id: "inner-ai", @@ -169,6 +180,7 @@ export const WEB_COOKIE_PROVIDERS = { riskNoticeVariant: "webCookie", authHint: "Paste your token cookie and email separated by a space: open DevTools → Application → Cookies → .innerai.com, copy the token value, then append a space and your Inner.ai login email. Example: eyJhbG... user@example.com", + toolCalling: "emulated", }, "adapta-web": { id: "adapta-web", @@ -182,6 +194,7 @@ export const WEB_COOKIE_PROVIDERS = { riskNoticeVariant: "webCookie", authHint: "Paste your __client cookie value from .clerk.agent.adapta.one (DevTools → Application → Cookies)", + toolCalling: "emulated", }, lmarena: { // Wire id stays `lmarena` for DB/combo/model-prefix back-compat. @@ -311,6 +324,7 @@ export const WEB_COOKIE_PROVIDERS = { authHint: "Open chat.qwen.ai, log in, then open DevTools → Application → Local Storage → " + 'copy the "token" value (or use tongyi_sso_ticket cookie as Bearer token).', + toolCalling: "emulated", }, "gemini-business": { id: "gemini-business", diff --git a/tests/unit/gemini-web-tool-calling-7286.test.ts b/tests/unit/gemini-web-tool-calling-7286.test.ts new file mode 100644 index 0000000000..e7a6ec34ee --- /dev/null +++ b/tests/unit/gemini-web-tool-calling-7286.test.ts @@ -0,0 +1,293 @@ +// Tool calling for the Gemini Web executor (#7286) — Level 2 of the staged +// approach in the issue: wire the existing `webTools.ts` prompt-emulation +// shim (already proven across 11 other web-cookie executors) into +// gemini-web.ts. claude-web (Level 3) is explicitly out of scope. +// +// gemini-web buffers the *entire* Gemini response before returning (it has +// no true token-by-token streaming — see the executor's docblock), so the +// tool-call decision is made once on the full text, then replayed either as +// a buffered JSON completion or a single terminal SSE chunk. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { GeminiWebExecutor, buildGeminiToolResponse, buildGeminiToolPrompt } = await import( + "../../open-sse/executors/gemini-web.ts" +); + +interface ToolCallLike { + function: { name: string; arguments: string }; +} + +interface ChatCompletionLike { + choices: Array<{ + message: { content: string | null; tool_calls?: ToolCallLike[] }; + finish_reason: string; + }>; +} + +interface StreamChunkLike { + choices: Array<{ + delta: { role?: string; content?: string; tool_calls?: ToolCallLike[] }; + finish_reason: string | null; + }>; +} + +const GET_WEATHER_TOOL = { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather for a city", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, +}; + +// ─── Pure-function tests: buildGeminiToolResponse ─────────────────────────── +// (the branching/parsing logic extracted out of the tool-mode path so it is +// directly testable without a full Playwright mock) + +test("#7286: well-formed tool call → tool_calls, content:null, finish_reason:tool_calls (non-streaming)", async () => { + const responseText = + 'Sure, let me check.\n{"name":"get_weather","arguments":{"city":"Paris"}}'; + + const response = await buildGeminiToolResponse( + responseText, + [GET_WEATHER_TOOL], + false, + "gemini-3.1-pro", + "chatcmpl-gwe-test", + 1700000000 + ); + + assert.equal(response.status, 200); + const json = (await response.json()) as ChatCompletionLike; + const choice = json.choices[0]; + assert.equal(choice.message.content, null); + assert.equal(choice.finish_reason, "tool_calls"); + assert.equal(choice.message.tool_calls.length, 1); + assert.equal(choice.message.tool_calls[0].function.name, "get_weather"); + assert.deepEqual(JSON.parse(choice.message.tool_calls[0].function.arguments), { + city: "Paris", + }); +}); + +test("#7286: malformed JSON degrades to ordinary chat content — never throws, never a 500", async () => { + const responseText = "Here's what I found: {not valid json"; + + const response = await buildGeminiToolResponse( + responseText, + [GET_WEATHER_TOOL], + false, + "gemini-3.1-pro", + "chatcmpl-gwe-test", + 1700000000 + ); + + assert.equal(response.status, 200); + const json = (await response.json()) as ChatCompletionLike; + const choice = json.choices[0]; + assert.equal(choice.finish_reason, "stop"); + assert.equal(choice.message.tool_calls, undefined); + assert.ok(typeof choice.message.content === "string" && choice.message.content.length > 0); +}); + +test("#7286: streaming assembly — role chunk, then terminal chunk with delta.tool_calls, then [DONE]", async () => { + const responseText = '{"name":"get_weather","arguments":{"city":"Tokyo"}}'; + + const response = await buildGeminiToolResponse( + responseText, + [GET_WEATHER_TOOL], + true, + "gemini-3.1-pro", + "chatcmpl-gwe-test", + 1700000000 + ); + + assert.equal(response.status, 200); + assert.equal(response.headers.get("Content-Type"), "text/event-stream"); + const bodyText = await response.text(); + const dataLines = bodyText + .split("\n\n") + .map((chunk) => chunk.trim()) + .filter((chunk) => chunk.startsWith("data: ")) + .map((chunk) => chunk.slice("data: ".length)); + + assert.equal(dataLines[dataLines.length - 1], "[DONE]"); + const parsed: StreamChunkLike[] = dataLines + .slice(0, -1) + .map((line) => JSON.parse(line) as StreamChunkLike); + assert.equal(parsed.length, 2, "expected a role chunk + one terminal chunk"); + assert.deepEqual(parsed[0].choices[0].delta, { role: "assistant" }); + assert.equal(parsed[0].choices[0].finish_reason, null); + + const terminal = parsed[1]; + assert.equal(terminal.choices[0].finish_reason, "tool_calls"); + assert.equal(terminal.choices[0].delta.tool_calls?.length, 1); + assert.equal(terminal.choices[0].delta.tool_calls?.[0].function.name, "get_weather"); +}); + +test("#7286: no block and no requested tools → toolCalls untouched (plain content passthrough)", async () => { + const response = await buildGeminiToolResponse( + "Just a normal answer, nothing to call.", + [GET_WEATHER_TOOL], + false, + "gemini-3.1-pro", + "chatcmpl-gwe-test", + 1700000000 + ); + + const json = (await response.json()) as ChatCompletionLike; + const choice = json.choices[0]; + assert.equal(choice.finish_reason, "stop"); + assert.equal(choice.message.tool_calls, undefined); + assert.equal(choice.message.content, "Just a normal answer, nothing to call."); +}); + +// ─── buildGeminiToolPrompt ─────────────────────────────────────────────────── + +test("#7286: buildGeminiToolPrompt concatenates the synthetic tool system prompt + last user message", () => { + const effectiveMessages = [ + { role: "system", content: "TOOL CONTRACT HERE" }, + { role: "user", content: "what's the weather in Paris?" }, + ]; + const prompt = buildGeminiToolPrompt(effectiveMessages); + assert.equal(prompt, "TOOL CONTRACT HERE\n\nwhat's the weather in Paris?"); +}); + +test("#7286: buildGeminiToolPrompt falls back to the user text alone when there is no system message", () => { + const effectiveMessages = [{ role: "user", content: "hello" }]; + assert.equal(buildGeminiToolPrompt(effectiveMessages), "hello"); +}); + +// ─── Full executor integration (Playwright mocked) ────────────────────────── +// +// Mirrors the mocking pattern already used in tests/unit/gemini-web.test.ts +// ("Normalizes a bare __Secure-1PSID value..."): fake out `chromium.launch` +// so `execute()` runs the real request/response wiring — including +// `prepareToolMessages` → prompt construction → `parseStreamResponse` → +// `buildGeminiToolResponse` — end to end, proving the tool-mode wiring +// actually reaches the Playwright-driven code path (not just the extracted +// pure function). + +function makeStreamGenerateChunk(text: string): string { + const inner = new Array(80).fill(null); + inner[4] = [[null, [text]]]; + return `[["wrb.fr", null, ${JSON.stringify(JSON.stringify(inner))}]]`; +} + +function makeStreamGenerateRaw(text: string): string { + return `)]}'\n10\n${makeStreamGenerateChunk(text)}`; +} + +interface FakePlaywrightResponse { + url: () => string; + text: () => Promise; +} + +type FakeResponseHandler = (resp: FakePlaywrightResponse) => Promise; + +async function withMockedGeminiBrowser( + responseText: string, + fn: (typedPrompt: { value: string }) => Promise +): Promise { + const playwright = await import("playwright"); + const originalLaunch = playwright.chromium.launch; + const typedPrompt = { value: "" }; + + playwright.chromium.launch = (async () => ({ + newContext: async () => ({ + addCookies: async () => {}, + newPage: async () => { + let respHandler: FakeResponseHandler | null = null; + const page = { + on: (event: string, cb: FakeResponseHandler) => { + if (event === "response") respHandler = cb; + }, + goto: async () => {}, + waitForTimeout: async () => {}, + waitForSelector: async () => ({ click: async () => {} }), + keyboard: { + type: async (text: string) => { + typedPrompt.value = text; + }, + press: async () => { + if (respHandler) { + await respHandler({ + url: () => "https://gemini.google.com/_/BardChatUi/data/.../StreamGenerate?x", + text: async () => makeStreamGenerateRaw(responseText), + }); + } + }, + }, + }; + return page; + }, + }), + close: async () => {}, + })) as unknown as typeof playwright.chromium.launch; + + try { + return await fn(typedPrompt); + } finally { + playwright.chromium.launch = originalLaunch; + } +} + +test("#7286: executor integration — tools[] present reaches tool_calls end to end", async () => { + const responseText = + '{"name":"get_weather","arguments":{"city":"Berlin"}}'; + + await withMockedGeminiBrowser(responseText, async () => { + const executor = new GeminiWebExecutor(); + const result = await executor.execute({ + model: "gemini-3.1-pro", + body: { + messages: [{ role: "user", content: "what's the weather in Berlin?" }], + stream: false, + tools: [GET_WEATHER_TOOL], + }, + stream: false, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10000), + log: null, + }); + + assert.equal(result.response.status, 200); + const json = (await result.response.json()) as ChatCompletionLike; + const choice = json.choices[0]; + assert.equal(choice.finish_reason, "tool_calls"); + assert.equal(choice.message.tool_calls?.[0].function.name, "get_weather"); + }); +}); + +test("#7286: no-tool passthrough regression — unchanged prompt derivation + response shape when tools absent", async () => { + await withMockedGeminiBrowser("Just chatting, no tools here.", async (typedPrompt) => { + const executor = new GeminiWebExecutor(); + const result = await executor.execute({ + model: "gemini-3.1-pro", + body: { + messages: [{ role: "user", content: "hello there" }], + stream: false, + }, + stream: false, + credentials: { apiKey: "test-cookie" }, + signal: AbortSignal.timeout(10000), + log: null, + }); + + // Old behavior: the prompt typed into the UI is exactly the last user + // message — no tool contract prepended. + assert.equal(typedPrompt.value, "hello there"); + + assert.equal(result.response.status, 200); + const json = (await result.response.json()) as ChatCompletionLike; + const choice = json.choices[0]; + assert.equal(choice.message.content, "Just chatting, no tools here."); + assert.equal(choice.message.tool_calls, undefined); + assert.equal(choice.finish_reason, "stop"); + }); +});