diff --git a/open-sse/config/providers/registry/gemini/web/index.ts b/open-sse/config/providers/registry/gemini/web/index.ts index 6843ae86a8..276cfaf589 100644 --- a/open-sse/config/providers/registry/gemini/web/index.ts +++ b/open-sse/config/providers/registry/gemini/web/index.ts @@ -8,9 +8,32 @@ export const gemini_webProvider: RegistryEntry = { baseUrl: "https://gemini.google.com/app", authType: "apikey", authHeader: "cookie", + // #9356: `supportsReasoning: false` is a live-behavior statement, not a guess + // about the underlying Gemini model. The executor drives the gemini.google.com + // web UI by typing a prompt, so it has no thinking-budget control to set and + // never surfaces `reasoning_content` — agent routers reading /v1/models must + // not select these for reasoning work. `toolCalling: false` is the matching + // statement for native function calling; the prompt-emulation shim (#7286) + // stays available and is advertised separately as `toolCalling: "emulated"` + // on the provider constant (src/shared/constants/providers/web-cookie.ts). models: [ - { id: "gemini-3.1-pro", name: "Gemini 3.1 Pro", toolCalling: false }, - { id: "gemini-3.5-flash", name: "Gemini 3.5 Flash", toolCalling: false }, - { id: "gemini-3.1-flash-lite", name: "Gemini 3.1 Flash-Lite", toolCalling: false }, + { + id: "gemini-3.1-pro", + name: "Gemini 3.1 Pro", + toolCalling: false, + supportsReasoning: false, + }, + { + id: "gemini-3.5-flash", + name: "Gemini 3.5 Flash", + toolCalling: false, + supportsReasoning: false, + }, + { + id: "gemini-3.1-flash-lite", + name: "Gemini 3.1 Flash-Lite", + toolCalling: false, + supportsReasoning: false, + }, ], }; diff --git a/open-sse/executors/gemini-web.ts b/open-sse/executors/gemini-web.ts index 975d13093f..8810b43cc3 100644 --- a/open-sse/executors/gemini-web.ts +++ b/open-sse/executors/gemini-web.ts @@ -14,9 +14,13 @@ */ import { BaseExecutor, type ExecuteInput } from "./base.ts"; -import { sanitizeErrorMessage } from "../utils/error.ts"; +import { buildErrorBody, sanitizeErrorMessage } from "../utils/error.ts"; import { prepareToolMessages } from "../translator/webTools.ts"; import { buildToolModeResponse } from "./chatgptWebTools.ts"; +import { + checkGeminiWebUnsupportedControls, + GEMINI_WEB_UNSUPPORTED_CONTROL_CODE, +} from "./gemini-web/capabilities.ts"; // ─── Constants ────────────────────────────────────────────────────────────── @@ -406,6 +410,33 @@ export class GeminiWebExecutor extends BaseExecutor { const { model, body, stream, credentials, signal, log, onCredentialsRefreshed } = input; const requestBody = body as GeminiRequestBody; + // #9356: fail fast on controls this provider cannot honor (reasoning_effort + // above "minimal", forced tool_choice). Runs before the credential check and + // before Playwright launches — the request is unservable no matter which + // cookie is used, and answering 200 with ordinary prose made agents believe + // their reasoning/tool requirements had been met. See ./gemini-web/capabilities.ts. + const violation = checkGeminiWebUnsupportedControls(body as Record); + if (violation) { + log?.warn?.( + "GEMINI-WEB", + `Rejected request: "${violation.param}" is not supported by this provider` + ); + return { + response: new Response( + JSON.stringify( + buildErrorBody(400, violation.message, null, { + type: "invalid_request_error", + code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE, + }) + ), + { status: 400, headers: { "Content-Type": "application/json" } } + ), + url: GEMINI_URL, + headers: {}, + transformedBody: body, + }; + } + const cookie = resolveGeminiWebCookie(credentials); if (!cookie) { return { diff --git a/open-sse/executors/gemini-web/capabilities.ts b/open-sse/executors/gemini-web/capabilities.ts new file mode 100644 index 0000000000..6eefe3072f --- /dev/null +++ b/open-sse/executors/gemini-web/capabilities.ts @@ -0,0 +1,121 @@ +/** + * Request-contract guards for the Gemini Web executor (#9356). + * + * gemini-web is not an API client. It launches Playwright, types ONE flat + * prompt string into the gemini.google.com `.ql-editor` contenteditable, + * presses Enter, and captures the first `StreamGenerate` response off the page + * (see ../gemini-web.ts). There is no JSON request body on the wire, which + * makes two OpenAI controls structurally impossible to honor: + * + * • `reasoning_effort` — no field exists to carry a thinking budget. Unlike + * deepseek-web or perplexity-web, which post a real payload and can flip a + * `thinking_enabled` flag or swap the model preference, there is nothing + * here to set. + * • forced `tool_choice` — the tools support gemini-web does have is the + * prompt-emulation shim (`translator/webTools.ts`, #7286): it ASKS the + * model, in prose, to answer with `{...}` and parses whatever + * comes back. That is best-effort by construction. "required" / "any" / + * a named function is a GUARANTEE, and a prompt cannot make one. + * + * Before this module both were accepted and quietly ignored, so an agent got a + * 200 with `finish_reason: "stop"`, no `reasoning_content`, and `tool_calls: []` + * and concluded its requirements had been met (#9356). Failing the request is + * the honest answer: the caller can drop the control, or route to a model that + * actually implements it. + * + * Deliberately NOT rejected — these are already satisfied or already work: + * • `reasoning_effort: "none" | "minimal"` — asking for as little reasoning as + * possible is something a non-thinking provider trivially complies with. + * • `tool_choice: "auto" | "none"` and plain `tools[]` — the #7286 emulation + * path, which several shipped combos depend on (#5240, #8488). Untouched. + * + * Pure and dependency-free so the whole contract is unit-testable without a + * browser. + */ + +/** `error.code` on every compatibility rejection raised here. */ +export const GEMINI_WEB_UNSUPPORTED_CONTROL_CODE = "unsupported_control_for_provider"; + +/** Effort levels a non-thinking provider already complies with. */ +const SATISFIED_EFFORT_LEVELS = new Set(["none", "minimal"]); + +/** `tool_choice` strings that demand a tool call rather than merely offering one. */ +const FORCING_TOOL_CHOICE_STRINGS = new Set(["required", "any"]); + +/** `tool_choice: { type }` values that pin the model to a specific/any tool. */ +const FORCING_TOOL_CHOICE_TYPES = new Set(["function", "tool", "any"]); + +export interface GeminiWebCapabilityViolation { + /** Which request field could not be honored. */ + param: "reasoning_effort" | "tool_choice"; + /** Client-facing explanation — already safe to put in a response body. */ + message: string; +} + +function normalizeString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim().toLowerCase() : null; +} + +/** + * True when `tool_choice` demands a tool call. Covers the OpenAI strings + * ("required"), the Anthropic-flavored ones the translators also emit ("any"), + * and the object forms that name a function or force any tool. "auto" / "none" + * and every unrecognized shape are treated as non-forcing — this guard only + * blocks contracts it is certain gemini-web cannot keep. + */ +export function isForcingToolChoice(toolChoice: unknown): boolean { + const asString = normalizeString(toolChoice); + if (asString) return FORCING_TOOL_CHOICE_STRINGS.has(asString); + + if (toolChoice && typeof toolChoice === "object" && !Array.isArray(toolChoice)) { + const type = normalizeString((toolChoice as Record).type); + return type !== null && FORCING_TOOL_CHOICE_TYPES.has(type); + } + + return false; +} + +/** True when `reasoning_effort` asks for MORE thinking than "none at all". */ +export function requestsThinkingBudget(reasoningEffort: unknown): boolean { + const effort = normalizeString(reasoningEffort); + if (effort === null) return false; + return !SATISFIED_EFFORT_LEVELS.has(effort); +} + +/** + * Inspect an OpenAI-shaped request body for controls gemini-web cannot honor. + * Returns the first violation found, or `null` when the request is servable. + * + * `reasoning_effort` is checked before `tool_choice` only for determinism; a + * request carrying both is rejected either way. + */ +export function checkGeminiWebUnsupportedControls( + body: Record | null | undefined +): GeminiWebCapabilityViolation | null { + if (!body || typeof body !== "object") return null; + + if (requestsThinkingBudget(body.reasoning_effort)) { + return { + param: "reasoning_effort", + message: + 'Model provider "gemini-web" does not support "reasoning_effort". It drives the ' + + "gemini.google.com web UI through a typed prompt and has no thinking-budget control " + + 'to set, so any effort above "minimal" would be silently ignored. Remove ' + + '"reasoning_effort" (or send "none"/"minimal") or route to a reasoning-capable model.', + }; + } + + if (isForcingToolChoice(body.tool_choice)) { + return { + param: "tool_choice", + message: + 'Model provider "gemini-web" cannot guarantee a forced tool call. Its tool support is ' + + "prompt-emulated — the model is asked to emit a tool block and may answer with prose " + + 'instead — so "tool_choice" values that require one ("required", "any", or a named ' + + 'function) cannot be honored. Use "auto" to keep best-effort tool calling, or route to ' + + "a model with native function calling.", + }; + } + + return null; +} diff --git a/tests/unit/gemini-web-capabilities-9356.test.ts b/tests/unit/gemini-web-capabilities-9356.test.ts new file mode 100644 index 0000000000..110e455c85 --- /dev/null +++ b/tests/unit/gemini-web-capabilities-9356.test.ts @@ -0,0 +1,258 @@ +// Capability enforcement for the Gemini Web executor (#9356). +// +// Reported: gemini-web silently ACCEPTS `reasoning_effort` and +// `tool_choice: "required"` and answers with ordinary prose — HTTP 200, no +// `reasoning_content`, `tool_calls: []`, `finish_reason: "stop"`. An +// AgentChakra/OpenClaw agent then believes its reasoning and tool requirements +// were honored when they were not. +// +// Why neither can be implemented for THIS provider: gemini-web is not an API +// client. It launches Playwright, types a single flat prompt string into the +// gemini.google.com `.ql-editor` contenteditable, presses Enter, and captures +// the first `StreamGenerate` response off the page. There is no request payload +// to carry a thinking budget, and no function-calling channel to force — the +// tools support it does have is the prompt-emulation shim (`webTools.ts`, #7286), +// which ASKS the model to emit `{...}` and cannot GUARANTEE it. +// +// So this suite pins the issue's option (b) for both controls: reject the +// requests we cannot honor, and keep honoring the ones we can. The line drawn: +// +// reasoning_effort none | minimal → allowed (gemini-web not thinking +// IS compliance with "spend little") +// low | medium | high… → 400, a positive request to think +// tool_choice absent | auto | none → allowed (emulation path, #7286) +// required | any | {fn} → 400, a guarantee we cannot make +// +// The guard must run BEFORE Playwright launches, so every executor assertion +// here completes without a browser. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { GeminiWebExecutor } = await import("../../open-sse/executors/gemini-web.ts"); +const { checkGeminiWebUnsupportedControls, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } = + await import("../../open-sse/executors/gemini-web/capabilities.ts"); +const { gemini_webProvider } = + await import("../../open-sse/config/providers/registry/gemini/web/index.ts"); +const { supportsReasoning, supportsToolCalling } = + await import("../../src/lib/modelCapabilities.ts"); +const { providerSupportsEmulatedToolCalling } = + await import("../../open-sse/services/combo/comboStructure.ts"); + +const GET_WEATHER_TOOL = { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather for a city", + parameters: { type: "object", properties: { city: { type: "string" } }, required: ["city"] }, + }, +}; + +interface ErrorBodyLike { + error: { message: string; type: string; code: string }; +} + +/** + * Run the executor with valid-looking credentials. Every case in this suite is + * expected to short-circuit on the capability guard, so Playwright is never + * reached — a test that hangs here means the guard did not fire. + */ +async function run(body: Record) { + return new GeminiWebExecutor().execute({ + model: "gemini-3.6-flash", + body: { messages: [{ role: "user", content: "hi" }], stream: false, ...body }, + stream: false, + credentials: { apiKey: "__Secure-1PSID=test-cookie" }, + signal: AbortSignal.timeout(10_000), + log: null, + }); +} + +// ─── Pure checker: reasoning_effort ───────────────────────────────────────── + +test("#9356 reasoning_effort low/medium/high/xhigh are rejected as unsupported", () => { + for (const effort of ["low", "medium", "high", "xhigh"]) { + const violation = checkGeminiWebUnsupportedControls({ reasoning_effort: effort }); + assert.equal( + violation?.param, + "reasoning_effort", + `reasoning_effort="${effort}" asks gemini-web to think harder, which a typed browser ` + + `prompt cannot express — it must be rejected, not silently dropped` + ); + assert.match(violation!.message, /reasoning_effort/); + } +}); + +test("#9356 reasoning_effort none/minimal and absent stay allowed", () => { + assert.equal(checkGeminiWebUnsupportedControls({}), null); + assert.equal(checkGeminiWebUnsupportedControls({ reasoning_effort: null }), null); + assert.equal(checkGeminiWebUnsupportedControls({ reasoning_effort: "none" }), null); + assert.equal( + checkGeminiWebUnsupportedControls({ reasoning_effort: "minimal" }), + null, + '"minimal" means spend as little reasoning as possible — a non-thinking provider ' + + "already satisfies it, so rejecting it would be gratuitous" + ); + assert.equal(checkGeminiWebUnsupportedControls({ reasoning_effort: " NONE " }), null); +}); + +// ─── Pure checker: tool_choice ────────────────────────────────────────────── + +test("#9356 tool_choice required/any is rejected as unsupported", () => { + for (const choice of ["required", "any"]) { + const violation = checkGeminiWebUnsupportedControls({ + tools: [GET_WEATHER_TOOL], + tool_choice: choice, + }); + assert.equal( + violation?.param, + "tool_choice", + `tool_choice="${choice}" is a guarantee the prompt-emulation shim cannot make` + ); + assert.match(violation!.message, /tool_choice/); + } +}); + +test("#9356 a forced-function tool_choice object is rejected as unsupported", () => { + const violation = checkGeminiWebUnsupportedControls({ + tools: [GET_WEATHER_TOOL], + tool_choice: { type: "function", function: { name: "get_weather" } }, + }); + assert.equal(violation?.param, "tool_choice"); + + // Anthropic-style forcing, which the translators also emit. + assert.equal( + checkGeminiWebUnsupportedControls({ + tools: [GET_WEATHER_TOOL], + tool_choice: { type: "any" }, + })?.param, + "tool_choice" + ); +}); + +test("#9356 tool_choice auto/none and absent keep the #7286 emulation path open", () => { + assert.equal(checkGeminiWebUnsupportedControls({ tools: [GET_WEATHER_TOOL] }), null); + assert.equal( + checkGeminiWebUnsupportedControls({ tools: [GET_WEATHER_TOOL], tool_choice: "auto" }), + null + ); + assert.equal( + checkGeminiWebUnsupportedControls({ tools: [GET_WEATHER_TOOL], tool_choice: "none" }), + null + ); +}); + +test("#9356 forcing is rejected on its own terms, even with no tools[] array", () => { + // An agent that sets tool_choice without tools is already malformed, but the + // point stands: never report success for a forcing contract we ignore. + assert.equal( + checkGeminiWebUnsupportedControls({ tool_choice: "required" })?.param, + "tool_choice" + ); +}); + +// ─── Executor wiring ──────────────────────────────────────────────────────── + +test("#9356 executor returns 400 for reasoning_effort=high before launching a browser", async () => { + const result = await run({ reasoning_effort: "high" }); + + assert.equal(result.response.status, 400); + const body = (await result.response.json()) as ErrorBodyLike; + assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + assert.match(body.error.message, /reasoning_effort/); + assert.equal( + body.error.message.includes("at /"), + false, + "error bodies must stay sanitized — no stack traces" + ); +}); + +test("#9356 executor returns 400 for tool_choice=required before launching a browser", async () => { + const result = await run({ tools: [GET_WEATHER_TOOL], tool_choice: "required" }); + + assert.equal(result.response.status, 400); + const body = (await result.response.json()) as ErrorBodyLike; + assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + assert.match(body.error.message, /tool_choice/); +}); + +test("#9356 the capability guard runs ahead of the credential check", async () => { + // A request that is BOTH uncredentialed and incompatible must report the + // incompatibility: adding a cookie would not make it work. + const result = await new GeminiWebExecutor().execute({ + model: "gemini-3.6-flash", + body: { messages: [{ role: "user", content: "hi" }], reasoning_effort: "high" }, + stream: false, + credentials: {}, + signal: AbortSignal.timeout(10_000), + log: null, + }); + + assert.equal(result.response.status, 400); + const body = (await result.response.json()) as ErrorBodyLike; + assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); +}); + +test("#9356 a supported request still falls through the guard untouched", async () => { + // tool_choice:"auto" + tools[] is the #7286 emulation contract. It must NOT + // be blocked — reaching the (missing) credential check proves the guard let + // it pass, without needing a browser to prove it. + const result = await new GeminiWebExecutor().execute({ + model: "gemini-3.6-flash", + body: { + messages: [{ role: "user", content: "hi" }], + tools: [GET_WEATHER_TOOL], + tool_choice: "auto", + }, + stream: false, + credentials: {}, + signal: AbortSignal.timeout(10_000), + log: null, + }); + + assert.equal(result.response.status, 401, "should reach the cookie check, not the guard"); +}); + +// ─── Catalog metadata ─────────────────────────────────────────────────────── + +test("#9356 registry advertises no native tool calling and no reasoning for gemini-web", () => { + assert.ok(gemini_webProvider.models.length > 0); + for (const model of gemini_webProvider.models) { + assert.equal( + model.toolCalling, + false, + `${model.id} must not advertise native tool calling — /v1/models feeds agent routers` + ); + assert.equal( + model.supportsReasoning, + false, + `${model.id} must advertise reasoning:false so agent routers stop selecting it for ` + + "reasoning work (the executor has no thinking control to drive)" + ); + } +}); + +test("#9356 resolved capabilities — not just the raw registry — report no reasoning/tools", () => { + // The registry literal is only the input; `getResolvedModelCapabilities` is what + // the catalog, the combo compatibility filter and the thinking-budget translator + // actually read. Assert the resolved view so a downstream default cannot quietly + // re-advertise a capability the executor does not have. + for (const model of gemini_webProvider.models) { + const input = { provider: "gemini-web", model: model.id }; + assert.equal(supportsReasoning(input), false, `${model.id} resolved reasoning must be false`); + assert.equal( + supportsToolCalling(input), + false, + `${model.id} resolved NATIVE tool calling must be false — prompt emulation is advertised ` + + 'separately as toolCalling:"emulated" on the provider constant' + ); + } +}); + +test("#9356 the provider still advertises emulated tool calling, so #7286 combos keep routing", () => { + // Guard against over-correcting: dropping the emulation advertisement here would + // make filterTargetsByRequestCompatibility fail these targets closed and break + // emulation-only combos (#5240 / #8488). + assert.equal(providerSupportsEmulatedToolCalling("gemini-web"), true); + assert.equal(providerSupportsEmulatedToolCalling("gweb"), true); +});