From ceafa55824f6648983bf593de91a0976af791ce9 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Thu, 17 Sep 2026 13:06:37 -0300 Subject: [PATCH] fix(providers): select and verify the requested gemini-web model/mode before answering (#13381) (#13919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: GeminiWebExecutor.execute() opened the identical fixed https://gemini.google.com/app URL and ran the identical Playwright interaction sequence for every advertised gweb/ id. `model` was read only AFTER the response was captured, purely to stamp the OpenAI-shaped response — never to influence what was actually clicked/typed, so two different advertised models produced byte-identical automation and the response `model` field was a caller-supplied label, not an observed fact. Fix (owner decision, Option B): a new model -> Gemini UI mode map (open-sse/executors/gemini-web/modeSelection.ts) drives an in-browser selection step before anything is typed — try the mode control, read back the active-mode indicator, and only proceed on a confirmed match. An unconfirmed model, or a requested Extended Thinking control that cannot be confirmed (#13381 follow-up comment), fails closed with 400 unsupported_control_for_provider instead of silently running the account default under the requested label. The selectors involved are UNVALIDATED (no live Gemini account from this checkout) — see the PR's "Selector set is UNVALIDATED" section and the required live smoke. Regression test: tests/unit/issue-13381-gemini-web-model-selection.test.ts --- .../fixes/13381-gemini-web-model-selection.md | 1 + open-sse/executors/gemini-web.ts | 106 +++++- .../executors/gemini-web/modeSelection.ts | 214 ++++++++++++ .../unit/gemini-web-capabilities-9356.test.ts | 92 ++++-- ...e-13381-gemini-web-model-selection.test.ts | 305 ++++++++++++++++++ 5 files changed, 688 insertions(+), 30 deletions(-) create mode 100644 changelog.d/fixes/13381-gemini-web-model-selection.md create mode 100644 open-sse/executors/gemini-web/modeSelection.ts create mode 100644 tests/unit/issue-13381-gemini-web-model-selection.test.ts diff --git a/changelog.d/fixes/13381-gemini-web-model-selection.md b/changelog.d/fixes/13381-gemini-web-model-selection.md new file mode 100644 index 0000000000..2d6d7d8a12 --- /dev/null +++ b/changelog.d/fixes/13381-gemini-web-model-selection.md @@ -0,0 +1 @@ +- **fix(providers):** `gemini-web` now attempts to select and verify the requested Gemini UI mode (and Extended Thinking) before answering, and fails closed with a clear 400 instead of silently running the account default under a mismatched model label (#13381) — thanks @formilw diff --git a/open-sse/executors/gemini-web.ts b/open-sse/executors/gemini-web.ts index a55d8bb331..08af30ace9 100644 --- a/open-sse/executors/gemini-web.ts +++ b/open-sse/executors/gemini-web.ts @@ -22,11 +22,20 @@ import { buildToolModeResponse } from "./chatgptWebTools.ts"; import { checkGeminiWebUnsupportedControls, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE, + isForcingToolChoice, + requestsThinkingBudget, } from "./gemini-web/capabilities.ts"; +import { + describeModeSelectionFailure, + selectGeminiExtendedThinking, + selectGeminiModel, +} from "./gemini-web/modeSelection.ts"; // ─── Constants ────────────────────────────────────────────────────────────── const GEMINI_URL = "https://gemini.google.com/app"; +/** Response-label fallback when no `model` was requested at all (unchanged since pre-#13381). */ +const DEFAULT_MODEL_ID = "gemini-2.5-pro"; // Re-exported for backward compatibility: some tests/callers import this classification helper // from gemini-web.ts, its original home (#3516). The implementation now lives in @@ -416,21 +425,28 @@ export class GeminiWebExecutor extends BaseExecutor { const { model, body, stream, credentials, signal, log, onCredentialsRefreshed } = input; const requestBody = body as GeminiRequestBody; - // #9356: fail fast on controls this provider cannot honor (reasoning_effort - // above "minimal", forced tool_choice). Runs before the credential check and - // before Playwright launches — the request is unservable no matter which - // cookie is used, and answering 200 with ordinary prose made agents believe - // their reasoning/tool requirements had been met. See ./gemini-web/capabilities.ts. - const violation = checkGeminiWebUnsupportedControls(body as Record); - if (violation) { + // #9356: forced tool_choice is a guarantee gemini-web's prompt-emulation shim can + // never make — no UI control for it exists at all, on any account, so this still + // fails fast before the credential check and before Playwright launches. See + // ./gemini-web/capabilities.ts. + // + // `reasoning_effort` (Extended Thinking) is different (#13381 follow-up comment, + // 2026-09-11): eligible Gemini accounts DO expose a real Extended Thinking toggle + // in the UI, so a blanket "we can never do this" would be the same dishonesty this + // fix exists to remove. It is no longer rejected here — it is attempted, verified + // via read-back, and only THEN rejected if it cannot be confirmed. See the in-browser + // step below and ./gemini-web/modeSelection.ts. + const rawBody = body as Record; + if (isForcingToolChoice(rawBody.tool_choice)) { + const violation = checkGeminiWebUnsupportedControls({ tool_choice: rawBody.tool_choice }); log?.warn?.( "GEMINI-WEB", - `Rejected request: "${violation.param}" is not supported by this provider` + `Rejected request: "${violation!.param}" is not supported by this provider` ); return { response: new Response( JSON.stringify( - buildErrorBody(400, violation.message, null, { + buildErrorBody(400, violation!.message, null, { type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE, }) @@ -442,6 +458,7 @@ export class GeminiWebExecutor extends BaseExecutor { transformedBody: body, }; } + const wantsExtendedThinking = requestsThinkingBudget(rawBody.reasoning_effort); const cookie = resolveGeminiWebCookie(credentials); if (!cookie) { @@ -493,6 +510,11 @@ export class GeminiWebExecutor extends BaseExecutor { }; } + // Resolved up front (#13381) — the pre-fix code only read `model` AFTER the + // response was already captured, purely to stamp it on the reply. It now also + // drives the in-browser mode-selection step below, before anything is typed. + const modelId = model || DEFAULT_MODEL_ID; + let browser: any = null; let abortBrowser: (() => void) | null = null; try { @@ -549,6 +571,70 @@ export class GeminiWebExecutor extends BaseExecutor { } await page.waitForTimeout(3000); + // #13381 (Option B): verify the requested Gemini UI mode is actually active + // BEFORE anything is typed. `gemini-3.1-pro` is the mode gemini.google.com/app + // already opens to (no interaction attempted); every other advertised model is + // switched to and its active-mode indicator is read back — a confirmed match is + // required, or the request is rejected instead of silently running the account + // default under the requested model's label. See ./gemini-web/modeSelection.ts + // for why the selectors involved are UNVALIDATED and why that is safe here. + if (model) { + const modelSelection = await selectGeminiModel(page, modelId); + if (!modelSelection.confirmed) { + log?.warn?.( + "GEMINI-WEB", + `Rejected request: could not confirm Gemini UI mode for "${modelId}" ` + + `(${modelSelection.reason})` + ); + return { + response: new Response( + JSON.stringify( + buildErrorBody( + 400, + describeModeSelectionFailure(modelId, modelSelection.reason), + null, + { type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } + ) + ), + { status: 400, headers: { "Content-Type": "application/json" } } + ), + url: GEMINI_URL, + headers: {}, + transformedBody: body, + }; + } + } + + // Extended Thinking (#13381 follow-up, 2026-09-11): same detect-and-verify, + // fail-closed pattern as model selection above — attempted only when requested + // (`reasoning_effort` above "minimal"), never assumed available. + if (wantsExtendedThinking) { + const thinkingSelection = await selectGeminiExtendedThinking(page); + if (!thinkingSelection.confirmed) { + log?.warn?.( + "GEMINI-WEB", + `Rejected request: could not confirm Extended Thinking is available ` + + `(${thinkingSelection.reason})` + ); + return { + response: new Response( + JSON.stringify( + buildErrorBody( + 400, + describeModeSelectionFailure("Extended Thinking", thinkingSelection.reason), + null, + { type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } + ) + ), + { status: 400, headers: { "Content-Type": "application/json" } } + ), + url: GEMINI_URL, + headers: {}, + transformedBody: body, + }; + } + } + // Type and send message const inputEl = await page.waitForSelector(".ql-editor, [contenteditable='true']", { timeout: 10000, @@ -584,8 +670,6 @@ export class GeminiWebExecutor extends BaseExecutor { await this.persistRotatedCookies(context, cookie, credentials, onCredentialsRefreshed, log); - const modelId = model || "gemini-2.5-pro"; - if (hasTools) { const cid = `chatcmpl-gwe-${crypto.randomUUID().slice(0, 12)}`; const created = Math.floor(Date.now() / 1000); diff --git a/open-sse/executors/gemini-web/modeSelection.ts b/open-sse/executors/gemini-web/modeSelection.ts new file mode 100644 index 0000000000..3dcb96a154 --- /dev/null +++ b/open-sse/executors/gemini-web/modeSelection.ts @@ -0,0 +1,214 @@ +/** + * Per-model Gemini Web UI mode selection + verification (#13381, Option B). + * + * Before this module, `GeminiWebExecutor.execute()` read the `model` field + * only AFTER Gemini had already answered — purely to stamp the OpenAI-shaped + * response, never to influence what was actually typed/clicked in the + * browser (../gemini-web.ts). Every advertised `gweb/` alias ran the + * identical automation, so the response `model` field was a caller-supplied + * label, not an observed fact. + * + * This module gives the executor a real selection step: for each advertised + * model (and for the Extended Thinking control, #13381 follow-up comment + * 2026-09-11), it tries the corresponding UI control, then READS BACK the + * active-mode indicator to confirm the switch actually took effect before + * the caller is allowed to proceed. `gemini-3.1-pro` is treated as the mode + * `gemini.google.com/app` already opens to (no interaction needed); every + * other advertised model requires a confirmed switch. + * + * ⚠️ SELECTOR SET IS UNVALIDATED. Every CSS selector below is a best-effort + * guess at Gemini's current DOM, not a value confirmed against a live + * account — this checkout has no live Gemini session and no way to inspect + * Google's current markup (which also changes without notice). That is fine + * BY DESIGN: `selectGeminiUiMode()` never trusts an unconfirmed guess. If a + * selector is wrong — the likely outcome until the mandatory live smoke in + * the PR is run and these values are corrected against the real DOM — the + * toggle/indicator lookup simply fails, the read-back does not confirm the + * requested mode, and the caller gets a clear 400 + * (`unsupported_control_for_provider`) instead of a response silently + * labeled with a model/mode that never actually ran — the exact dishonesty + * #13381 reported. Update the selectors only after inspecting the real DOM + * on a live account, never from guesswork. + */ + +/** Reasons `selectGeminiUiMode`/`selectGeminiModel` can fail closed. */ +export type GeminiModeSelectionFailureReason = + "control_not_found" | "indicator_not_found" | "indicator_mismatch" | "unknown_model"; + +export interface GeminiWebModeDescriptor { + /** The advertised `gweb/` id, or a synthetic label for non-model controls. */ + readonly id: string; + /** + * true: this is the mode `gemini.google.com/app` already shows on load — no + * UI interaction is attempted, nothing to confirm. + */ + readonly isDefault: boolean; + /** UNVALIDATED — opens/activates the control for this mode. Ignored when `isDefault`. */ + readonly toggleSelector?: string; + /** UNVALIDATED — element whose text proves which mode is now active. */ + readonly activeIndicatorSelector?: string; + /** Text the active-mode indicator must contain once the switch is confirmed. */ + readonly expectedIndicatorPattern?: RegExp; +} + +/** + * Model -> Gemini UI mode map for the three `gweb/` ids advertised by + * `open-sse/config/providers/registry/gemini/web/index.ts`. Keep the two + * catalogs in sync: an advertised model with no entry here fails closed as + * `unknown_model` (see `resolveGeminiModelMode`), which is the intended, + * honest behavior for a model this module cannot yet select. + */ +export const GEMINI_WEB_MODEL_MODES: Readonly> = { + "gemini-3.1-pro": { id: "gemini-3.1-pro", isDefault: true }, + "gemini-3.7-flash": { + id: "gemini-3.7-flash", + isDefault: false, + toggleSelector: + '[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]', + activeIndicatorSelector: + '[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]', + expectedIndicatorPattern: /\bflash\b(?!.*\blite\b)/i, + }, + "gemini-3.1-flash-lite": { + id: "gemini-3.1-flash-lite", + isDefault: false, + toggleSelector: + '[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]', + activeIndicatorSelector: + '[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]', + expectedIndicatorPattern: /flash.*\blite\b/i, + }, +}; + +/** The Extended Thinking control (#13381 follow-up, 2026-09-11) — account-dependent, not a model. */ +export const GEMINI_WEB_EXTENDED_THINKING_MODE: GeminiWebModeDescriptor = { + id: "extended-thinking", + isDefault: false, + toggleSelector: + '[data-test-id="deep-think-toggle"], button[aria-label*="extended thinking" i], ' + + 'button[aria-label*="deep think" i]', + activeIndicatorSelector: + '[data-test-id="deep-think-toggle"][aria-pressed="true"], [data-test-id="deep-think-toggle"].is-active', + expectedIndicatorPattern: /thinking|deep think/i, +}; + +/** Minimal Playwright-shaped element the read-back needs — real `ElementHandle`s satisfy this. */ +export interface GeminiAutomationElement { + click(): Promise; + textContent?(): Promise; + innerText?(): Promise; +} + +/** Minimal Playwright-shaped page the selection step needs — the real `Page` satisfies this. */ +export interface GeminiAutomationPage { + waitForSelector( + selector: string, + opts?: { timeout?: number } + ): Promise; +} + +export interface GeminiModeSelectionResult { + confirmed: boolean; + reason?: GeminiModeSelectionFailureReason; + descriptor?: GeminiWebModeDescriptor; +} + +async function readIndicatorText(el: GeminiAutomationElement): Promise { + try { + if (typeof el.textContent === "function") { + const text = await el.textContent(); + if (typeof text === "string") return text; + } + if (typeof el.innerText === "function") { + return await el.innerText(); + } + } catch { + // Unreadable is treated the same as "did not confirm" below — never a pass. + } + return null; +} + +/** + * Try to switch the live Gemini Web page to `descriptor`'s mode, then READ + * BACK the active-mode indicator before reporting success. A click landing + * is never itself treated as success — only a confirmed, pattern-matching + * indicator text is (#13381). + */ +export async function selectGeminiUiMode( + page: GeminiAutomationPage, + descriptor: GeminiWebModeDescriptor, + timeoutMs = 5000 +): Promise { + if (descriptor.isDefault) return { confirmed: true, descriptor }; + + const { toggleSelector, activeIndicatorSelector, expectedIndicatorPattern } = descriptor; + if (!toggleSelector || !activeIndicatorSelector || !expectedIndicatorPattern) { + return { confirmed: false, reason: "control_not_found", descriptor }; + } + + const toggle = await page + .waitForSelector(toggleSelector, { timeout: timeoutMs }) + .catch(() => null); + if (!toggle) return { confirmed: false, reason: "control_not_found", descriptor }; + + await toggle.click(); + + const indicator = await page + .waitForSelector(activeIndicatorSelector, { timeout: timeoutMs }) + .catch(() => null); + if (!indicator) return { confirmed: false, reason: "indicator_not_found", descriptor }; + + const text = await readIndicatorText(indicator); + if (!text || !expectedIndicatorPattern.test(text)) { + return { confirmed: false, reason: "indicator_mismatch", descriptor }; + } + + return { confirmed: true, descriptor }; +} + +export function resolveGeminiModelMode(modelId: string): GeminiWebModeDescriptor | null { + return GEMINI_WEB_MODEL_MODES[modelId] ?? null; +} + +/** Select + verify the requested advertised model's Gemini UI mode. */ +export async function selectGeminiModel( + page: GeminiAutomationPage, + modelId: string, + timeoutMs?: number +): Promise { + const descriptor = resolveGeminiModelMode(modelId); + if (!descriptor) return { confirmed: false, reason: "unknown_model" }; + return selectGeminiUiMode(page, descriptor, timeoutMs); +} + +/** Select + verify the Extended Thinking control (#13381 follow-up). */ +export async function selectGeminiExtendedThinking( + page: GeminiAutomationPage, + timeoutMs?: number +): Promise { + return selectGeminiUiMode(page, GEMINI_WEB_EXTENDED_THINKING_MODE, timeoutMs); +} + +const REASON_DETAIL: Record = { + unknown_model: "it is not one of the advertised, selectable gweb models", + control_not_found: "the Gemini UI control used to switch modes was not found", + indicator_not_found: "the active-mode indicator used to confirm the switch was not found", + indicator_mismatch: "the active-mode indicator did not confirm the switch after attempting it", +}; + +/** + * Client-facing explanation for a failed-closed selection, safe to put + * directly in a response body (no selector text, no stack trace). + */ +export function describeModeSelectionFailure( + controlLabel: string, + reason: GeminiModeSelectionFailureReason | undefined +): string { + const detail = (reason && REASON_DETAIL[reason]) || "the switch could not be verified"; + return ( + `Model provider "gemini-web" could not verify that "${controlLabel}" is actually active in ` + + `the Gemini web UI (${detail}). Rather than silently answering under a model/mode label that ` + + 'may not be accurate, the request was rejected with "unsupported_control_for_provider" — ' + + "see #13381." + ); +} diff --git a/tests/unit/gemini-web-capabilities-9356.test.ts b/tests/unit/gemini-web-capabilities-9356.test.ts index 110e455c85..6079464a13 100644 --- a/tests/unit/gemini-web-capabilities-9356.test.ts +++ b/tests/unit/gemini-web-capabilities-9356.test.ts @@ -53,13 +53,15 @@ interface ErrorBodyLike { } /** - * Run the executor with valid-looking credentials. Every case in this suite is - * expected to short-circuit on the capability guard, so Playwright is never - * reached — a test that hangs here means the guard did not fire. + * Run the executor with valid-looking credentials. `tool_choice` cases short- + * circuit on the static capability guard, so Playwright is never reached. + * `reasoning_effort` cases (#13381) now require a mocked browser to reach the + * Extended Thinking selection step — `gemini-3.1-pro` is the default model + * mode (no interaction attempted), so it never interferes with that check. */ async function run(body: Record) { return new GeminiWebExecutor().execute({ - model: "gemini-3.6-flash", + model: "gemini-3.1-pro", body: { messages: [{ role: "user", content: "hi" }], stream: false, ...body }, stream: false, credentials: { apiKey: "__Secure-1PSID=test-cookie" }, @@ -152,20 +154,66 @@ test("#9356 forcing is rejected on its own terms, even with no tools[] array", ( }); // ─── Executor wiring ──────────────────────────────────────────────────────── +// +// #13381 (2026-09-15 owner decision, Option B) changed how `reasoning_effort` +// is enforced at the executor level: eligible Gemini accounts DO expose a +// real Extended Thinking toggle in the web UI (reporter follow-up comment, +// 2026-09-11), so a blanket "we can never do this" — rejecting BEFORE any +// browser is even launched, regardless of account — would be the same +// dishonesty #13381 exists to remove. `reasoning_effort` above "minimal" is +// therefore no longer rejected by the static pre-browser guard below; it is +// now ATTEMPTED via a genuine in-browser Extended Thinking selection step +// (open-sse/executors/gemini-web/modeSelection.ts) and only rejected if that +// attempt cannot confirm the control — see +// tests/unit/issue-13381-gemini-web-model-selection.test.ts for that full +// confirmed/unconfirmed contract. The two tests below are updated to match: +// they still prove the SAME `checkGeminiWebUnsupportedControls` pure-function +// classification above is honored end-to-end, adapted to the fact that +// reaching it now requires a (mocked) browser session. +// +// `tool_choice` forcing is UNCHANGED: no UI control for it exists on any +// account, ever, so it still fails fast before Playwright launches and +// before the credential check. -test("#9356 executor returns 400 for reasoning_effort=high before launching a browser", async () => { - const result = await run({ reasoning_effort: "high" }); +test( + "#9356/#13381 executor returns 400 for reasoning_effort=high once the Extended Thinking " + + "control cannot be confirmed (genuinely attempted via a mocked browser, not a blanket reject)", + async () => { + const playwright = await import("playwright"); + const originalLaunch = playwright.chromium.launch; + playwright.chromium.launch = (async () => ({ + newContext: async () => ({ + addCookies: async () => {}, + newPage: async () => ({ + on: () => {}, + goto: async () => {}, + waitForTimeout: async () => {}, + // No Extended Thinking control exists in this fake page — every + // selector lookup reports "not found", proving the fail-closed path. + waitForSelector: async () => null, + keyboard: { type: async () => {}, insertText: async () => {}, press: async () => {} }, + }), + }), + close: async () => {}, + })) as unknown as typeof originalLaunch; - assert.equal(result.response.status, 400); - const body = (await result.response.json()) as ErrorBodyLike; - assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); - assert.match(body.error.message, /reasoning_effort/); - assert.equal( - body.error.message.includes("at /"), - false, - "error bodies must stay sanitized — no stack traces" - ); -}); + try { + const result = await run({ reasoning_effort: "high" }); + + assert.equal(result.response.status, 400); + const body = (await result.response.json()) as ErrorBodyLike; + assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + assert.match(body.error.message, /Extended Thinking/); + assert.equal( + body.error.message.includes("at /"), + false, + "error bodies must stay sanitized — no stack traces" + ); + } finally { + playwright.chromium.launch = originalLaunch; + } + } +); test("#9356 executor returns 400 for tool_choice=required before launching a browser", async () => { const result = await run({ tools: [GET_WEATHER_TOOL], tool_choice: "required" }); @@ -176,12 +224,18 @@ test("#9356 executor returns 400 for tool_choice=required before launching a bro assert.match(body.error.message, /tool_choice/); }); -test("#9356 the capability guard runs ahead of the credential check", async () => { +test("#9356 the tool_choice capability guard runs ahead of the credential check", async () => { // A request that is BOTH uncredentialed and incompatible must report the - // incompatibility: adding a cookie would not make it work. + // incompatibility: adding a cookie would not make it work. `tool_choice` is + // the control this applies to post-#13381 — it never requires a browser to + // know it cannot be honored (unlike `reasoning_effort`, see above). const result = await new GeminiWebExecutor().execute({ model: "gemini-3.6-flash", - body: { messages: [{ role: "user", content: "hi" }], reasoning_effort: "high" }, + body: { + messages: [{ role: "user", content: "hi" }], + tools: [GET_WEATHER_TOOL], + tool_choice: "required", + }, stream: false, credentials: {}, signal: AbortSignal.timeout(10_000), diff --git a/tests/unit/issue-13381-gemini-web-model-selection.test.ts b/tests/unit/issue-13381-gemini-web-model-selection.test.ts new file mode 100644 index 0000000000..a769cb7609 --- /dev/null +++ b/tests/unit/issue-13381-gemini-web-model-selection.test.ts @@ -0,0 +1,305 @@ +// #13381 — gemini-web advertised model IDs but never selected the requested +// model in the Gemini UI: every alias opened the identical page and ran the +// identical automation, so two structurally different advertised models +// produced byte-identical Playwright traces, and the OpenAI-shaped `model` +// field on the response was a caller-supplied label, not an observed fact. +// +// Owner decision (2026-09-15, Option B): implement real selection. gemini-web +// now drives an in-browser mode-selection step (open-sse/executors/gemini-web/ +// modeSelection.ts) before typing anything: `gemini-3.1-pro` is the mode +// gemini.google.com/app already opens to (no interaction, matches every other +// pre-existing gemini-web test's fake page); every OTHER advertised model +// (and Extended Thinking, the reporter's 2026-09-11 follow-up) is switched to +// and READ BACK before the executor proceeds — a confirmed match is required, +// or the request fails closed with 400 `unsupported_control_for_provider` +// rather than silently running the account default under the requested +// label. The mode-selection selectors themselves are UNVALIDATED (no live +// Gemini account/DOM access from this checkout) — see modeSelection.ts and +// the PR's "Live check" section. This suite proves the STRUCTURE: the +// confirmed path proceeds, the unconfirmed path never does. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { GeminiWebExecutor } = await import("../../open-sse/executors/gemini-web.ts"); +const { GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } = + await import("../../open-sse/executors/gemini-web/capabilities.ts"); +const { GEMINI_WEB_MODEL_MODES, GEMINI_WEB_EXTENDED_THINKING_MODE } = + await import("../../open-sse/executors/gemini-web/modeSelection.ts"); + +type Call = { fn: string; args: unknown[] }; + +interface SelectorScriptEntry { + found: boolean; + text?: string | null; +} + +interface PageScript { + /** Keyed by the EXACT selector string the production descriptor uses. */ + selectors?: Record; +} + +/** + * A configurable fake Playwright whose `waitForSelector` resolves the prompt + * editor unconditionally (every pre-#13381 gemini-web test relies on that) + * and otherwise consults `script.selectors`, keyed by the exact selector + * string — read from the real `GEMINI_WEB_MODEL_MODES` / + * `GEMINI_WEB_EXTENDED_THINKING_MODE` descriptors below, never duplicated as + * a literal, so this suite stays valid across a future selector update. + */ +function makeFakePlaywright(calls: Call[], script: PageScript = {}) { + return { + newContext: async (opts: unknown) => { + calls.push({ fn: "newContext", args: [opts] }); + return { + addCookies: async (cookies: unknown) => { + calls.push({ fn: "addCookies", args: [cookies] }); + }, + cookies: async () => [], + newPage: async () => ({ + on: (event: string) => { + calls.push({ fn: "page.on", args: [event] }); + }, + goto: async (url: string, opts2: unknown) => { + calls.push({ fn: "goto", args: [url, opts2] }); + }, + waitForTimeout: async (ms: number) => { + calls.push({ fn: "waitForTimeout", args: [ms] }); + }, + waitForSelector: async (selector: string, opts2: unknown) => { + calls.push({ fn: "waitForSelector", args: [selector, opts2] }); + if (selector.includes(".ql-editor")) { + return { + click: async () => { + calls.push({ fn: "editor.click", args: [] }); + }, + }; + } + const entry = script.selectors?.[selector]; + if (!entry || !entry.found) return null; + return { + click: async () => { + calls.push({ fn: "mode.click", args: [selector] }); + }, + textContent: async () => entry.text ?? null, + }; + }, + keyboard: { + type: async (text: string, opts2: unknown) => { + calls.push({ fn: "keyboard.type", args: [text, opts2] }); + }, + insertText: async (text: string) => { + calls.push({ fn: "keyboard.insertText", args: [text] }); + }, + press: async (key: string) => { + calls.push({ fn: "keyboard.press", args: [key] }); + }, + }, + }), + }; + }, + close: async () => { + calls.push({ fn: "browser.close", args: [] }); + }, + }; +} + +interface ErrorBodyLike { + error: { message: string; type: string; code: string }; +} + +async function runWithFakePage( + model: string, + script: PageScript, + extra: Record = {} +): Promise<{ calls: Call[]; status: number; body: ErrorBodyLike | Record }> { + const playwright = await import("playwright"); + const originalLaunch = playwright.chromium.launch; + const calls: Call[] = []; + playwright.chromium.launch = (async () => + makeFakePlaywright(calls, script)) as unknown as typeof originalLaunch; + try { + const executor = new GeminiWebExecutor(); + const result = await executor.execute({ + model, + body: { messages: [{ role: "user", content: "What is 2+2?" }], stream: false, ...extra }, + stream: false, + credentials: { apiKey: "__Secure-1PSID=fake" }, + signal: AbortSignal.timeout(5000), + log: null, + }); + const body = (await result.response.json()) as ErrorBodyLike | Record; + return { calls, status: result.response.status, body }; + } finally { + playwright.chromium.launch = originalLaunch; + } +} + +// ─── Regression guard: two different advertised models must diverge ──────── + +test( + "#13381: distinct advertised gemini-web model IDs drive distinguishable Playwright " + + "automation (per-model selection is genuinely attempted)", + async () => { + // gemini-3.1-pro is the mode gemini.google.com/app already opens to — no mode + // switch is attempted, so it reaches the prompt editor and (since this fake page + // never fires the "response" event) ends in 502. + const runA = await runWithFakePage("gemini-3.1-pro", {}); + // gemini-3.7-flash requires a confirmed switch. This fake page's script has no + // entry for its toggle selector, so the control is reported "not found" and the + // request fails closed BEFORE the editor is ever touched. + const runB = await runWithFakePage("gemini-3.7-flash", {}); + + assert.notDeepEqual( + runA.calls, + runB.calls, + "expected the automation trace to DIFFER between two distinct advertised models " + + "(proving model selection is attempted) — an identical trace would mean the " + + "executor still never selects the requested Gemini UI mode" + ); + + assert.equal(runA.status, 502, "the default model proceeds to the (unanswered) prompt"); + assert.equal( + runB.status, + 400, + "an unconfirmed non-default model must fail closed, never silently proceed" + ); + assert.equal((runB.body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + + const gotoCall = runA.calls.find((c) => c.fn === "goto"); + assert.ok(gotoCall); + assert.equal(gotoCall!.args[0], "https://gemini.google.com/app"); + + // The core dishonesty this issue reported: runB must NOT reach the editor at all — + // proceeding under an unconfirmed model label is exactly the bug being fixed. + assert.equal( + runB.calls.some((c) => c.fn === "editor.click"), + false, + "an unconfirmed model must never reach the prompt editor" + ); + } +); + +// ─── (a) Read-back confirms -> proceeds ───────────────────────────────────── + +test("#13381: a confirmed model-mode read-back lets the request proceed to the prompt", async () => { + const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.7-flash"]; + assert.ok(descriptor.toggleSelector && descriptor.activeIndicatorSelector); + + const { status, calls } = await runWithFakePage("gemini-3.7-flash", { + selectors: { + [descriptor.toggleSelector!]: { found: true }, + [descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.7 Flash" }, + }, + }); + + assert.equal( + status, + 502, + "a CONFIRMED mode switch must proceed to the (unanswered-by-the-fake-page) prompt, not 400" + ); + assert.ok( + calls.some((c) => c.fn === "mode.click"), + "the mode toggle must have been clicked" + ); + assert.ok( + calls.some((c) => c.fn === "editor.click"), + "must reach the prompt editor once confirmed" + ); +}); + +// ─── (b) Control missing -> 400, never a silent 200 ───────────────────────── + +test("#13381: a missing model-mode control fails closed with 400, not a silent 200", async () => { + const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", { selectors: {} }); + + assert.notEqual(status, 200); + assert.equal(status, 400); + const err = (body as ErrorBodyLike).error; + assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + assert.match(err.message, /gemini-3\.1-flash-lite/); + assert.equal(err.message.includes("at /"), false, "error bodies must stay sanitized"); +}); + +// ─── (c) Read-back disagrees -> 400, not a silent success ─────────────────── + +test("#13381: a mismatched read-back fails closed with 400, not a silent success", async () => { + const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.1-flash-lite"]; + const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", { + selectors: { + [descriptor.toggleSelector!]: { found: true }, + // Toggle clicked, but the UI actually shows a different mode than requested. + [descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.1 Pro" }, + }, + }); + + assert.notEqual(status, 200); + assert.equal(status, 400); + assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); +}); + +// ─── Unknown model id -> 400, not a silent account-default run ────────────── + +test("#13381: an advertised-but-unmapped model id fails closed instead of running the account default", async () => { + const { status, body } = await runWithFakePage("gemini-4.0-ultra-does-not-exist", {}); + + assert.equal(status, 400); + assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); +}); + +// ─── Extended Thinking (#13381 follow-up, 2026-09-11 reporter comment) ────── + +test("#13381: Extended Thinking is attempted and, once confirmed, the request proceeds", async () => { + const descriptor = GEMINI_WEB_EXTENDED_THINKING_MODE; + const { status, calls } = await runWithFakePage( + "gemini-3.1-pro", + { + selectors: { + [descriptor.toggleSelector!]: { found: true }, + [descriptor.activeIndicatorSelector!]: { found: true, text: "Deep Think enabled" }, + }, + }, + { reasoning_effort: "high" } + ); + + assert.equal(status, 502, "a CONFIRMED Extended Thinking switch must proceed to the prompt"); + assert.ok(calls.some((c) => c.fn === "mode.click")); +}); + +test( + "#13381: Extended Thinking fails closed with 400 when the control cannot be confirmed " + + "(no live account can verify it from this checkout)", + async () => { + const { status, body } = await runWithFakePage( + "gemini-3.1-pro", + { selectors: {} }, + { reasoning_effort: "high" } + ); + + assert.notEqual(status, 200); + assert.equal(status, 400); + const err = (body as ErrorBodyLike).error; + assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE); + assert.match(err.message, /Extended Thinking/); + } +); + +test("#13381: reasoning_effort none/minimal never triggers the Extended Thinking control at all", async () => { + for (const effort of ["none", "minimal"]) { + const { status, calls } = await runWithFakePage( + "gemini-3.1-pro", + {}, + { reasoning_effort: effort } + ); + assert.equal( + status, + 502, + `effort="${effort}" must reach the ordinary (unanswered) prompt path` + ); + assert.equal( + calls.some((c) => c.fn === "mode.click"), + false, + `effort="${effort}" must not attempt any UI control switch` + ); + } +});