fix(providers): select and verify the requested gemini-web model/mode before answering (#13381) (#13919)

Root cause: GeminiWebExecutor.execute() opened the identical fixed
https://gemini.google.com/app URL and ran the identical Playwright
interaction sequence for every advertised gweb/<model> id. `model` was
read only AFTER the response was captured, purely to stamp the
OpenAI-shaped response — never to influence what was actually
clicked/typed, so two different advertised models produced
byte-identical automation and the response `model` field was a
caller-supplied label, not an observed fact.

Fix (owner decision, Option B): a new model -> Gemini UI mode map
(open-sse/executors/gemini-web/modeSelection.ts) drives an in-browser
selection step before anything is typed — try the mode control, read
back the active-mode indicator, and only proceed on a confirmed match.
An unconfirmed model, or a requested Extended Thinking control that
cannot be confirmed (#13381 follow-up comment), fails closed with 400
unsupported_control_for_provider instead of silently running the
account default under the requested label. The selectors involved are
UNVALIDATED (no live Gemini account from this checkout) — see the PR's
"Selector set is UNVALIDATED" section and the required live smoke.

Regression test: tests/unit/issue-13381-gemini-web-model-selection.test.ts
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-09-17 13:06:37 -03:00
committed by GitHub
parent d8ad12f22d
commit ceafa55824
5 changed files with 688 additions and 30 deletions

View File

@@ -0,0 +1 @@
- **fix(providers):** `gemini-web` now attempts to select and verify the requested Gemini UI mode (and Extended Thinking) before answering, and fails closed with a clear 400 instead of silently running the account default under a mismatched model label (#13381) — thanks @formilw

View File

@@ -22,11 +22,20 @@ import { buildToolModeResponse } from "./chatgptWebTools.ts";
import {
checkGeminiWebUnsupportedControls,
GEMINI_WEB_UNSUPPORTED_CONTROL_CODE,
isForcingToolChoice,
requestsThinkingBudget,
} from "./gemini-web/capabilities.ts";
import {
describeModeSelectionFailure,
selectGeminiExtendedThinking,
selectGeminiModel,
} from "./gemini-web/modeSelection.ts";
// ─── Constants ──────────────────────────────────────────────────────────────
const GEMINI_URL = "https://gemini.google.com/app";
/** Response-label fallback when no `model` was requested at all (unchanged since pre-#13381). */
const DEFAULT_MODEL_ID = "gemini-2.5-pro";
// Re-exported for backward compatibility: some tests/callers import this classification helper
// from gemini-web.ts, its original home (#3516). The implementation now lives in
@@ -416,21 +425,28 @@ export class GeminiWebExecutor extends BaseExecutor {
const { model, body, stream, credentials, signal, log, onCredentialsRefreshed } = input;
const requestBody = body as GeminiRequestBody;
// #9356: fail fast on controls this provider cannot honor (reasoning_effort
// above "minimal", forced tool_choice). Runs before the credential check and
// before Playwright launches — the request is unservable no matter which
// cookie is used, and answering 200 with ordinary prose made agents believe
// their reasoning/tool requirements had been met. See ./gemini-web/capabilities.ts.
const violation = checkGeminiWebUnsupportedControls(body as Record<string, unknown>);
if (violation) {
// #9356: forced tool_choice is a guarantee gemini-web's prompt-emulation shim can
// never make — no UI control for it exists at all, on any account, so this still
// fails fast before the credential check and before Playwright launches. See
// ./gemini-web/capabilities.ts.
//
// `reasoning_effort` (Extended Thinking) is different (#13381 follow-up comment,
// 2026-09-11): eligible Gemini accounts DO expose a real Extended Thinking toggle
// in the UI, so a blanket "we can never do this" would be the same dishonesty this
// fix exists to remove. It is no longer rejected here — it is attempted, verified
// via read-back, and only THEN rejected if it cannot be confirmed. See the in-browser
// step below and ./gemini-web/modeSelection.ts.
const rawBody = body as Record<string, unknown>;
if (isForcingToolChoice(rawBody.tool_choice)) {
const violation = checkGeminiWebUnsupportedControls({ tool_choice: rawBody.tool_choice });
log?.warn?.(
"GEMINI-WEB",
`Rejected request: "${violation.param}" is not supported by this provider`
`Rejected request: "${violation!.param}" is not supported by this provider`
);
return {
response: new Response(
JSON.stringify(
buildErrorBody(400, violation.message, null, {
buildErrorBody(400, violation!.message, null, {
type: "invalid_request_error",
code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE,
})
@@ -442,6 +458,7 @@ export class GeminiWebExecutor extends BaseExecutor {
transformedBody: body,
};
}
const wantsExtendedThinking = requestsThinkingBudget(rawBody.reasoning_effort);
const cookie = resolveGeminiWebCookie(credentials);
if (!cookie) {
@@ -493,6 +510,11 @@ export class GeminiWebExecutor extends BaseExecutor {
};
}
// Resolved up front (#13381) — the pre-fix code only read `model` AFTER the
// response was already captured, purely to stamp it on the reply. It now also
// drives the in-browser mode-selection step below, before anything is typed.
const modelId = model || DEFAULT_MODEL_ID;
let browser: any = null;
let abortBrowser: (() => void) | null = null;
try {
@@ -549,6 +571,70 @@ export class GeminiWebExecutor extends BaseExecutor {
}
await page.waitForTimeout(3000);
// #13381 (Option B): verify the requested Gemini UI mode is actually active
// BEFORE anything is typed. `gemini-3.1-pro` is the mode gemini.google.com/app
// already opens to (no interaction attempted); every other advertised model is
// switched to and its active-mode indicator is read back — a confirmed match is
// required, or the request is rejected instead of silently running the account
// default under the requested model's label. See ./gemini-web/modeSelection.ts
// for why the selectors involved are UNVALIDATED and why that is safe here.
if (model) {
const modelSelection = await selectGeminiModel(page, modelId);
if (!modelSelection.confirmed) {
log?.warn?.(
"GEMINI-WEB",
`Rejected request: could not confirm Gemini UI mode for "${modelId}" ` +
`(${modelSelection.reason})`
);
return {
response: new Response(
JSON.stringify(
buildErrorBody(
400,
describeModeSelectionFailure(modelId, modelSelection.reason),
null,
{ type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE }
)
),
{ status: 400, headers: { "Content-Type": "application/json" } }
),
url: GEMINI_URL,
headers: {},
transformedBody: body,
};
}
}
// Extended Thinking (#13381 follow-up, 2026-09-11): same detect-and-verify,
// fail-closed pattern as model selection above — attempted only when requested
// (`reasoning_effort` above "minimal"), never assumed available.
if (wantsExtendedThinking) {
const thinkingSelection = await selectGeminiExtendedThinking(page);
if (!thinkingSelection.confirmed) {
log?.warn?.(
"GEMINI-WEB",
`Rejected request: could not confirm Extended Thinking is available ` +
`(${thinkingSelection.reason})`
);
return {
response: new Response(
JSON.stringify(
buildErrorBody(
400,
describeModeSelectionFailure("Extended Thinking", thinkingSelection.reason),
null,
{ type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE }
)
),
{ status: 400, headers: { "Content-Type": "application/json" } }
),
url: GEMINI_URL,
headers: {},
transformedBody: body,
};
}
}
// Type and send message
const inputEl = await page.waitForSelector(".ql-editor, [contenteditable='true']", {
timeout: 10000,
@@ -584,8 +670,6 @@ export class GeminiWebExecutor extends BaseExecutor {
await this.persistRotatedCookies(context, cookie, credentials, onCredentialsRefreshed, log);
const modelId = model || "gemini-2.5-pro";
if (hasTools) {
const cid = `chatcmpl-gwe-${crypto.randomUUID().slice(0, 12)}`;
const created = Math.floor(Date.now() / 1000);

View File

@@ -0,0 +1,214 @@
/**
* Per-model Gemini Web UI mode selection + verification (#13381, Option B).
*
* Before this module, `GeminiWebExecutor.execute()` read the `model` field
* only AFTER Gemini had already answered — purely to stamp the OpenAI-shaped
* response, never to influence what was actually typed/clicked in the
* browser (../gemini-web.ts). Every advertised `gweb/<model>` alias ran the
* identical automation, so the response `model` field was a caller-supplied
* label, not an observed fact.
*
* This module gives the executor a real selection step: for each advertised
* model (and for the Extended Thinking control, #13381 follow-up comment
* 2026-09-11), it tries the corresponding UI control, then READS BACK the
* active-mode indicator to confirm the switch actually took effect before
* the caller is allowed to proceed. `gemini-3.1-pro` is treated as the mode
* `gemini.google.com/app` already opens to (no interaction needed); every
* other advertised model requires a confirmed switch.
*
* ⚠️ SELECTOR SET IS UNVALIDATED. Every CSS selector below is a best-effort
* guess at Gemini's current DOM, not a value confirmed against a live
* account — this checkout has no live Gemini session and no way to inspect
* Google's current markup (which also changes without notice). That is fine
* BY DESIGN: `selectGeminiUiMode()` never trusts an unconfirmed guess. If a
* selector is wrong — the likely outcome until the mandatory live smoke in
* the PR is run and these values are corrected against the real DOM — the
* toggle/indicator lookup simply fails, the read-back does not confirm the
* requested mode, and the caller gets a clear 400
* (`unsupported_control_for_provider`) instead of a response silently
* labeled with a model/mode that never actually ran — the exact dishonesty
* #13381 reported. Update the selectors only after inspecting the real DOM
* on a live account, never from guesswork.
*/
/** Reasons `selectGeminiUiMode`/`selectGeminiModel` can fail closed. */
export type GeminiModeSelectionFailureReason =
"control_not_found" | "indicator_not_found" | "indicator_mismatch" | "unknown_model";
export interface GeminiWebModeDescriptor {
/** The advertised `gweb/<model>` id, or a synthetic label for non-model controls. */
readonly id: string;
/**
* true: this is the mode `gemini.google.com/app` already shows on load — no
* UI interaction is attempted, nothing to confirm.
*/
readonly isDefault: boolean;
/** UNVALIDATED — opens/activates the control for this mode. Ignored when `isDefault`. */
readonly toggleSelector?: string;
/** UNVALIDATED — element whose text proves which mode is now active. */
readonly activeIndicatorSelector?: string;
/** Text the active-mode indicator must contain once the switch is confirmed. */
readonly expectedIndicatorPattern?: RegExp;
}
/**
* Model -> Gemini UI mode map for the three `gweb/<model>` ids advertised by
* `open-sse/config/providers/registry/gemini/web/index.ts`. Keep the two
* catalogs in sync: an advertised model with no entry here fails closed as
* `unknown_model` (see `resolveGeminiModelMode`), which is the intended,
* honest behavior for a model this module cannot yet select.
*/
export const GEMINI_WEB_MODEL_MODES: Readonly<Record<string, GeminiWebModeDescriptor>> = {
"gemini-3.1-pro": { id: "gemini-3.1-pro", isDefault: true },
"gemini-3.7-flash": {
id: "gemini-3.7-flash",
isDefault: false,
toggleSelector:
'[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]',
activeIndicatorSelector:
'[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]',
expectedIndicatorPattern: /\bflash\b(?!.*\blite\b)/i,
},
"gemini-3.1-flash-lite": {
id: "gemini-3.1-flash-lite",
isDefault: false,
toggleSelector:
'[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]',
activeIndicatorSelector:
'[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]',
expectedIndicatorPattern: /flash.*\blite\b/i,
},
};
/** The Extended Thinking control (#13381 follow-up, 2026-09-11) — account-dependent, not a model. */
export const GEMINI_WEB_EXTENDED_THINKING_MODE: GeminiWebModeDescriptor = {
id: "extended-thinking",
isDefault: false,
toggleSelector:
'[data-test-id="deep-think-toggle"], button[aria-label*="extended thinking" i], ' +
'button[aria-label*="deep think" i]',
activeIndicatorSelector:
'[data-test-id="deep-think-toggle"][aria-pressed="true"], [data-test-id="deep-think-toggle"].is-active',
expectedIndicatorPattern: /thinking|deep think/i,
};
/** Minimal Playwright-shaped element the read-back needs — real `ElementHandle`s satisfy this. */
export interface GeminiAutomationElement {
click(): Promise<void>;
textContent?(): Promise<string | null>;
innerText?(): Promise<string>;
}
/** Minimal Playwright-shaped page the selection step needs — the real `Page` satisfies this. */
export interface GeminiAutomationPage {
waitForSelector(
selector: string,
opts?: { timeout?: number }
): Promise<GeminiAutomationElement | null>;
}
export interface GeminiModeSelectionResult {
confirmed: boolean;
reason?: GeminiModeSelectionFailureReason;
descriptor?: GeminiWebModeDescriptor;
}
async function readIndicatorText(el: GeminiAutomationElement): Promise<string | null> {
try {
if (typeof el.textContent === "function") {
const text = await el.textContent();
if (typeof text === "string") return text;
}
if (typeof el.innerText === "function") {
return await el.innerText();
}
} catch {
// Unreadable is treated the same as "did not confirm" below — never a pass.
}
return null;
}
/**
* Try to switch the live Gemini Web page to `descriptor`'s mode, then READ
* BACK the active-mode indicator before reporting success. A click landing
* is never itself treated as success — only a confirmed, pattern-matching
* indicator text is (#13381).
*/
export async function selectGeminiUiMode(
page: GeminiAutomationPage,
descriptor: GeminiWebModeDescriptor,
timeoutMs = 5000
): Promise<GeminiModeSelectionResult> {
if (descriptor.isDefault) return { confirmed: true, descriptor };
const { toggleSelector, activeIndicatorSelector, expectedIndicatorPattern } = descriptor;
if (!toggleSelector || !activeIndicatorSelector || !expectedIndicatorPattern) {
return { confirmed: false, reason: "control_not_found", descriptor };
}
const toggle = await page
.waitForSelector(toggleSelector, { timeout: timeoutMs })
.catch(() => null);
if (!toggle) return { confirmed: false, reason: "control_not_found", descriptor };
await toggle.click();
const indicator = await page
.waitForSelector(activeIndicatorSelector, { timeout: timeoutMs })
.catch(() => null);
if (!indicator) return { confirmed: false, reason: "indicator_not_found", descriptor };
const text = await readIndicatorText(indicator);
if (!text || !expectedIndicatorPattern.test(text)) {
return { confirmed: false, reason: "indicator_mismatch", descriptor };
}
return { confirmed: true, descriptor };
}
export function resolveGeminiModelMode(modelId: string): GeminiWebModeDescriptor | null {
return GEMINI_WEB_MODEL_MODES[modelId] ?? null;
}
/** Select + verify the requested advertised model's Gemini UI mode. */
export async function selectGeminiModel(
page: GeminiAutomationPage,
modelId: string,
timeoutMs?: number
): Promise<GeminiModeSelectionResult> {
const descriptor = resolveGeminiModelMode(modelId);
if (!descriptor) return { confirmed: false, reason: "unknown_model" };
return selectGeminiUiMode(page, descriptor, timeoutMs);
}
/** Select + verify the Extended Thinking control (#13381 follow-up). */
export async function selectGeminiExtendedThinking(
page: GeminiAutomationPage,
timeoutMs?: number
): Promise<GeminiModeSelectionResult> {
return selectGeminiUiMode(page, GEMINI_WEB_EXTENDED_THINKING_MODE, timeoutMs);
}
const REASON_DETAIL: Record<GeminiModeSelectionFailureReason, string> = {
unknown_model: "it is not one of the advertised, selectable gweb models",
control_not_found: "the Gemini UI control used to switch modes was not found",
indicator_not_found: "the active-mode indicator used to confirm the switch was not found",
indicator_mismatch: "the active-mode indicator did not confirm the switch after attempting it",
};
/**
* Client-facing explanation for a failed-closed selection, safe to put
* directly in a response body (no selector text, no stack trace).
*/
export function describeModeSelectionFailure(
controlLabel: string,
reason: GeminiModeSelectionFailureReason | undefined
): string {
const detail = (reason && REASON_DETAIL[reason]) || "the switch could not be verified";
return (
`Model provider "gemini-web" could not verify that "${controlLabel}" is actually active in ` +
`the Gemini web UI (${detail}). Rather than silently answering under a model/mode label that ` +
'may not be accurate, the request was rejected with "unsupported_control_for_provider" — ' +
"see #13381."
);
}

View File

@@ -53,13 +53,15 @@ interface ErrorBodyLike {
}
/**
* Run the executor with valid-looking credentials. Every case in this suite is
* expected to short-circuit on the capability guard, so Playwright is never
* reached — a test that hangs here means the guard did not fire.
* Run the executor with valid-looking credentials. `tool_choice` cases short-
* circuit on the static capability guard, so Playwright is never reached.
* `reasoning_effort` cases (#13381) now require a mocked browser to reach the
* Extended Thinking selection step — `gemini-3.1-pro` is the default model
* mode (no interaction attempted), so it never interferes with that check.
*/
async function run(body: Record<string, unknown>) {
return new GeminiWebExecutor().execute({
model: "gemini-3.6-flash",
model: "gemini-3.1-pro",
body: { messages: [{ role: "user", content: "hi" }], stream: false, ...body },
stream: false,
credentials: { apiKey: "__Secure-1PSID=test-cookie" },
@@ -152,20 +154,66 @@ test("#9356 forcing is rejected on its own terms, even with no tools[] array", (
});
// ─── Executor wiring ────────────────────────────────────────────────────────
//
// #13381 (2026-09-15 owner decision, Option B) changed how `reasoning_effort`
// is enforced at the executor level: eligible Gemini accounts DO expose a
// real Extended Thinking toggle in the web UI (reporter follow-up comment,
// 2026-09-11), so a blanket "we can never do this" — rejecting BEFORE any
// browser is even launched, regardless of account — would be the same
// dishonesty #13381 exists to remove. `reasoning_effort` above "minimal" is
// therefore no longer rejected by the static pre-browser guard below; it is
// now ATTEMPTED via a genuine in-browser Extended Thinking selection step
// (open-sse/executors/gemini-web/modeSelection.ts) and only rejected if that
// attempt cannot confirm the control — see
// tests/unit/issue-13381-gemini-web-model-selection.test.ts for that full
// confirmed/unconfirmed contract. The two tests below are updated to match:
// they still prove the SAME `checkGeminiWebUnsupportedControls` pure-function
// classification above is honored end-to-end, adapted to the fact that
// reaching it now requires a (mocked) browser session.
//
// `tool_choice` forcing is UNCHANGED: no UI control for it exists on any
// account, ever, so it still fails fast before Playwright launches and
// before the credential check.
test("#9356 executor returns 400 for reasoning_effort=high before launching a browser", async () => {
const result = await run({ reasoning_effort: "high" });
test(
"#9356/#13381 executor returns 400 for reasoning_effort=high once the Extended Thinking " +
"control cannot be confirmed (genuinely attempted via a mocked browser, not a blanket reject)",
async () => {
const playwright = await import("playwright");
const originalLaunch = playwright.chromium.launch;
playwright.chromium.launch = (async () => ({
newContext: async () => ({
addCookies: async () => {},
newPage: async () => ({
on: () => {},
goto: async () => {},
waitForTimeout: async () => {},
// No Extended Thinking control exists in this fake page — every
// selector lookup reports "not found", proving the fail-closed path.
waitForSelector: async () => null,
keyboard: { type: async () => {}, insertText: async () => {}, press: async () => {} },
}),
}),
close: async () => {},
})) as unknown as typeof originalLaunch;
assert.equal(result.response.status, 400);
const body = (await result.response.json()) as ErrorBodyLike;
assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
assert.match(body.error.message, /reasoning_effort/);
assert.equal(
body.error.message.includes("at /"),
false,
"error bodies must stay sanitized — no stack traces"
);
});
try {
const result = await run({ reasoning_effort: "high" });
assert.equal(result.response.status, 400);
const body = (await result.response.json()) as ErrorBodyLike;
assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
assert.match(body.error.message, /Extended Thinking/);
assert.equal(
body.error.message.includes("at /"),
false,
"error bodies must stay sanitized — no stack traces"
);
} finally {
playwright.chromium.launch = originalLaunch;
}
}
);
test("#9356 executor returns 400 for tool_choice=required before launching a browser", async () => {
const result = await run({ tools: [GET_WEATHER_TOOL], tool_choice: "required" });
@@ -176,12 +224,18 @@ test("#9356 executor returns 400 for tool_choice=required before launching a bro
assert.match(body.error.message, /tool_choice/);
});
test("#9356 the capability guard runs ahead of the credential check", async () => {
test("#9356 the tool_choice capability guard runs ahead of the credential check", async () => {
// A request that is BOTH uncredentialed and incompatible must report the
// incompatibility: adding a cookie would not make it work.
// incompatibility: adding a cookie would not make it work. `tool_choice` is
// the control this applies to post-#13381 — it never requires a browser to
// know it cannot be honored (unlike `reasoning_effort`, see above).
const result = await new GeminiWebExecutor().execute({
model: "gemini-3.6-flash",
body: { messages: [{ role: "user", content: "hi" }], reasoning_effort: "high" },
body: {
messages: [{ role: "user", content: "hi" }],
tools: [GET_WEATHER_TOOL],
tool_choice: "required",
},
stream: false,
credentials: {},
signal: AbortSignal.timeout(10_000),

View File

@@ -0,0 +1,305 @@
// #13381 — gemini-web advertised model IDs but never selected the requested
// model in the Gemini UI: every alias opened the identical page and ran the
// identical automation, so two structurally different advertised models
// produced byte-identical Playwright traces, and the OpenAI-shaped `model`
// field on the response was a caller-supplied label, not an observed fact.
//
// Owner decision (2026-09-15, Option B): implement real selection. gemini-web
// now drives an in-browser mode-selection step (open-sse/executors/gemini-web/
// modeSelection.ts) before typing anything: `gemini-3.1-pro` is the mode
// gemini.google.com/app already opens to (no interaction, matches every other
// pre-existing gemini-web test's fake page); every OTHER advertised model
// (and Extended Thinking, the reporter's 2026-09-11 follow-up) is switched to
// and READ BACK before the executor proceeds — a confirmed match is required,
// or the request fails closed with 400 `unsupported_control_for_provider`
// rather than silently running the account default under the requested
// label. The mode-selection selectors themselves are UNVALIDATED (no live
// Gemini account/DOM access from this checkout) — see modeSelection.ts and
// the PR's "Live check" section. This suite proves the STRUCTURE: the
// confirmed path proceeds, the unconfirmed path never does.
import test from "node:test";
import assert from "node:assert/strict";
const { GeminiWebExecutor } = await import("../../open-sse/executors/gemini-web.ts");
const { GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } =
await import("../../open-sse/executors/gemini-web/capabilities.ts");
const { GEMINI_WEB_MODEL_MODES, GEMINI_WEB_EXTENDED_THINKING_MODE } =
await import("../../open-sse/executors/gemini-web/modeSelection.ts");
type Call = { fn: string; args: unknown[] };
interface SelectorScriptEntry {
found: boolean;
text?: string | null;
}
interface PageScript {
/** Keyed by the EXACT selector string the production descriptor uses. */
selectors?: Record<string, SelectorScriptEntry>;
}
/**
* A configurable fake Playwright whose `waitForSelector` resolves the prompt
* editor unconditionally (every pre-#13381 gemini-web test relies on that)
* and otherwise consults `script.selectors`, keyed by the exact selector
* string — read from the real `GEMINI_WEB_MODEL_MODES` /
* `GEMINI_WEB_EXTENDED_THINKING_MODE` descriptors below, never duplicated as
* a literal, so this suite stays valid across a future selector update.
*/
function makeFakePlaywright(calls: Call[], script: PageScript = {}) {
return {
newContext: async (opts: unknown) => {
calls.push({ fn: "newContext", args: [opts] });
return {
addCookies: async (cookies: unknown) => {
calls.push({ fn: "addCookies", args: [cookies] });
},
cookies: async () => [],
newPage: async () => ({
on: (event: string) => {
calls.push({ fn: "page.on", args: [event] });
},
goto: async (url: string, opts2: unknown) => {
calls.push({ fn: "goto", args: [url, opts2] });
},
waitForTimeout: async (ms: number) => {
calls.push({ fn: "waitForTimeout", args: [ms] });
},
waitForSelector: async (selector: string, opts2: unknown) => {
calls.push({ fn: "waitForSelector", args: [selector, opts2] });
if (selector.includes(".ql-editor")) {
return {
click: async () => {
calls.push({ fn: "editor.click", args: [] });
},
};
}
const entry = script.selectors?.[selector];
if (!entry || !entry.found) return null;
return {
click: async () => {
calls.push({ fn: "mode.click", args: [selector] });
},
textContent: async () => entry.text ?? null,
};
},
keyboard: {
type: async (text: string, opts2: unknown) => {
calls.push({ fn: "keyboard.type", args: [text, opts2] });
},
insertText: async (text: string) => {
calls.push({ fn: "keyboard.insertText", args: [text] });
},
press: async (key: string) => {
calls.push({ fn: "keyboard.press", args: [key] });
},
},
}),
};
},
close: async () => {
calls.push({ fn: "browser.close", args: [] });
},
};
}
interface ErrorBodyLike {
error: { message: string; type: string; code: string };
}
async function runWithFakePage(
model: string,
script: PageScript,
extra: Record<string, unknown> = {}
): Promise<{ calls: Call[]; status: number; body: ErrorBodyLike | Record<string, unknown> }> {
const playwright = await import("playwright");
const originalLaunch = playwright.chromium.launch;
const calls: Call[] = [];
playwright.chromium.launch = (async () =>
makeFakePlaywright(calls, script)) as unknown as typeof originalLaunch;
try {
const executor = new GeminiWebExecutor();
const result = await executor.execute({
model,
body: { messages: [{ role: "user", content: "What is 2+2?" }], stream: false, ...extra },
stream: false,
credentials: { apiKey: "__Secure-1PSID=fake" },
signal: AbortSignal.timeout(5000),
log: null,
});
const body = (await result.response.json()) as ErrorBodyLike | Record<string, unknown>;
return { calls, status: result.response.status, body };
} finally {
playwright.chromium.launch = originalLaunch;
}
}
// ─── Regression guard: two different advertised models must diverge ────────
test(
"#13381: distinct advertised gemini-web model IDs drive distinguishable Playwright " +
"automation (per-model selection is genuinely attempted)",
async () => {
// gemini-3.1-pro is the mode gemini.google.com/app already opens to — no mode
// switch is attempted, so it reaches the prompt editor and (since this fake page
// never fires the "response" event) ends in 502.
const runA = await runWithFakePage("gemini-3.1-pro", {});
// gemini-3.7-flash requires a confirmed switch. This fake page's script has no
// entry for its toggle selector, so the control is reported "not found" and the
// request fails closed BEFORE the editor is ever touched.
const runB = await runWithFakePage("gemini-3.7-flash", {});
assert.notDeepEqual(
runA.calls,
runB.calls,
"expected the automation trace to DIFFER between two distinct advertised models " +
"(proving model selection is attempted) — an identical trace would mean the " +
"executor still never selects the requested Gemini UI mode"
);
assert.equal(runA.status, 502, "the default model proceeds to the (unanswered) prompt");
assert.equal(
runB.status,
400,
"an unconfirmed non-default model must fail closed, never silently proceed"
);
assert.equal((runB.body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
const gotoCall = runA.calls.find((c) => c.fn === "goto");
assert.ok(gotoCall);
assert.equal(gotoCall!.args[0], "https://gemini.google.com/app");
// The core dishonesty this issue reported: runB must NOT reach the editor at all —
// proceeding under an unconfirmed model label is exactly the bug being fixed.
assert.equal(
runB.calls.some((c) => c.fn === "editor.click"),
false,
"an unconfirmed model must never reach the prompt editor"
);
}
);
// ─── (a) Read-back confirms -> proceeds ─────────────────────────────────────
test("#13381: a confirmed model-mode read-back lets the request proceed to the prompt", async () => {
const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.7-flash"];
assert.ok(descriptor.toggleSelector && descriptor.activeIndicatorSelector);
const { status, calls } = await runWithFakePage("gemini-3.7-flash", {
selectors: {
[descriptor.toggleSelector!]: { found: true },
[descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.7 Flash" },
},
});
assert.equal(
status,
502,
"a CONFIRMED mode switch must proceed to the (unanswered-by-the-fake-page) prompt, not 400"
);
assert.ok(
calls.some((c) => c.fn === "mode.click"),
"the mode toggle must have been clicked"
);
assert.ok(
calls.some((c) => c.fn === "editor.click"),
"must reach the prompt editor once confirmed"
);
});
// ─── (b) Control missing -> 400, never a silent 200 ─────────────────────────
test("#13381: a missing model-mode control fails closed with 400, not a silent 200", async () => {
const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", { selectors: {} });
assert.notEqual(status, 200);
assert.equal(status, 400);
const err = (body as ErrorBodyLike).error;
assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
assert.match(err.message, /gemini-3\.1-flash-lite/);
assert.equal(err.message.includes("at /"), false, "error bodies must stay sanitized");
});
// ─── (c) Read-back disagrees -> 400, not a silent success ───────────────────
test("#13381: a mismatched read-back fails closed with 400, not a silent success", async () => {
const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.1-flash-lite"];
const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", {
selectors: {
[descriptor.toggleSelector!]: { found: true },
// Toggle clicked, but the UI actually shows a different mode than requested.
[descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.1 Pro" },
},
});
assert.notEqual(status, 200);
assert.equal(status, 400);
assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
});
// ─── Unknown model id -> 400, not a silent account-default run ──────────────
test("#13381: an advertised-but-unmapped model id fails closed instead of running the account default", async () => {
const { status, body } = await runWithFakePage("gemini-4.0-ultra-does-not-exist", {});
assert.equal(status, 400);
assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
});
// ─── Extended Thinking (#13381 follow-up, 2026-09-11 reporter comment) ──────
test("#13381: Extended Thinking is attempted and, once confirmed, the request proceeds", async () => {
const descriptor = GEMINI_WEB_EXTENDED_THINKING_MODE;
const { status, calls } = await runWithFakePage(
"gemini-3.1-pro",
{
selectors: {
[descriptor.toggleSelector!]: { found: true },
[descriptor.activeIndicatorSelector!]: { found: true, text: "Deep Think enabled" },
},
},
{ reasoning_effort: "high" }
);
assert.equal(status, 502, "a CONFIRMED Extended Thinking switch must proceed to the prompt");
assert.ok(calls.some((c) => c.fn === "mode.click"));
});
test(
"#13381: Extended Thinking fails closed with 400 when the control cannot be confirmed " +
"(no live account can verify it from this checkout)",
async () => {
const { status, body } = await runWithFakePage(
"gemini-3.1-pro",
{ selectors: {} },
{ reasoning_effort: "high" }
);
assert.notEqual(status, 200);
assert.equal(status, 400);
const err = (body as ErrorBodyLike).error;
assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
assert.match(err.message, /Extended Thinking/);
}
);
test("#13381: reasoning_effort none/minimal never triggers the Extended Thinking control at all", async () => {
for (const effort of ["none", "minimal"]) {
const { status, calls } = await runWithFakePage(
"gemini-3.1-pro",
{},
{ reasoning_effort: effort }
);
assert.equal(
status,
502,
`effort="${effort}" must reach the ordinary (unanswered) prompt path`
);
assert.equal(
calls.some((c) => c.fn === "mode.click"),
false,
`effort="${effort}" must not attempt any UI control switch`
);
}
});