mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-17 20:32:25 +03:00
fix(providers): select and verify the requested gemini-web model/mode before answering (#13381) (#13919)
Root cause: GeminiWebExecutor.execute() opened the identical fixed https://gemini.google.com/app URL and ran the identical Playwright interaction sequence for every advertised gweb/<model> id. `model` was read only AFTER the response was captured, purely to stamp the OpenAI-shaped response — never to influence what was actually clicked/typed, so two different advertised models produced byte-identical automation and the response `model` field was a caller-supplied label, not an observed fact. Fix (owner decision, Option B): a new model -> Gemini UI mode map (open-sse/executors/gemini-web/modeSelection.ts) drives an in-browser selection step before anything is typed — try the mode control, read back the active-mode indicator, and only proceed on a confirmed match. An unconfirmed model, or a requested Extended Thinking control that cannot be confirmed (#13381 follow-up comment), fails closed with 400 unsupported_control_for_provider instead of silently running the account default under the requested label. The selectors involved are UNVALIDATED (no live Gemini account from this checkout) — see the PR's "Selector set is UNVALIDATED" section and the required live smoke. Regression test: tests/unit/issue-13381-gemini-web-model-selection.test.ts
This commit is contained in:
committed by
GitHub
parent
d8ad12f22d
commit
ceafa55824
1
changelog.d/fixes/13381-gemini-web-model-selection.md
Normal file
1
changelog.d/fixes/13381-gemini-web-model-selection.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(providers):** `gemini-web` now attempts to select and verify the requested Gemini UI mode (and Extended Thinking) before answering, and fails closed with a clear 400 instead of silently running the account default under a mismatched model label (#13381) — thanks @formilw
|
||||
@@ -22,11 +22,20 @@ import { buildToolModeResponse } from "./chatgptWebTools.ts";
|
||||
import {
|
||||
checkGeminiWebUnsupportedControls,
|
||||
GEMINI_WEB_UNSUPPORTED_CONTROL_CODE,
|
||||
isForcingToolChoice,
|
||||
requestsThinkingBudget,
|
||||
} from "./gemini-web/capabilities.ts";
|
||||
import {
|
||||
describeModeSelectionFailure,
|
||||
selectGeminiExtendedThinking,
|
||||
selectGeminiModel,
|
||||
} from "./gemini-web/modeSelection.ts";
|
||||
|
||||
// ─── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
const GEMINI_URL = "https://gemini.google.com/app";
|
||||
/** Response-label fallback when no `model` was requested at all (unchanged since pre-#13381). */
|
||||
const DEFAULT_MODEL_ID = "gemini-2.5-pro";
|
||||
|
||||
// Re-exported for backward compatibility: some tests/callers import this classification helper
|
||||
// from gemini-web.ts, its original home (#3516). The implementation now lives in
|
||||
@@ -416,21 +425,28 @@ export class GeminiWebExecutor extends BaseExecutor {
|
||||
const { model, body, stream, credentials, signal, log, onCredentialsRefreshed } = input;
|
||||
const requestBody = body as GeminiRequestBody;
|
||||
|
||||
// #9356: fail fast on controls this provider cannot honor (reasoning_effort
|
||||
// above "minimal", forced tool_choice). Runs before the credential check and
|
||||
// before Playwright launches — the request is unservable no matter which
|
||||
// cookie is used, and answering 200 with ordinary prose made agents believe
|
||||
// their reasoning/tool requirements had been met. See ./gemini-web/capabilities.ts.
|
||||
const violation = checkGeminiWebUnsupportedControls(body as Record<string, unknown>);
|
||||
if (violation) {
|
||||
// #9356: forced tool_choice is a guarantee gemini-web's prompt-emulation shim can
|
||||
// never make — no UI control for it exists at all, on any account, so this still
|
||||
// fails fast before the credential check and before Playwright launches. See
|
||||
// ./gemini-web/capabilities.ts.
|
||||
//
|
||||
// `reasoning_effort` (Extended Thinking) is different (#13381 follow-up comment,
|
||||
// 2026-09-11): eligible Gemini accounts DO expose a real Extended Thinking toggle
|
||||
// in the UI, so a blanket "we can never do this" would be the same dishonesty this
|
||||
// fix exists to remove. It is no longer rejected here — it is attempted, verified
|
||||
// via read-back, and only THEN rejected if it cannot be confirmed. See the in-browser
|
||||
// step below and ./gemini-web/modeSelection.ts.
|
||||
const rawBody = body as Record<string, unknown>;
|
||||
if (isForcingToolChoice(rawBody.tool_choice)) {
|
||||
const violation = checkGeminiWebUnsupportedControls({ tool_choice: rawBody.tool_choice });
|
||||
log?.warn?.(
|
||||
"GEMINI-WEB",
|
||||
`Rejected request: "${violation.param}" is not supported by this provider`
|
||||
`Rejected request: "${violation!.param}" is not supported by this provider`
|
||||
);
|
||||
return {
|
||||
response: new Response(
|
||||
JSON.stringify(
|
||||
buildErrorBody(400, violation.message, null, {
|
||||
buildErrorBody(400, violation!.message, null, {
|
||||
type: "invalid_request_error",
|
||||
code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE,
|
||||
})
|
||||
@@ -442,6 +458,7 @@ export class GeminiWebExecutor extends BaseExecutor {
|
||||
transformedBody: body,
|
||||
};
|
||||
}
|
||||
const wantsExtendedThinking = requestsThinkingBudget(rawBody.reasoning_effort);
|
||||
|
||||
const cookie = resolveGeminiWebCookie(credentials);
|
||||
if (!cookie) {
|
||||
@@ -493,6 +510,11 @@ export class GeminiWebExecutor extends BaseExecutor {
|
||||
};
|
||||
}
|
||||
|
||||
// Resolved up front (#13381) — the pre-fix code only read `model` AFTER the
|
||||
// response was already captured, purely to stamp it on the reply. It now also
|
||||
// drives the in-browser mode-selection step below, before anything is typed.
|
||||
const modelId = model || DEFAULT_MODEL_ID;
|
||||
|
||||
let browser: any = null;
|
||||
let abortBrowser: (() => void) | null = null;
|
||||
try {
|
||||
@@ -549,6 +571,70 @@ export class GeminiWebExecutor extends BaseExecutor {
|
||||
}
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
// #13381 (Option B): verify the requested Gemini UI mode is actually active
|
||||
// BEFORE anything is typed. `gemini-3.1-pro` is the mode gemini.google.com/app
|
||||
// already opens to (no interaction attempted); every other advertised model is
|
||||
// switched to and its active-mode indicator is read back — a confirmed match is
|
||||
// required, or the request is rejected instead of silently running the account
|
||||
// default under the requested model's label. See ./gemini-web/modeSelection.ts
|
||||
// for why the selectors involved are UNVALIDATED and why that is safe here.
|
||||
if (model) {
|
||||
const modelSelection = await selectGeminiModel(page, modelId);
|
||||
if (!modelSelection.confirmed) {
|
||||
log?.warn?.(
|
||||
"GEMINI-WEB",
|
||||
`Rejected request: could not confirm Gemini UI mode for "${modelId}" ` +
|
||||
`(${modelSelection.reason})`
|
||||
);
|
||||
return {
|
||||
response: new Response(
|
||||
JSON.stringify(
|
||||
buildErrorBody(
|
||||
400,
|
||||
describeModeSelectionFailure(modelId, modelSelection.reason),
|
||||
null,
|
||||
{ type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE }
|
||||
)
|
||||
),
|
||||
{ status: 400, headers: { "Content-Type": "application/json" } }
|
||||
),
|
||||
url: GEMINI_URL,
|
||||
headers: {},
|
||||
transformedBody: body,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Extended Thinking (#13381 follow-up, 2026-09-11): same detect-and-verify,
|
||||
// fail-closed pattern as model selection above — attempted only when requested
|
||||
// (`reasoning_effort` above "minimal"), never assumed available.
|
||||
if (wantsExtendedThinking) {
|
||||
const thinkingSelection = await selectGeminiExtendedThinking(page);
|
||||
if (!thinkingSelection.confirmed) {
|
||||
log?.warn?.(
|
||||
"GEMINI-WEB",
|
||||
`Rejected request: could not confirm Extended Thinking is available ` +
|
||||
`(${thinkingSelection.reason})`
|
||||
);
|
||||
return {
|
||||
response: new Response(
|
||||
JSON.stringify(
|
||||
buildErrorBody(
|
||||
400,
|
||||
describeModeSelectionFailure("Extended Thinking", thinkingSelection.reason),
|
||||
null,
|
||||
{ type: "invalid_request_error", code: GEMINI_WEB_UNSUPPORTED_CONTROL_CODE }
|
||||
)
|
||||
),
|
||||
{ status: 400, headers: { "Content-Type": "application/json" } }
|
||||
),
|
||||
url: GEMINI_URL,
|
||||
headers: {},
|
||||
transformedBody: body,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Type and send message
|
||||
const inputEl = await page.waitForSelector(".ql-editor, [contenteditable='true']", {
|
||||
timeout: 10000,
|
||||
@@ -584,8 +670,6 @@ export class GeminiWebExecutor extends BaseExecutor {
|
||||
|
||||
await this.persistRotatedCookies(context, cookie, credentials, onCredentialsRefreshed, log);
|
||||
|
||||
const modelId = model || "gemini-2.5-pro";
|
||||
|
||||
if (hasTools) {
|
||||
const cid = `chatcmpl-gwe-${crypto.randomUUID().slice(0, 12)}`;
|
||||
const created = Math.floor(Date.now() / 1000);
|
||||
|
||||
214
open-sse/executors/gemini-web/modeSelection.ts
Normal file
214
open-sse/executors/gemini-web/modeSelection.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* Per-model Gemini Web UI mode selection + verification (#13381, Option B).
|
||||
*
|
||||
* Before this module, `GeminiWebExecutor.execute()` read the `model` field
|
||||
* only AFTER Gemini had already answered — purely to stamp the OpenAI-shaped
|
||||
* response, never to influence what was actually typed/clicked in the
|
||||
* browser (../gemini-web.ts). Every advertised `gweb/<model>` alias ran the
|
||||
* identical automation, so the response `model` field was a caller-supplied
|
||||
* label, not an observed fact.
|
||||
*
|
||||
* This module gives the executor a real selection step: for each advertised
|
||||
* model (and for the Extended Thinking control, #13381 follow-up comment
|
||||
* 2026-09-11), it tries the corresponding UI control, then READS BACK the
|
||||
* active-mode indicator to confirm the switch actually took effect before
|
||||
* the caller is allowed to proceed. `gemini-3.1-pro` is treated as the mode
|
||||
* `gemini.google.com/app` already opens to (no interaction needed); every
|
||||
* other advertised model requires a confirmed switch.
|
||||
*
|
||||
* ⚠️ SELECTOR SET IS UNVALIDATED. Every CSS selector below is a best-effort
|
||||
* guess at Gemini's current DOM, not a value confirmed against a live
|
||||
* account — this checkout has no live Gemini session and no way to inspect
|
||||
* Google's current markup (which also changes without notice). That is fine
|
||||
* BY DESIGN: `selectGeminiUiMode()` never trusts an unconfirmed guess. If a
|
||||
* selector is wrong — the likely outcome until the mandatory live smoke in
|
||||
* the PR is run and these values are corrected against the real DOM — the
|
||||
* toggle/indicator lookup simply fails, the read-back does not confirm the
|
||||
* requested mode, and the caller gets a clear 400
|
||||
* (`unsupported_control_for_provider`) instead of a response silently
|
||||
* labeled with a model/mode that never actually ran — the exact dishonesty
|
||||
* #13381 reported. Update the selectors only after inspecting the real DOM
|
||||
* on a live account, never from guesswork.
|
||||
*/
|
||||
|
||||
/** Reasons `selectGeminiUiMode`/`selectGeminiModel` can fail closed. */
|
||||
export type GeminiModeSelectionFailureReason =
|
||||
"control_not_found" | "indicator_not_found" | "indicator_mismatch" | "unknown_model";
|
||||
|
||||
export interface GeminiWebModeDescriptor {
|
||||
/** The advertised `gweb/<model>` id, or a synthetic label for non-model controls. */
|
||||
readonly id: string;
|
||||
/**
|
||||
* true: this is the mode `gemini.google.com/app` already shows on load — no
|
||||
* UI interaction is attempted, nothing to confirm.
|
||||
*/
|
||||
readonly isDefault: boolean;
|
||||
/** UNVALIDATED — opens/activates the control for this mode. Ignored when `isDefault`. */
|
||||
readonly toggleSelector?: string;
|
||||
/** UNVALIDATED — element whose text proves which mode is now active. */
|
||||
readonly activeIndicatorSelector?: string;
|
||||
/** Text the active-mode indicator must contain once the switch is confirmed. */
|
||||
readonly expectedIndicatorPattern?: RegExp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Model -> Gemini UI mode map for the three `gweb/<model>` ids advertised by
|
||||
* `open-sse/config/providers/registry/gemini/web/index.ts`. Keep the two
|
||||
* catalogs in sync: an advertised model with no entry here fails closed as
|
||||
* `unknown_model` (see `resolveGeminiModelMode`), which is the intended,
|
||||
* honest behavior for a model this module cannot yet select.
|
||||
*/
|
||||
export const GEMINI_WEB_MODEL_MODES: Readonly<Record<string, GeminiWebModeDescriptor>> = {
|
||||
"gemini-3.1-pro": { id: "gemini-3.1-pro", isDefault: true },
|
||||
"gemini-3.7-flash": {
|
||||
id: "gemini-3.7-flash",
|
||||
isDefault: false,
|
||||
toggleSelector:
|
||||
'[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]',
|
||||
activeIndicatorSelector:
|
||||
'[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]',
|
||||
expectedIndicatorPattern: /\bflash\b(?!.*\blite\b)/i,
|
||||
},
|
||||
"gemini-3.1-flash-lite": {
|
||||
id: "gemini-3.1-flash-lite",
|
||||
isDefault: false,
|
||||
toggleSelector:
|
||||
'[data-test-id="bard-mode-menu-button"], button[aria-haspopup="menu"][aria-label*="model" i]',
|
||||
activeIndicatorSelector:
|
||||
'[data-test-id="bard-mode-menu-button"] .mode-title, [data-test-id="bard-mode-menu-button"]',
|
||||
expectedIndicatorPattern: /flash.*\blite\b/i,
|
||||
},
|
||||
};
|
||||
|
||||
/** The Extended Thinking control (#13381 follow-up, 2026-09-11) — account-dependent, not a model. */
|
||||
export const GEMINI_WEB_EXTENDED_THINKING_MODE: GeminiWebModeDescriptor = {
|
||||
id: "extended-thinking",
|
||||
isDefault: false,
|
||||
toggleSelector:
|
||||
'[data-test-id="deep-think-toggle"], button[aria-label*="extended thinking" i], ' +
|
||||
'button[aria-label*="deep think" i]',
|
||||
activeIndicatorSelector:
|
||||
'[data-test-id="deep-think-toggle"][aria-pressed="true"], [data-test-id="deep-think-toggle"].is-active',
|
||||
expectedIndicatorPattern: /thinking|deep think/i,
|
||||
};
|
||||
|
||||
/** Minimal Playwright-shaped element the read-back needs — real `ElementHandle`s satisfy this. */
|
||||
export interface GeminiAutomationElement {
|
||||
click(): Promise<void>;
|
||||
textContent?(): Promise<string | null>;
|
||||
innerText?(): Promise<string>;
|
||||
}
|
||||
|
||||
/** Minimal Playwright-shaped page the selection step needs — the real `Page` satisfies this. */
|
||||
export interface GeminiAutomationPage {
|
||||
waitForSelector(
|
||||
selector: string,
|
||||
opts?: { timeout?: number }
|
||||
): Promise<GeminiAutomationElement | null>;
|
||||
}
|
||||
|
||||
export interface GeminiModeSelectionResult {
|
||||
confirmed: boolean;
|
||||
reason?: GeminiModeSelectionFailureReason;
|
||||
descriptor?: GeminiWebModeDescriptor;
|
||||
}
|
||||
|
||||
async function readIndicatorText(el: GeminiAutomationElement): Promise<string | null> {
|
||||
try {
|
||||
if (typeof el.textContent === "function") {
|
||||
const text = await el.textContent();
|
||||
if (typeof text === "string") return text;
|
||||
}
|
||||
if (typeof el.innerText === "function") {
|
||||
return await el.innerText();
|
||||
}
|
||||
} catch {
|
||||
// Unreadable is treated the same as "did not confirm" below — never a pass.
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to switch the live Gemini Web page to `descriptor`'s mode, then READ
|
||||
* BACK the active-mode indicator before reporting success. A click landing
|
||||
* is never itself treated as success — only a confirmed, pattern-matching
|
||||
* indicator text is (#13381).
|
||||
*/
|
||||
export async function selectGeminiUiMode(
|
||||
page: GeminiAutomationPage,
|
||||
descriptor: GeminiWebModeDescriptor,
|
||||
timeoutMs = 5000
|
||||
): Promise<GeminiModeSelectionResult> {
|
||||
if (descriptor.isDefault) return { confirmed: true, descriptor };
|
||||
|
||||
const { toggleSelector, activeIndicatorSelector, expectedIndicatorPattern } = descriptor;
|
||||
if (!toggleSelector || !activeIndicatorSelector || !expectedIndicatorPattern) {
|
||||
return { confirmed: false, reason: "control_not_found", descriptor };
|
||||
}
|
||||
|
||||
const toggle = await page
|
||||
.waitForSelector(toggleSelector, { timeout: timeoutMs })
|
||||
.catch(() => null);
|
||||
if (!toggle) return { confirmed: false, reason: "control_not_found", descriptor };
|
||||
|
||||
await toggle.click();
|
||||
|
||||
const indicator = await page
|
||||
.waitForSelector(activeIndicatorSelector, { timeout: timeoutMs })
|
||||
.catch(() => null);
|
||||
if (!indicator) return { confirmed: false, reason: "indicator_not_found", descriptor };
|
||||
|
||||
const text = await readIndicatorText(indicator);
|
||||
if (!text || !expectedIndicatorPattern.test(text)) {
|
||||
return { confirmed: false, reason: "indicator_mismatch", descriptor };
|
||||
}
|
||||
|
||||
return { confirmed: true, descriptor };
|
||||
}
|
||||
|
||||
export function resolveGeminiModelMode(modelId: string): GeminiWebModeDescriptor | null {
|
||||
return GEMINI_WEB_MODEL_MODES[modelId] ?? null;
|
||||
}
|
||||
|
||||
/** Select + verify the requested advertised model's Gemini UI mode. */
|
||||
export async function selectGeminiModel(
|
||||
page: GeminiAutomationPage,
|
||||
modelId: string,
|
||||
timeoutMs?: number
|
||||
): Promise<GeminiModeSelectionResult> {
|
||||
const descriptor = resolveGeminiModelMode(modelId);
|
||||
if (!descriptor) return { confirmed: false, reason: "unknown_model" };
|
||||
return selectGeminiUiMode(page, descriptor, timeoutMs);
|
||||
}
|
||||
|
||||
/** Select + verify the Extended Thinking control (#13381 follow-up). */
|
||||
export async function selectGeminiExtendedThinking(
|
||||
page: GeminiAutomationPage,
|
||||
timeoutMs?: number
|
||||
): Promise<GeminiModeSelectionResult> {
|
||||
return selectGeminiUiMode(page, GEMINI_WEB_EXTENDED_THINKING_MODE, timeoutMs);
|
||||
}
|
||||
|
||||
const REASON_DETAIL: Record<GeminiModeSelectionFailureReason, string> = {
|
||||
unknown_model: "it is not one of the advertised, selectable gweb models",
|
||||
control_not_found: "the Gemini UI control used to switch modes was not found",
|
||||
indicator_not_found: "the active-mode indicator used to confirm the switch was not found",
|
||||
indicator_mismatch: "the active-mode indicator did not confirm the switch after attempting it",
|
||||
};
|
||||
|
||||
/**
|
||||
* Client-facing explanation for a failed-closed selection, safe to put
|
||||
* directly in a response body (no selector text, no stack trace).
|
||||
*/
|
||||
export function describeModeSelectionFailure(
|
||||
controlLabel: string,
|
||||
reason: GeminiModeSelectionFailureReason | undefined
|
||||
): string {
|
||||
const detail = (reason && REASON_DETAIL[reason]) || "the switch could not be verified";
|
||||
return (
|
||||
`Model provider "gemini-web" could not verify that "${controlLabel}" is actually active in ` +
|
||||
`the Gemini web UI (${detail}). Rather than silently answering under a model/mode label that ` +
|
||||
'may not be accurate, the request was rejected with "unsupported_control_for_provider" — ' +
|
||||
"see #13381."
|
||||
);
|
||||
}
|
||||
@@ -53,13 +53,15 @@ interface ErrorBodyLike {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the executor with valid-looking credentials. Every case in this suite is
|
||||
* expected to short-circuit on the capability guard, so Playwright is never
|
||||
* reached — a test that hangs here means the guard did not fire.
|
||||
* Run the executor with valid-looking credentials. `tool_choice` cases short-
|
||||
* circuit on the static capability guard, so Playwright is never reached.
|
||||
* `reasoning_effort` cases (#13381) now require a mocked browser to reach the
|
||||
* Extended Thinking selection step — `gemini-3.1-pro` is the default model
|
||||
* mode (no interaction attempted), so it never interferes with that check.
|
||||
*/
|
||||
async function run(body: Record<string, unknown>) {
|
||||
return new GeminiWebExecutor().execute({
|
||||
model: "gemini-3.6-flash",
|
||||
model: "gemini-3.1-pro",
|
||||
body: { messages: [{ role: "user", content: "hi" }], stream: false, ...body },
|
||||
stream: false,
|
||||
credentials: { apiKey: "__Secure-1PSID=test-cookie" },
|
||||
@@ -152,20 +154,66 @@ test("#9356 forcing is rejected on its own terms, even with no tools[] array", (
|
||||
});
|
||||
|
||||
// ─── Executor wiring ────────────────────────────────────────────────────────
|
||||
//
|
||||
// #13381 (2026-09-15 owner decision, Option B) changed how `reasoning_effort`
|
||||
// is enforced at the executor level: eligible Gemini accounts DO expose a
|
||||
// real Extended Thinking toggle in the web UI (reporter follow-up comment,
|
||||
// 2026-09-11), so a blanket "we can never do this" — rejecting BEFORE any
|
||||
// browser is even launched, regardless of account — would be the same
|
||||
// dishonesty #13381 exists to remove. `reasoning_effort` above "minimal" is
|
||||
// therefore no longer rejected by the static pre-browser guard below; it is
|
||||
// now ATTEMPTED via a genuine in-browser Extended Thinking selection step
|
||||
// (open-sse/executors/gemini-web/modeSelection.ts) and only rejected if that
|
||||
// attempt cannot confirm the control — see
|
||||
// tests/unit/issue-13381-gemini-web-model-selection.test.ts for that full
|
||||
// confirmed/unconfirmed contract. The two tests below are updated to match:
|
||||
// they still prove the SAME `checkGeminiWebUnsupportedControls` pure-function
|
||||
// classification above is honored end-to-end, adapted to the fact that
|
||||
// reaching it now requires a (mocked) browser session.
|
||||
//
|
||||
// `tool_choice` forcing is UNCHANGED: no UI control for it exists on any
|
||||
// account, ever, so it still fails fast before Playwright launches and
|
||||
// before the credential check.
|
||||
|
||||
test("#9356 executor returns 400 for reasoning_effort=high before launching a browser", async () => {
|
||||
const result = await run({ reasoning_effort: "high" });
|
||||
test(
|
||||
"#9356/#13381 executor returns 400 for reasoning_effort=high once the Extended Thinking " +
|
||||
"control cannot be confirmed (genuinely attempted via a mocked browser, not a blanket reject)",
|
||||
async () => {
|
||||
const playwright = await import("playwright");
|
||||
const originalLaunch = playwright.chromium.launch;
|
||||
playwright.chromium.launch = (async () => ({
|
||||
newContext: async () => ({
|
||||
addCookies: async () => {},
|
||||
newPage: async () => ({
|
||||
on: () => {},
|
||||
goto: async () => {},
|
||||
waitForTimeout: async () => {},
|
||||
// No Extended Thinking control exists in this fake page — every
|
||||
// selector lookup reports "not found", proving the fail-closed path.
|
||||
waitForSelector: async () => null,
|
||||
keyboard: { type: async () => {}, insertText: async () => {}, press: async () => {} },
|
||||
}),
|
||||
}),
|
||||
close: async () => {},
|
||||
})) as unknown as typeof originalLaunch;
|
||||
|
||||
assert.equal(result.response.status, 400);
|
||||
const body = (await result.response.json()) as ErrorBodyLike;
|
||||
assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
assert.match(body.error.message, /reasoning_effort/);
|
||||
assert.equal(
|
||||
body.error.message.includes("at /"),
|
||||
false,
|
||||
"error bodies must stay sanitized — no stack traces"
|
||||
);
|
||||
});
|
||||
try {
|
||||
const result = await run({ reasoning_effort: "high" });
|
||||
|
||||
assert.equal(result.response.status, 400);
|
||||
const body = (await result.response.json()) as ErrorBodyLike;
|
||||
assert.equal(body.error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
assert.match(body.error.message, /Extended Thinking/);
|
||||
assert.equal(
|
||||
body.error.message.includes("at /"),
|
||||
false,
|
||||
"error bodies must stay sanitized — no stack traces"
|
||||
);
|
||||
} finally {
|
||||
playwright.chromium.launch = originalLaunch;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
test("#9356 executor returns 400 for tool_choice=required before launching a browser", async () => {
|
||||
const result = await run({ tools: [GET_WEATHER_TOOL], tool_choice: "required" });
|
||||
@@ -176,12 +224,18 @@ test("#9356 executor returns 400 for tool_choice=required before launching a bro
|
||||
assert.match(body.error.message, /tool_choice/);
|
||||
});
|
||||
|
||||
test("#9356 the capability guard runs ahead of the credential check", async () => {
|
||||
test("#9356 the tool_choice capability guard runs ahead of the credential check", async () => {
|
||||
// A request that is BOTH uncredentialed and incompatible must report the
|
||||
// incompatibility: adding a cookie would not make it work.
|
||||
// incompatibility: adding a cookie would not make it work. `tool_choice` is
|
||||
// the control this applies to post-#13381 — it never requires a browser to
|
||||
// know it cannot be honored (unlike `reasoning_effort`, see above).
|
||||
const result = await new GeminiWebExecutor().execute({
|
||||
model: "gemini-3.6-flash",
|
||||
body: { messages: [{ role: "user", content: "hi" }], reasoning_effort: "high" },
|
||||
body: {
|
||||
messages: [{ role: "user", content: "hi" }],
|
||||
tools: [GET_WEATHER_TOOL],
|
||||
tool_choice: "required",
|
||||
},
|
||||
stream: false,
|
||||
credentials: {},
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
|
||||
305
tests/unit/issue-13381-gemini-web-model-selection.test.ts
Normal file
305
tests/unit/issue-13381-gemini-web-model-selection.test.ts
Normal file
@@ -0,0 +1,305 @@
|
||||
// #13381 — gemini-web advertised model IDs but never selected the requested
|
||||
// model in the Gemini UI: every alias opened the identical page and ran the
|
||||
// identical automation, so two structurally different advertised models
|
||||
// produced byte-identical Playwright traces, and the OpenAI-shaped `model`
|
||||
// field on the response was a caller-supplied label, not an observed fact.
|
||||
//
|
||||
// Owner decision (2026-09-15, Option B): implement real selection. gemini-web
|
||||
// now drives an in-browser mode-selection step (open-sse/executors/gemini-web/
|
||||
// modeSelection.ts) before typing anything: `gemini-3.1-pro` is the mode
|
||||
// gemini.google.com/app already opens to (no interaction, matches every other
|
||||
// pre-existing gemini-web test's fake page); every OTHER advertised model
|
||||
// (and Extended Thinking, the reporter's 2026-09-11 follow-up) is switched to
|
||||
// and READ BACK before the executor proceeds — a confirmed match is required,
|
||||
// or the request fails closed with 400 `unsupported_control_for_provider`
|
||||
// rather than silently running the account default under the requested
|
||||
// label. The mode-selection selectors themselves are UNVALIDATED (no live
|
||||
// Gemini account/DOM access from this checkout) — see modeSelection.ts and
|
||||
// the PR's "Live check" section. This suite proves the STRUCTURE: the
|
||||
// confirmed path proceeds, the unconfirmed path never does.
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const { GeminiWebExecutor } = await import("../../open-sse/executors/gemini-web.ts");
|
||||
const { GEMINI_WEB_UNSUPPORTED_CONTROL_CODE } =
|
||||
await import("../../open-sse/executors/gemini-web/capabilities.ts");
|
||||
const { GEMINI_WEB_MODEL_MODES, GEMINI_WEB_EXTENDED_THINKING_MODE } =
|
||||
await import("../../open-sse/executors/gemini-web/modeSelection.ts");
|
||||
|
||||
type Call = { fn: string; args: unknown[] };
|
||||
|
||||
interface SelectorScriptEntry {
|
||||
found: boolean;
|
||||
text?: string | null;
|
||||
}
|
||||
|
||||
interface PageScript {
|
||||
/** Keyed by the EXACT selector string the production descriptor uses. */
|
||||
selectors?: Record<string, SelectorScriptEntry>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A configurable fake Playwright whose `waitForSelector` resolves the prompt
|
||||
* editor unconditionally (every pre-#13381 gemini-web test relies on that)
|
||||
* and otherwise consults `script.selectors`, keyed by the exact selector
|
||||
* string — read from the real `GEMINI_WEB_MODEL_MODES` /
|
||||
* `GEMINI_WEB_EXTENDED_THINKING_MODE` descriptors below, never duplicated as
|
||||
* a literal, so this suite stays valid across a future selector update.
|
||||
*/
|
||||
function makeFakePlaywright(calls: Call[], script: PageScript = {}) {
|
||||
return {
|
||||
newContext: async (opts: unknown) => {
|
||||
calls.push({ fn: "newContext", args: [opts] });
|
||||
return {
|
||||
addCookies: async (cookies: unknown) => {
|
||||
calls.push({ fn: "addCookies", args: [cookies] });
|
||||
},
|
||||
cookies: async () => [],
|
||||
newPage: async () => ({
|
||||
on: (event: string) => {
|
||||
calls.push({ fn: "page.on", args: [event] });
|
||||
},
|
||||
goto: async (url: string, opts2: unknown) => {
|
||||
calls.push({ fn: "goto", args: [url, opts2] });
|
||||
},
|
||||
waitForTimeout: async (ms: number) => {
|
||||
calls.push({ fn: "waitForTimeout", args: [ms] });
|
||||
},
|
||||
waitForSelector: async (selector: string, opts2: unknown) => {
|
||||
calls.push({ fn: "waitForSelector", args: [selector, opts2] });
|
||||
if (selector.includes(".ql-editor")) {
|
||||
return {
|
||||
click: async () => {
|
||||
calls.push({ fn: "editor.click", args: [] });
|
||||
},
|
||||
};
|
||||
}
|
||||
const entry = script.selectors?.[selector];
|
||||
if (!entry || !entry.found) return null;
|
||||
return {
|
||||
click: async () => {
|
||||
calls.push({ fn: "mode.click", args: [selector] });
|
||||
},
|
||||
textContent: async () => entry.text ?? null,
|
||||
};
|
||||
},
|
||||
keyboard: {
|
||||
type: async (text: string, opts2: unknown) => {
|
||||
calls.push({ fn: "keyboard.type", args: [text, opts2] });
|
||||
},
|
||||
insertText: async (text: string) => {
|
||||
calls.push({ fn: "keyboard.insertText", args: [text] });
|
||||
},
|
||||
press: async (key: string) => {
|
||||
calls.push({ fn: "keyboard.press", args: [key] });
|
||||
},
|
||||
},
|
||||
}),
|
||||
};
|
||||
},
|
||||
close: async () => {
|
||||
calls.push({ fn: "browser.close", args: [] });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface ErrorBodyLike {
|
||||
error: { message: string; type: string; code: string };
|
||||
}
|
||||
|
||||
async function runWithFakePage(
|
||||
model: string,
|
||||
script: PageScript,
|
||||
extra: Record<string, unknown> = {}
|
||||
): Promise<{ calls: Call[]; status: number; body: ErrorBodyLike | Record<string, unknown> }> {
|
||||
const playwright = await import("playwright");
|
||||
const originalLaunch = playwright.chromium.launch;
|
||||
const calls: Call[] = [];
|
||||
playwright.chromium.launch = (async () =>
|
||||
makeFakePlaywright(calls, script)) as unknown as typeof originalLaunch;
|
||||
try {
|
||||
const executor = new GeminiWebExecutor();
|
||||
const result = await executor.execute({
|
||||
model,
|
||||
body: { messages: [{ role: "user", content: "What is 2+2?" }], stream: false, ...extra },
|
||||
stream: false,
|
||||
credentials: { apiKey: "__Secure-1PSID=fake" },
|
||||
signal: AbortSignal.timeout(5000),
|
||||
log: null,
|
||||
});
|
||||
const body = (await result.response.json()) as ErrorBodyLike | Record<string, unknown>;
|
||||
return { calls, status: result.response.status, body };
|
||||
} finally {
|
||||
playwright.chromium.launch = originalLaunch;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Regression guard: two different advertised models must diverge ────────
|
||||
|
||||
test(
|
||||
"#13381: distinct advertised gemini-web model IDs drive distinguishable Playwright " +
|
||||
"automation (per-model selection is genuinely attempted)",
|
||||
async () => {
|
||||
// gemini-3.1-pro is the mode gemini.google.com/app already opens to — no mode
|
||||
// switch is attempted, so it reaches the prompt editor and (since this fake page
|
||||
// never fires the "response" event) ends in 502.
|
||||
const runA = await runWithFakePage("gemini-3.1-pro", {});
|
||||
// gemini-3.7-flash requires a confirmed switch. This fake page's script has no
|
||||
// entry for its toggle selector, so the control is reported "not found" and the
|
||||
// request fails closed BEFORE the editor is ever touched.
|
||||
const runB = await runWithFakePage("gemini-3.7-flash", {});
|
||||
|
||||
assert.notDeepEqual(
|
||||
runA.calls,
|
||||
runB.calls,
|
||||
"expected the automation trace to DIFFER between two distinct advertised models " +
|
||||
"(proving model selection is attempted) — an identical trace would mean the " +
|
||||
"executor still never selects the requested Gemini UI mode"
|
||||
);
|
||||
|
||||
assert.equal(runA.status, 502, "the default model proceeds to the (unanswered) prompt");
|
||||
assert.equal(
|
||||
runB.status,
|
||||
400,
|
||||
"an unconfirmed non-default model must fail closed, never silently proceed"
|
||||
);
|
||||
assert.equal((runB.body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
|
||||
const gotoCall = runA.calls.find((c) => c.fn === "goto");
|
||||
assert.ok(gotoCall);
|
||||
assert.equal(gotoCall!.args[0], "https://gemini.google.com/app");
|
||||
|
||||
// The core dishonesty this issue reported: runB must NOT reach the editor at all —
|
||||
// proceeding under an unconfirmed model label is exactly the bug being fixed.
|
||||
assert.equal(
|
||||
runB.calls.some((c) => c.fn === "editor.click"),
|
||||
false,
|
||||
"an unconfirmed model must never reach the prompt editor"
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
// ─── (a) Read-back confirms -> proceeds ─────────────────────────────────────
|
||||
|
||||
test("#13381: a confirmed model-mode read-back lets the request proceed to the prompt", async () => {
|
||||
const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.7-flash"];
|
||||
assert.ok(descriptor.toggleSelector && descriptor.activeIndicatorSelector);
|
||||
|
||||
const { status, calls } = await runWithFakePage("gemini-3.7-flash", {
|
||||
selectors: {
|
||||
[descriptor.toggleSelector!]: { found: true },
|
||||
[descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.7 Flash" },
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
status,
|
||||
502,
|
||||
"a CONFIRMED mode switch must proceed to the (unanswered-by-the-fake-page) prompt, not 400"
|
||||
);
|
||||
assert.ok(
|
||||
calls.some((c) => c.fn === "mode.click"),
|
||||
"the mode toggle must have been clicked"
|
||||
);
|
||||
assert.ok(
|
||||
calls.some((c) => c.fn === "editor.click"),
|
||||
"must reach the prompt editor once confirmed"
|
||||
);
|
||||
});
|
||||
|
||||
// ─── (b) Control missing -> 400, never a silent 200 ─────────────────────────
|
||||
|
||||
test("#13381: a missing model-mode control fails closed with 400, not a silent 200", async () => {
|
||||
const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", { selectors: {} });
|
||||
|
||||
assert.notEqual(status, 200);
|
||||
assert.equal(status, 400);
|
||||
const err = (body as ErrorBodyLike).error;
|
||||
assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
assert.match(err.message, /gemini-3\.1-flash-lite/);
|
||||
assert.equal(err.message.includes("at /"), false, "error bodies must stay sanitized");
|
||||
});
|
||||
|
||||
// ─── (c) Read-back disagrees -> 400, not a silent success ───────────────────
|
||||
|
||||
test("#13381: a mismatched read-back fails closed with 400, not a silent success", async () => {
|
||||
const descriptor = GEMINI_WEB_MODEL_MODES["gemini-3.1-flash-lite"];
|
||||
const { status, body } = await runWithFakePage("gemini-3.1-flash-lite", {
|
||||
selectors: {
|
||||
[descriptor.toggleSelector!]: { found: true },
|
||||
// Toggle clicked, but the UI actually shows a different mode than requested.
|
||||
[descriptor.activeIndicatorSelector!]: { found: true, text: "Gemini 3.1 Pro" },
|
||||
},
|
||||
});
|
||||
|
||||
assert.notEqual(status, 200);
|
||||
assert.equal(status, 400);
|
||||
assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
});
|
||||
|
||||
// ─── Unknown model id -> 400, not a silent account-default run ──────────────
|
||||
|
||||
test("#13381: an advertised-but-unmapped model id fails closed instead of running the account default", async () => {
|
||||
const { status, body } = await runWithFakePage("gemini-4.0-ultra-does-not-exist", {});
|
||||
|
||||
assert.equal(status, 400);
|
||||
assert.equal((body as ErrorBodyLike).error.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
});
|
||||
|
||||
// ─── Extended Thinking (#13381 follow-up, 2026-09-11 reporter comment) ──────
|
||||
|
||||
test("#13381: Extended Thinking is attempted and, once confirmed, the request proceeds", async () => {
|
||||
const descriptor = GEMINI_WEB_EXTENDED_THINKING_MODE;
|
||||
const { status, calls } = await runWithFakePage(
|
||||
"gemini-3.1-pro",
|
||||
{
|
||||
selectors: {
|
||||
[descriptor.toggleSelector!]: { found: true },
|
||||
[descriptor.activeIndicatorSelector!]: { found: true, text: "Deep Think enabled" },
|
||||
},
|
||||
},
|
||||
{ reasoning_effort: "high" }
|
||||
);
|
||||
|
||||
assert.equal(status, 502, "a CONFIRMED Extended Thinking switch must proceed to the prompt");
|
||||
assert.ok(calls.some((c) => c.fn === "mode.click"));
|
||||
});
|
||||
|
||||
test(
|
||||
"#13381: Extended Thinking fails closed with 400 when the control cannot be confirmed " +
|
||||
"(no live account can verify it from this checkout)",
|
||||
async () => {
|
||||
const { status, body } = await runWithFakePage(
|
||||
"gemini-3.1-pro",
|
||||
{ selectors: {} },
|
||||
{ reasoning_effort: "high" }
|
||||
);
|
||||
|
||||
assert.notEqual(status, 200);
|
||||
assert.equal(status, 400);
|
||||
const err = (body as ErrorBodyLike).error;
|
||||
assert.equal(err.code, GEMINI_WEB_UNSUPPORTED_CONTROL_CODE);
|
||||
assert.match(err.message, /Extended Thinking/);
|
||||
}
|
||||
);
|
||||
|
||||
test("#13381: reasoning_effort none/minimal never triggers the Extended Thinking control at all", async () => {
|
||||
for (const effort of ["none", "minimal"]) {
|
||||
const { status, calls } = await runWithFakePage(
|
||||
"gemini-3.1-pro",
|
||||
{},
|
||||
{ reasoning_effort: effort }
|
||||
);
|
||||
assert.equal(
|
||||
status,
|
||||
502,
|
||||
`effort="${effort}" must reach the ordinary (unanswered) prompt path`
|
||||
);
|
||||
assert.equal(
|
||||
calls.some((c) => c.fn === "mode.click"),
|
||||
false,
|
||||
`effort="${effort}" must not attempt any UI control switch`
|
||||
);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user