diff --git a/changelog.d/features/7679-chatgpt-web-thinking-tool-emulation.md b/changelog.d/features/7679-chatgpt-web-thinking-tool-emulation.md new file mode 100644 index 0000000000..fcd0ec4a14 --- /dev/null +++ b/changelog.d/features/7679-chatgpt-web-thinking-tool-emulation.md @@ -0,0 +1 @@ +- **feat(chatgpt-web):** harden prompt-emulated tool contract for thinking models (#7679 — thanks @horacecar) diff --git a/open-sse/executors/chatgpt-web.ts b/open-sse/executors/chatgpt-web.ts index 15fc3b7355..c96db39586 100644 --- a/open-sse/executors/chatgpt-web.ts +++ b/open-sse/executors/chatgpt-web.ts @@ -2816,11 +2816,14 @@ export class ChatGptWebExecutor extends BaseExecutor { }; } - // Tool-call emulation (#5240): inject a `` contract when `tools` are - // present; parsed back on the response side. Mirrors qwen-web/perplexity-web. + // Tool-call emulation (#5240, #7679): inject a `` contract when tools + // are present; parsed back on the response side. Hardened for thinking models. + const resolvedModel = resolveChatGptModel(model, body, credentials.providerSpecificData); + const modelSlug = resolvedModel.slug; const { hasTools, requestedTools, effectiveMessages } = prepareToolMessages( (body || {}) as Record, - messages as Array<{ role: string; content: unknown }> + messages as Array<{ role: string; content: unknown }>, + { hardened: isThinkingCapableModel(model, modelSlug) } ); if (!credentials.apiKey) { @@ -2918,12 +2921,9 @@ export class ChatGptWebExecutor extends BaseExecutor { log ); - // 2a''. Resolve model + effort and apply thinking-effort preference for - // thinking-capable models. Dedicated thinking models mirror the browser's - // user-config PATCH; GPT-5.5 Pro sends the effort with the conversation - // body because the Pro standard/extended budget is part of that turn. - const resolvedModel = resolveChatGptModel(model, body, credentials.providerSpecificData); - const modelSlug = resolvedModel.slug; + // 2a''. Apply thinking-effort preference for thinking models. + // Dedicated thinking models mirror the browser's user-config PATCH; + // GPT-5.5 Pro effort is sent with the conversation body. const requestedEffort = resolvedModel.effort; if (requestedEffort && isThinkingCapableModel(model, modelSlug)) { await setUserThinkingEffort( diff --git a/open-sse/translator/webTools.ts b/open-sse/translator/webTools.ts index ecc291ce26..7ae5c7732b 100644 --- a/open-sse/translator/webTools.ts +++ b/open-sse/translator/webTools.ts @@ -356,24 +356,18 @@ export function toArgumentsString(value: unknown): string { } } -/** - * Serialize an OpenAI `tools` array into a system-prompt block that instructs the - * web UI model how to invoke a tool (emit a `{...}` block). Returns an - * empty string when there are no usable tools. - * - * Each invocation generates a per-request nonce that is embedded in the tool format - * instructions. The parser (parseToolCallsFromText) requires this nonce in the model's - * `` JSON to distinguish legitimate tool calls from bare JSON, code-fenced JSON, - * or copy-attacked envelopes (#9343). - */ -export function serializeToolsToPrompt(tools: unknown): string { - if (!Array.isArray(tools) || tools.length === 0) return ""; +export interface SerializeToolOptions { + /** Hardened mode for thinking/reasoning models: repeat the instruction + * both before AND after the tool list, use a more distinctive tag format, + * and explicitly tell the model not to claim tools are unavailable. */ + hardened?: boolean; +} - const nonce = getToolNonce(tools); - if (!nonce) return ""; +// ── Tool list rendering (shared between standard and hardened) ───────────────── +function renderToolList(tools: OpenAIToolDef[]): string[] { const lines: string[] = []; - for (const t of tools as OpenAIToolDef[]) { + for (const t of tools) { const fn = t?.function; if (!fn?.name) continue; const desc = typeof fn.description === "string" && fn.description ? fn.description : ""; @@ -387,9 +381,47 @@ export function serializeToolsToPrompt(tools: unknown): string { `- ${fn.name}${desc ? `: ${desc}` : ""}${params ? `\n parameters: ${params}` : ""}` ); } + return lines; +} +/** + * Serialize an OpenAI `tools` array into a system-prompt block that instructs the + * web UI model how to invoke a tool (emit a `{...}` block). Returns an + * empty string when there are no usable tools. + * + * When `options.hardened` is set (intended for thinking/reasoning models), the + * contract is more emphatic: the `` format example is shown before the tool + * list, an explicit "IMPORTANT" directive is appended after the list, and the + * model is told not to claim tools are unavailable. + */ +export function serializeToolsToPrompt(tools: unknown, options?: SerializeToolOptions): string { + if (!Array.isArray(tools) || tools.length === 0) return ""; + + // #9343: the per-request nonce is mandatory in BOTH modes — the parser rejects + // any JSON without the matching `_nonce` binding. + const nonce = getToolNonce(tools); + if (!nonce) return ""; + + const defs = tools as OpenAIToolDef[]; + const lines = renderToolList(defs); if (lines.length === 0) return ""; + if (options?.hardened) { + return [ + "You have access to the following tools and you MUST use them when appropriate.", + "", + `{"name": "", "arguments": { ... }, "_nonce": "${nonce}"}`, + `Every tool call MUST include the secret binding "_nonce": "${nonce}" exactly as shown.`, + "", + "Available tools:", + ...lines, + "", + "IMPORTANT: You CAN and MUST use these tools. Do NOT say you cannot use tools or that", + "tools are unavailable — you have them and they are ready. If a task requires a tool,", + "call it using the TOOL block format described above.", + ].join("\n"); + } + return [ "You can call tools. To call a tool, reply with a single line containing a block", `with JSON that includes the secret binding "_nonce": "${nonce}":`, @@ -514,13 +546,14 @@ interface ToolPrepResult { */ export function prepareToolMessages( bodyObj: Record, - messages: Array<{ role: string; content: unknown }> + messages: Array<{ role: string; content: unknown }>, + options?: SerializeToolOptions ): ToolPrepResult { const requestedTools = bodyObj.tools; const hasTools = Array.isArray(requestedTools) && requestedTools.length > 0; if (!hasTools) return { hasTools: false, requestedTools, effectiveMessages: messages }; - const toolPrompt = serializeToolsToPrompt(requestedTools); + const toolPrompt = serializeToolsToPrompt(requestedTools, options); return { hasTools: true, requestedTools, diff --git a/tests/unit/chatgpt-web-tools-7679.test.ts b/tests/unit/chatgpt-web-tools-7679.test.ts new file mode 100644 index 0000000000..b7c069979d --- /dev/null +++ b/tests/unit/chatgpt-web-tools-7679.test.ts @@ -0,0 +1,225 @@ +// Hardened tool contract serialization for chatgpt-web thinking models (#7679). +// +// GPT-5.6 Thinking via chatgpt-web ignores the injected `` pseudo-contract +// and replies in prose claiming tools are unavailable. This test covers the +// hardened serialization variant that is more emphatic — repeated instruction +// both before and after the tool list, an explicit "DO NOT" directive, and a +// more distinctive tag format. +// +// The hardened variant is activated by passing `{ hardened: true }` to +// `serializeToolsToPrompt()` or `prepareToolMessages()`, and is used by the +// ChatGPT Web executor when a thinking-capable model is detected. + +import test from "node:test"; +import assert from "node:assert/strict"; + +const { + serializeToolsToPrompt, + prepareToolMessages, + parseToolCallsFromText, +} = await import("../../open-sse/translator/webTools.ts"); + +const WEATHER_TOOL = { + type: "function", + function: { + name: "get_weather", + description: "Get the current weather for a location", + parameters: { + type: "object", + properties: { location: { type: "string" } }, + required: ["location"], + }, + }, +}; + +const SEARCH_TOOL = { + type: "function", + function: { + name: "search_web", + description: "Search the web for current information", + parameters: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, +}; + +const TOOLS = [WEATHER_TOOL, SEARCH_TOOL]; + +// ─── serializeToolsToPrompt — hardened variant ─────────────────────────────── + +test("serializeToolsToPrompt({ hardened: true }) contains 'DO NOT' directive (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS, { hardened: true }); + assert.match(result, /Do NOT say you cannot use tools/); +}); + +test("serializeToolsToPrompt({ hardened: true }) contains 'CAN and MUST' directive (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS, { hardened: true }); + assert.match(result, /CAN and MUST use these tools/); +}); + +test("serializeToolsToPrompt({ hardened: true }) contains tool names from the input (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS, { hardened: true }); + assert.match(result, /get_weather/); + assert.match(result, /search_web/); +}); + +test("serializeToolsToPrompt({ hardened: true }) contains the tag format example (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS, { hardened: true }); + assert.match(result, /\{"name": ""/); +}); + +test("serializeToolsToPrompt({ hardened: true }) contains the post-list instruction block (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS, { hardened: true }); + + // The tool list comes before the post-list instruction. + // Confirm both are present in order: tools list then IMPORTANT. + const toolIdx = result.indexOf("get_weather"); + const importantIdx = result.indexOf("IMPORTANT:"); + assert.ok(toolIdx >= 0, "tool name appears in the output"); + assert.ok(importantIdx >= 0, "IMPORTANT block appears in the output"); + assert.ok( + importantIdx > toolIdx, + "IMPORTANT block appears AFTER the tool list" + ); +}); + +test("serializeToolsToPrompt({ hardened: true }) returns empty string for empty tools (#7679)", () => { + assert.equal(serializeToolsToPrompt([], { hardened: true }), ""); +}); + +test("serializeToolsToPrompt({ hardened: true }) returns empty string for null/undefined tools (#7679)", () => { + assert.equal(serializeToolsToPrompt(null, { hardened: true }), ""); + assert.equal(serializeToolsToPrompt(undefined, { hardened: true }), ""); +}); + +// ─── serializeToolsToPrompt — backward compatibility ───────────────────────── + +test("serializeToolsToPrompt({ hardened: false }) produces same output as no-options (#7679)", () => { + const withFalse = serializeToolsToPrompt(TOOLS, { hardened: false }); + const withDefault = serializeToolsToPrompt(TOOLS); + assert.equal(withFalse, withDefault); +}); + +test("serializeToolsToPrompt() without options uses the standard contract (#7679)", () => { + const result = serializeToolsToPrompt(TOOLS); + assert.doesNotMatch(result, /Do NOT say you cannot use tools/); + assert.doesNotMatch(result, /CAN and MUST use these tools/); + assert.match(result, /You can call tools/); +}); + +// ─── prepareToolMessages — hardened variant ────────────────────────────────── + +test("prepareToolMessages with { hardened: true } prepends system message with hardened content (#7679)", () => { + const body = { tools: TOOLS }; + const messages = [{ role: "user", content: "What is the weather?" }]; + const result = prepareToolMessages(body, messages, { hardened: true }); + + assert.equal(result.hasTools, true); + assert.ok(Array.isArray(result.effectiveMessages)); + assert.equal(result.effectiveMessages.length, 2); + + const sysMsg = result.effectiveMessages[0]; + assert.equal(sysMsg.role, "system"); + assert.match( + String(sysMsg.content), + /Do NOT say you cannot use tools/ + ); + assert.match( + String(sysMsg.content), + /CAN and MUST use these tools/ + ); +}); + +test("prepareToolMessages without options uses standard contract (#7679)", () => { + const body = { tools: TOOLS }; + const messages = [{ role: "user", content: "hi" }]; + const result = prepareToolMessages(body, messages); + + assert.equal(result.hasTools, true); + const sysMsg = result.effectiveMessages[0]; + assert.equal(sysMsg.role, "system"); + assert.match(String(sysMsg.content), /You can call tools/); + assert.doesNotMatch(String(sysMsg.content), /Do NOT say you cannot use tools/); +}); + +test("prepareToolMessages with { hardened: true } and no tools returns hasTools: false (#7679)", () => { + const body = {}; + const messages = [{ role: "user", content: "hi" }]; + const result = prepareToolMessages(body, messages, { hardened: true }); + assert.equal(result.hasTools, false); + assert.equal(result.effectiveMessages.length, 1); +}); + +// ─── parseToolCallsFromText — compatibility with hardened instruction text ─── + +test("parseToolCallsFromText correctly extracts blocks from hardened instruction text (#7679)", () => { + const hardenedPrompt = serializeToolsToPrompt(TOOLS, { hardened: true }); + + const text = [ + hardenedPrompt, + "", + "Let me look up the weather in Tokyo.", + '{"name":"get_weather","arguments":{"location":"Tokyo"}}', + "", + 'And now search the web: {"name":"search_web","arguments":{"query":"latest news 2026"}}', + ].join("\n"); + + const result = parseToolCallsFromText(text, "call", TOOLS); + + assert.ok(result.toolCalls !== null, "tool calls should be parsed"); + assert.equal(result.toolCalls.length, 2, "should find two tool calls"); + + assert.equal(result.toolCalls[0].function.name, "get_weather"); + assert.equal(result.toolCalls[0].type, "function"); + assert.deepEqual(JSON.parse(result.toolCalls[0].function.arguments), { + location: "Tokyo", + }); + + assert.equal(result.toolCalls[1].function.name, "search_web"); + assert.deepEqual(JSON.parse(result.toolCalls[1].function.arguments), { + query: "latest news 2026", + }); + + // Assert the actual tool call blocks are stripped from the content. + // The tool names themselves remain in the content because they appear in the + // prompt's tool list (the "Available tools:" section) — only the `{json}` + // blocks that were parsed as tool calls are stripped. + assert.doesNotMatch(result.content, /\{"name":"get_weather"/); + assert.doesNotMatch(result.content, /\{"name":"search_web"/); + assert.match(result.content, /Let me look up/); + // The tool list in the prompt should still be present + assert.match(result.content, /get_weather/); + assert.match(result.content, /search_web/); +}); + +test("parseToolCallsFromText returns null when hardened text has no tool blocks (#7679)", () => { + const hardenedPrompt = serializeToolsToPrompt(TOOLS, { hardened: true }); + const text = [hardenedPrompt, "", "I don't need any tools for this."].join( + "\n" + ); + + const result = parseToolCallsFromText(text, "call", TOOLS); + + assert.equal(result.toolCalls, null, "no tool calls when no blocks present"); + assert.match(result.content, /I don't need any tools/); +}); + +test("parseToolCallsFromText handles blocks line-boundary crossing in hardened text (#7679)", () => { + // Some thinking models may emit the tool block adjacent to explanatory text + // with no preceding newline + const text = [ + 'I will use the weather tool. {"name":"get_weather","arguments":{"location":"Paris"}}', + "I hope this helps.", + ].join("\n"); + + const result = parseToolCallsFromText(text, "call", TOOLS); + + assert.ok(result.toolCalls !== null); + assert.equal(result.toolCalls.length, 1); + assert.equal(result.toolCalls[0].function.name, "get_weather"); + assert.deepEqual(JSON.parse(result.toolCalls[0].function.arguments), { + location: "Paris", + }); +});