From 286fdf8794ce96290753a2dec4cbc9d09a5cb1a9 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sun, 5 Jul 2026 02:29:37 -0300 Subject: [PATCH] fix(sse): surface ChatGPT-web image silent-drop as an accurate error (#6208) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ChatGPT Web generates an image as an image_asset_pointer but the pointer fails to resolve to a downloadable URL (unknown asset scheme, download 403/ expired, oversize), resolveImagePointers returned [] — indistinguishable from 'no image produced' — so the image-generation handler reported the misleading 502 'completed without returning image markdown'. The image genuinely existed upstream; OmniRoute dropped it silently. Fix: the executor flags x_image_resolution_failed when a pointer existed but none resolved (and logs the unresolved asset scheme for follow-up), and the handler surfaces a truthful 'generated but not retrievable' 502 instead of 'no image markdown'. Adds executorFactory DI for unit testing. TDD: tests/unit/chatgpt-web-image-silentdrop.test.ts (red -> green), plus the existing chatgpt-web / image-generation-handler suites stay green. Reported via community triage (mesh escalated backlog). --- open-sse/executors/chatgpt-web.ts | 33 +++++++ .../imageGeneration/providers/chatgptWeb.ts | 16 +++- .../unit/chatgpt-web-image-silentdrop.test.ts | 96 +++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 tests/unit/chatgpt-web-image-silentdrop.test.ts diff --git a/open-sse/executors/chatgpt-web.ts b/open-sse/executors/chatgpt-web.ts index 27085cf3c6..17125d4fcb 100644 --- a/open-sse/executors/chatgpt-web.ts +++ b/open-sse/executors/chatgpt-web.ts @@ -1579,6 +1579,21 @@ type ImageResolver = ( parentMessageId?: string | null ) => Promise; +/** + * True when ChatGPT emitted an image asset pointer (the image WAS generated + * upstream) but none of the pointers could be resolved to a downloadable URL + * — so the assistant text carries no image markdown. Lets callers surface an + * accurate "generated but not retrievable" error instead of the misleading + * "no image was produced". Escalated mesh report: image visible in the ChatGPT + * chat but returned to OmniRoute as a bare "completed without image markdown". + */ +export function detectImageResolutionFailure( + pointerCount: number, + resolvedCount: number +): boolean { + return pointerCount > 0 && resolvedCount === 0; +} + /** Build the final markdown block for a list of resolved image URLs. */ function imageMarkdown(urls: string[]): string { if (urls.length === 0) return ""; @@ -2017,6 +2032,23 @@ async function buildNonStreamingResponse( log, parentCandidateMessageId ); + // The image genuinely exists upstream but no pointer resolved to a URL + // (unknown asset scheme, download 403/expired, oversize). Flag it so the + // image-generation handler can report an accurate "generated but not + // retrievable" error instead of the misleading "no image markdown" 502. + const imageResolutionFailed = detectImageResolutionFailure( + imagePointers?.length ?? 0, + urls.length + ); + if (imageResolutionFailed && log?.warn) { + const schemes = (imagePointers ?? []) + .map((p) => p.pointer.split("://")[0] || p.pointer.slice(0, 24)) + .join(", "); + log.warn( + "CGPT-WEB", + `Image generated upstream but no asset pointer resolved (schemes: ${schemes}) — surfacing as unretrievable` + ); + } fullAnswer += imageMarkdown(urls); const promptTokens = Math.ceil(currentMsg.length / 4); const completionTokens = Math.ceil(fullAnswer.length / 4); @@ -2028,6 +2060,7 @@ async function buildNonStreamingResponse( created, model, system_fingerprint: null, + ...(imageResolutionFailed ? { x_image_resolution_failed: true } : {}), choices: [ { index: 0, diff --git a/open-sse/handlers/imageGeneration/providers/chatgptWeb.ts b/open-sse/handlers/imageGeneration/providers/chatgptWeb.ts index 7faa531f5c..c6b9688b7a 100644 --- a/open-sse/handlers/imageGeneration/providers/chatgptWeb.ts +++ b/open-sse/handlers/imageGeneration/providers/chatgptWeb.ts @@ -43,6 +43,9 @@ export async function handleChatGptWebImageGeneration({ log, signal, clientHeaders, + // Injectable so unit tests can drive the handler without a live ChatGPT + // session; production uses the real executor. + executorFactory = () => new ChatGptWebExecutor(), }) { const startTime = Date.now(); const prompt = typeof body.prompt === "string" ? body.prompt.trim() : ""; @@ -98,7 +101,7 @@ export async function handleChatGptWebImageGeneration({ }; for (let i = 0; i < requestedCount; i++) { - const executor = new ChatGptWebExecutor(); + const executor = executorFactory(); const result = await executor.execute({ model, body: { @@ -124,21 +127,30 @@ export async function handleChatGptWebImageGeneration({ } let content = ""; + let imageResolutionFailed = false; try { const json = JSON.parse(responseText); content = String(json?.choices?.[0]?.message?.content || ""); + imageResolutionFailed = json?.x_image_resolution_failed === true; } catch { content = responseText; } const urls = extractMarkdownImageUrls(content); if (urls.length === 0) { + // Distinguish "image was generated upstream but OmniRoute could not + // retrieve it" (executor flagged the unresolved asset pointer) from + // "no image was produced at all" — the former is our bug/limitation, + // not a failed prompt, so the message must not read as "no image made". + const error = imageResolutionFailed + ? `ChatGPT Web generated an image but OmniRoute could not retrieve it (the image asset could not be downloaded — the URL may have expired or ChatGPT changed its image delivery format). Please retry; if it persists, report it. Assistant text: ${content.slice(0, 200)}` + : `ChatGPT Web completed without returning image markdown: ${content.slice(0, 300)}`; return saveImageErrorResult({ provider, model, status: 502, startTime, - error: `ChatGPT Web completed without returning image markdown: ${content.slice(0, 300)}`, + error, requestBody, }); } diff --git a/tests/unit/chatgpt-web-image-silentdrop.test.ts b/tests/unit/chatgpt-web-image-silentdrop.test.ts new file mode 100644 index 0000000000..1b3c3c7309 --- /dev/null +++ b/tests/unit/chatgpt-web-image-silentdrop.test.ts @@ -0,0 +1,96 @@ +// Regression guard for the escalated mesh-bot report: a user generated an +// image via the ChatGPT Web provider; the image WAS produced upstream but +// OmniRoute returned `502 "ChatGPT Web completed without returning image +// markdown"` — i.e. the silent-drop path where an image_asset_pointer existed +// but resolution failed, and the handler reported it as "no image made". +// +// The fix distinguishes "image generated but not retrievable" (executor sets +// x_image_resolution_failed) from "no image at all", so the 502 is accurate +// and actionable instead of misleading. +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +process.env.DATA_DIR = mkdtempSync(join(tmpdir(), "omniroute-cgptweb-silentdrop-")); + +const { detectImageResolutionFailure } = await import("../../open-sse/executors/chatgpt-web.ts"); +const { handleChatGptWebImageGeneration } = await import( + "../../open-sse/handlers/imageGeneration/providers/chatgptWeb.ts" +); + +function fakeExecutor(jsonBody: object, status = 200) { + return { + execute: async () => ({ + response: new Response(JSON.stringify(jsonBody), { + status, + headers: { "Content-Type": "application/json" }, + }), + }), + }; +} + +const baseArgs = { + model: "gpt-4o", + provider: "chatgpt-web", + body: { prompt: "a kitten" }, + credentials: { apiKey: "sess-cookie" }, + log: null, + signal: null, + clientHeaders: {}, +}; + +test("detectImageResolutionFailure: true only when a pointer existed but none resolved", () => { + assert.equal(detectImageResolutionFailure(1, 0), true); + assert.equal(detectImageResolutionFailure(2, 0), true); + assert.equal(detectImageResolutionFailure(0, 0), false); // no image at all + assert.equal(detectImageResolutionFailure(1, 1), false); // resolved fine +}); + +test("handler surfaces a specific 502 when the image was generated but not retrievable", async () => { + const res = await handleChatGptWebImageGeneration({ + ...baseArgs, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "Here's your image:" } }], + x_image_resolution_failed: true, + }), + }); + assert.equal(res.success, false); + assert.equal(res.status, 502); + // must NOT be the misleading "completed without returning image markdown" + assert.ok( + !/completed without returning image markdown/i.test(res.error), + `expected specific retrieval error, got: ${res.error}` + ); + // must clearly say the image was generated but could not be retrieved + assert.match(res.error, /could not (be )?retriev|generated an image but/i); +}); + +test("handler keeps the generic 502 when no image was generated at all", async () => { + const res = await handleChatGptWebImageGeneration({ + ...baseArgs, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: "I can't create that." } }], + }), + }); + assert.equal(res.success, false); + assert.equal(res.status, 502); + assert.match(res.error, /completed without returning image markdown/i); +}); + +test("handler returns success when the executor produced image markdown", async () => { + const url = "/v1/chatgpt-web/image/abcdef0123456789"; + const res = await handleChatGptWebImageGeneration({ + ...baseArgs, + executorFactory: () => + fakeExecutor({ + choices: [{ message: { role: "assistant", content: `Here you go:\n\n![image](${url})` } }], + }), + }); + assert.equal(res.success, true); + assert.equal(res.data.data.length, 1); + assert.equal(res.data.data[0].url, url); +});