From 135b5daaa5a9fa07ad8a36355a1baca1e7d767d7 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 26 Jun 2026 17:11:15 -0300 Subject: [PATCH] fix(translator): forward image tool_result blocks as image_url (#5100) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Base-reds fixed (#5117); image tool_result→image_url. Integrated into release/v3.8.38. --- CHANGELOG.md | 1 + .../translator/request/claude-to-openai.ts | 26 +- .../claude-to-openai-image-toolresult.test.ts | 257 ++++++++++++++++++ 3 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 tests/unit/claude-to-openai-image-toolresult.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 45851bd604..0a6bfadaa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ _In development — bullets added per PR; finalized at release._ - **fix(translator):** preserve client `cache_control` breakpoints when routing Claude-format requests (e.g. Claude Code) to Alibaba DashScope's OpenAI-compatible providers (`alibaba` / `alibaba-cn`). The Claude→OpenAI translation previously stripped the markers from the system and message text blocks, so DashScope's explicit caching never engaged and every request was a cache miss. Cache hints now survive when preservation is requested for caching-capable OpenAI-format providers. (thanks @sacrtap) - **fix(tts):** resolve Gemini TTS models from catalog and add `gemini-3.1-flash-tts-preview` as the new default Vertex TTS model. (thanks @nguyenha935) - **fix(sse): don't cool down a healthy connection on a self-inflicted upstream timeout (504)** — when OmniRoute's own deadline elapses (surfaced as `TimeoutError`/`BodyTimeoutError` → 504), the connection is no longer disabled/failed-over, so a slow-but-healthy provider isn't penalised for our timeout. Genuine upstream 5xx/429 still trigger cooldown; antigravity keeps its own policy. (thanks @costaeder) +- **fix(translator):** forward image `tool_result` blocks as `image_url` instead of stringifying base64. (thanks @alican532) - **fix(sse): robust Anthropic `/v1/messages` streaming — real ping keepalive + client-disconnect guard** — slow first tokens on reasoning models could trip strict clients' idle-read watchdog; the route now keeps the stream warm with a real `event: ping` (Anthropic clients ignore SSE comments) from the very first frame, and a client disconnect (AbortError / controller-closed) no longer counts as a provider failure (no failover/cooldown). (thanks @costaeder) - **fix: preserve model hidden flags (`isHidden`) across model sync** — `replaceCustomModels` pruned the compat-override list to the new custom-model ids, silently wiping the `isHidden` flag of eye-hidden SYNCED models on every periodic sync / import (all hidden models turned back on). The redundant cleanup is removed (per-model removal already handles its own compat cleanup), so eye-hidden models stay hidden across re-sync. (#4389, thanks @herjarsa) - **fix(models): derive model-discovery config from the registry `modelsUrl`** — providers absent from the hardcoded `PROVIDER_MODELS_CONFIG` but carrying a registry `modelsUrl` (e.g. MiniMax) now get an auto-derived Bearer `/v1/models` discovery config, so "discover models" works instead of returning nothing. (thanks @herjarsa) diff --git a/open-sse/translator/request/claude-to-openai.ts b/open-sse/translator/request/claude-to-openai.ts index 770f1d3fd3..e96bf4b7e4 100644 --- a/open-sse/translator/request/claude-to-openai.ts +++ b/open-sse/translator/request/claude-to-openai.ts @@ -420,11 +420,29 @@ function convertClaudeMessage(msg, preserveCacheControl = false) { if (typeof block.content === "string") { resultContent = block.content; } else if (Array.isArray(block.content)) { + // Keep text in the tool message; lift any images out as a following user + // turn (OpenAI `tool` messages can't carry images). Without this, an + // image-only tool_result is JSON.stringify'd → base64 as text, which + // causes "input exceeds the context window" errors in OpenAI-protocol + // upstreams (port of decolua/9router#2123 by alican532). + const textParts: string[] = []; + let hasImage = false; + for (const c of block.content) { + if (c.type === "text") { + textParts.push(c.text); + } else if (c.type === "image" && c.source?.type === "base64") { + parts.push({ + type: "image_url", + image_url: { + url: `data:${c.source.media_type};base64,${c.source.data}`, + }, + }); + hasImage = true; + } + } resultContent = - block.content - .filter((c) => c.type === "text") - .map((c) => c.text) - .join("\n") || JSON.stringify(block.content); + textParts.join("\n") || + (hasImage ? "[tool returned an image; see attached]" : JSON.stringify(block.content)); } else if (block.content) { resultContent = JSON.stringify(block.content); } diff --git a/tests/unit/claude-to-openai-image-toolresult.test.ts b/tests/unit/claude-to-openai-image-toolresult.test.ts new file mode 100644 index 0000000000..8508dbc058 --- /dev/null +++ b/tests/unit/claude-to-openai-image-toolresult.test.ts @@ -0,0 +1,257 @@ +/** + * Tests for fix(translator): forward image tool_result blocks as image_url + * instead of stringifying base64. + * + * Port of decolua/9router PR #2123 (alican532). + * Without this fix, an image-only tool_result is JSON.stringify-d into the + * tool message as a base64 text blob — bloating context and causing + * "input exceeds the context window" errors in OpenAI-protocol upstreams. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { claudeToOpenAIRequest } = await import( + "../../open-sse/translator/request/claude-to-openai.ts" +); + +const FAKE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; +const MEDIA_TYPE = "image/png"; +const EXPECTED_DATA_URI = `data:${MEDIA_TYPE};base64,${FAKE_BASE64}`; + +// --------------------------------------------------------------------------- +// 1. image-only tool_result → image_url in a FOLLOWING user message +// --------------------------------------------------------------------------- +test("image-only tool_result produces image_url in following user turn (not stringified in tool msg)", () => { + const result = claudeToOpenAIRequest( + "gpt-4o", + { + messages: [ + // The assistant called a tool + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-abc", + name: "screenshot", + input: {}, + }, + ], + }, + // The user returned a tool_result containing only an image block + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-abc", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: MEDIA_TYPE, + data: FAKE_BASE64, + }, + }, + ], + }, + ], + }, + ], + }, + false + ); + + const msgs = result.messages as any[]; + + // There must be a tool message + const toolMsg = msgs.find((m) => m.role === "tool"); + assert.ok(toolMsg, "expected a tool message"); + + // The tool message must NOT contain raw base64 text + const toolContent = JSON.stringify(toolMsg.content); + assert.ok( + !toolContent.includes(FAKE_BASE64), + `tool message must not contain raw base64 data; got: ${toolContent.slice(0, 200)}` + ); + + // There must be a following user message with image_url + const userMsg = msgs.find((m) => m.role === "user"); + assert.ok(userMsg, "expected a following user message carrying the image"); + + const userContent: any[] = Array.isArray(userMsg.content) + ? userMsg.content + : [userMsg.content]; + + const imageUrlPart = userContent.find( + (p: any) => p.type === "image_url" && p.image_url?.url === EXPECTED_DATA_URI + ); + assert.ok( + imageUrlPart, + `expected image_url part with data URI in the user message; got: ${JSON.stringify(userContent)}` + ); + + // The tool message should have a placeholder text (not empty) + const toolContentStr = + typeof toolMsg.content === "string" ? toolMsg.content : JSON.stringify(toolMsg.content); + assert.ok(toolContentStr.length > 0, "tool message content should not be empty"); +}); + +// --------------------------------------------------------------------------- +// 2. mixed text+image tool_result → text stays in tool msg; image in user turn +// --------------------------------------------------------------------------- +test("mixed text+image tool_result: text stays in tool message, image appears as image_url in following user turn", () => { + const result = claudeToOpenAIRequest( + "gpt-4o", + { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-xyz", + name: "run_test", + input: {}, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-xyz", + content: [ + { type: "text", text: "Test passed." }, + { + type: "image", + source: { + type: "base64", + media_type: MEDIA_TYPE, + data: FAKE_BASE64, + }, + }, + ], + }, + ], + }, + ], + }, + false + ); + + const msgs = result.messages as any[]; + + const toolMsg = msgs.find((m) => m.role === "tool"); + assert.ok(toolMsg, "expected a tool message"); + assert.equal(toolMsg.content, "Test passed.", "text should remain in the tool message"); + + // base64 must NOT appear in the tool message + assert.ok( + !JSON.stringify(toolMsg.content).includes(FAKE_BASE64), + "tool message must not contain raw base64" + ); + + // Following user message must contain image_url + const userMsg = msgs.find((m) => m.role === "user"); + assert.ok(userMsg, "expected a following user message carrying the image"); + + const userContent: any[] = Array.isArray(userMsg.content) + ? userMsg.content + : [userMsg.content]; + + const imageUrlPart = userContent.find( + (p: any) => p.type === "image_url" && p.image_url?.url === EXPECTED_DATA_URI + ); + assert.ok( + imageUrlPart, + `expected image_url part in the following user message; got: ${JSON.stringify(userContent)}` + ); +}); + +// --------------------------------------------------------------------------- +// 3. text-only tool_result → completely unchanged (regression guard) +// --------------------------------------------------------------------------- +test("text-only tool_result is byte-identical to before the fix", () => { + const result = claudeToOpenAIRequest( + "gpt-4o", + { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-text", + name: "search", + input: { query: "hello" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-text", + content: [{ type: "text", text: "Result: 42" }], + }, + ], + }, + ], + }, + false + ); + + const msgs = result.messages as any[]; + const toolMsg = msgs.find((m) => m.role === "tool"); + assert.ok(toolMsg, "expected a tool message"); + assert.equal(toolMsg.content, "Result: 42"); + // No spurious user message + assert.ok( + !msgs.find((m) => m.role === "user"), + "text-only tool_result should not produce a following user message" + ); +}); + +// --------------------------------------------------------------------------- +// 4. string content tool_result → unchanged (regression guard) +// --------------------------------------------------------------------------- +test("string content tool_result is unchanged", () => { + const result = claudeToOpenAIRequest( + "gpt-4o", + { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool-str", + name: "echo", + input: {}, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool-str", + content: "Simple string result", + }, + ], + }, + ], + }, + false + ); + + const msgs = result.messages as any[]; + const toolMsg = msgs.find((m) => m.role === "tool"); + assert.ok(toolMsg, "expected a tool message"); + assert.equal(toolMsg.content, "Simple string result"); +});