fix(translator): forward image tool_result blocks as image_url (#5100)

Base-reds fixed (#5117); image tool_result→image_url. Integrated into release/v3.8.38.

CHANGELOG.md

@@ -34,6 +34,7 @@ _In development — bullets added per PR; finalized at release._
 - **fix(translator):** preserve client `cache_control` breakpoints when routing Claude-format requests (e.g. Claude Code) to Alibaba DashScope's OpenAI-compatible providers (`alibaba` / `alibaba-cn`). The Claude→OpenAI translation previously stripped the markers from the system and message text blocks, so DashScope's explicit caching never engaged and every request was a cache miss. Cache hints now survive when preservation is requested for caching-capable OpenAI-format providers. (thanks @sacrtap)
 - **fix(tts):** resolve Gemini TTS models from catalog and add `gemini-3.1-flash-tts-preview` as the new default Vertex TTS model. (thanks @nguyenha935)
 - **fix(sse): don't cool down a healthy connection on a self-inflicted upstream timeout (504)** — when OmniRoute's own deadline elapses (surfaced as `TimeoutError`/`BodyTimeoutError` → 504), the connection is no longer disabled/failed-over, so a slow-but-healthy provider isn't penalised for our timeout. Genuine upstream 5xx/429 still trigger cooldown; antigravity keeps its own policy. (thanks @costaeder)
+- **fix(translator):** forward image `tool_result` blocks as `image_url` instead of stringifying base64. (thanks @alican532)
 - **fix(sse): robust Anthropic `/v1/messages` streaming — real ping keepalive + client-disconnect guard** — slow first tokens on reasoning models could trip strict clients' idle-read watchdog; the route now keeps the stream warm with a real `event: ping` (Anthropic clients ignore SSE comments) from the very first frame, and a client disconnect (AbortError / controller-closed) no longer counts as a provider failure (no failover/cooldown). (thanks @costaeder)
 - **fix: preserve model hidden flags (`isHidden`) across model sync** — `replaceCustomModels` pruned the compat-override list to the new custom-model ids, silently wiping the `isHidden` flag of eye-hidden SYNCED models on every periodic sync / import (all hidden models turned back on). The redundant cleanup is removed (per-model removal already handles its own compat cleanup), so eye-hidden models stay hidden across re-sync. (#4389, thanks @herjarsa)
 - **fix(models): derive model-discovery config from the registry `modelsUrl`** — providers absent from the hardcoded `PROVIDER_MODELS_CONFIG` but carrying a registry `modelsUrl` (e.g. MiniMax) now get an auto-derived Bearer `/v1/models` discovery config, so "discover models" works instead of returning nothing. (thanks @herjarsa)

open-sse/translator/request/claude-to-openai.ts

@@ -420,11 +420,29 @@ function convertClaudeMessage(msg, preserveCacheControl = false) {
           if (typeof block.content === "string") {
             resultContent = block.content;
           } else if (Array.isArray(block.content)) {
+            // Keep text in the tool message; lift any images out as a following user
+            // turn (OpenAI `tool` messages can't carry images). Without this, an
+            // image-only tool_result is JSON.stringify'd → base64 as text, which
+            // causes "input exceeds the context window" errors in OpenAI-protocol
+            // upstreams (port of decolua/9router#2123 by alican532).
+            const textParts: string[] = [];
+            let hasImage = false;
+            for (const c of block.content) {
+              if (c.type === "text") {
+                textParts.push(c.text);
+              } else if (c.type === "image" && c.source?.type === "base64") {
+                parts.push({
+                  type: "image_url",
+                  image_url: {
+                    url: `data:${c.source.media_type};base64,${c.source.data}`,
+                  },
+                });
+                hasImage = true;
+              }
+            }
             resultContent =
-              block.content
-                .filter((c) => c.type === "text")
-                .map((c) => c.text)
-                .join("\n") || JSON.stringify(block.content);
+              textParts.join("\n") ||
+              (hasImage ? "[tool returned an image; see attached]" : JSON.stringify(block.content));
           } else if (block.content) {
             resultContent = JSON.stringify(block.content);
           }

tests/unit/claude-to-openai-image-toolresult.test.ts

@@ -0,0 +1,257 @@
+/**
+ * Tests for fix(translator): forward image tool_result blocks as image_url
+ * instead of stringifying base64.
+ *
+ * Port of decolua/9router PR #2123 (alican532).
+ * Without this fix, an image-only tool_result is JSON.stringify-d into the
+ * tool message as a base64 text blob — bloating context and causing
+ * "input exceeds the context window" errors in OpenAI-protocol upstreams.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { claudeToOpenAIRequest } = await import(
+  "../../open-sse/translator/request/claude-to-openai.ts"
+);
+
+const FAKE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
+const MEDIA_TYPE = "image/png";
+const EXPECTED_DATA_URI = `data:${MEDIA_TYPE};base64,${FAKE_BASE64}`;
+
+// ---------------------------------------------------------------------------
+// 1. image-only tool_result → image_url in a FOLLOWING user message
+// ---------------------------------------------------------------------------
+test("image-only tool_result produces image_url in following user turn (not stringified in tool msg)", () => {
+  const result = claudeToOpenAIRequest(
+    "gpt-4o",
+    {
+      messages: [
+        // The assistant called a tool
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "tool_use",
+              id: "tool-abc",
+              name: "screenshot",
+              input: {},
+            },
+          ],
+        },
+        // The user returned a tool_result containing only an image block
+        {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: "tool-abc",
+              content: [
+                {
+                  type: "image",
+                  source: {
+                    type: "base64",
+                    media_type: MEDIA_TYPE,
+                    data: FAKE_BASE64,
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const msgs = result.messages as any[];
+
+  // There must be a tool message
+  const toolMsg = msgs.find((m) => m.role === "tool");
+  assert.ok(toolMsg, "expected a tool message");
+
+  // The tool message must NOT contain raw base64 text
+  const toolContent = JSON.stringify(toolMsg.content);
+  assert.ok(
+    !toolContent.includes(FAKE_BASE64),
+    `tool message must not contain raw base64 data; got: ${toolContent.slice(0, 200)}`
+  );
+
+  // There must be a following user message with image_url
+  const userMsg = msgs.find((m) => m.role === "user");
+  assert.ok(userMsg, "expected a following user message carrying the image");
+
+  const userContent: any[] = Array.isArray(userMsg.content)
+    ? userMsg.content
+    : [userMsg.content];
+
+  const imageUrlPart = userContent.find(
+    (p: any) => p.type === "image_url" && p.image_url?.url === EXPECTED_DATA_URI
+  );
+  assert.ok(
+    imageUrlPart,
+    `expected image_url part with data URI in the user message; got: ${JSON.stringify(userContent)}`
+  );
+
+  // The tool message should have a placeholder text (not empty)
+  const toolContentStr =
+    typeof toolMsg.content === "string" ? toolMsg.content : JSON.stringify(toolMsg.content);
+  assert.ok(toolContentStr.length > 0, "tool message content should not be empty");
+});
+
+// ---------------------------------------------------------------------------
+// 2. mixed text+image tool_result → text stays in tool msg; image in user turn
+// ---------------------------------------------------------------------------
+test("mixed text+image tool_result: text stays in tool message, image appears as image_url in following user turn", () => {
+  const result = claudeToOpenAIRequest(
+    "gpt-4o",
+    {
+      messages: [
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "tool_use",
+              id: "tool-xyz",
+              name: "run_test",
+              input: {},
+            },
+          ],
+        },
+        {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: "tool-xyz",
+              content: [
+                { type: "text", text: "Test passed." },
+                {
+                  type: "image",
+                  source: {
+                    type: "base64",
+                    media_type: MEDIA_TYPE,
+                    data: FAKE_BASE64,
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const msgs = result.messages as any[];
+
+  const toolMsg = msgs.find((m) => m.role === "tool");
+  assert.ok(toolMsg, "expected a tool message");
+  assert.equal(toolMsg.content, "Test passed.", "text should remain in the tool message");
+
+  // base64 must NOT appear in the tool message
+  assert.ok(
+    !JSON.stringify(toolMsg.content).includes(FAKE_BASE64),
+    "tool message must not contain raw base64"
+  );
+
+  // Following user message must contain image_url
+  const userMsg = msgs.find((m) => m.role === "user");
+  assert.ok(userMsg, "expected a following user message carrying the image");
+
+  const userContent: any[] = Array.isArray(userMsg.content)
+    ? userMsg.content
+    : [userMsg.content];
+
+  const imageUrlPart = userContent.find(
+    (p: any) => p.type === "image_url" && p.image_url?.url === EXPECTED_DATA_URI
+  );
+  assert.ok(
+    imageUrlPart,
+    `expected image_url part in the following user message; got: ${JSON.stringify(userContent)}`
+  );
+});
+
+// ---------------------------------------------------------------------------
+// 3. text-only tool_result → completely unchanged (regression guard)
+// ---------------------------------------------------------------------------
+test("text-only tool_result is byte-identical to before the fix", () => {
+  const result = claudeToOpenAIRequest(
+    "gpt-4o",
+    {
+      messages: [
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "tool_use",
+              id: "tool-text",
+              name: "search",
+              input: { query: "hello" },
+            },
+          ],
+        },
+        {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: "tool-text",
+              content: [{ type: "text", text: "Result: 42" }],
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const msgs = result.messages as any[];
+  const toolMsg = msgs.find((m) => m.role === "tool");
+  assert.ok(toolMsg, "expected a tool message");
+  assert.equal(toolMsg.content, "Result: 42");
+  // No spurious user message
+  assert.ok(
+    !msgs.find((m) => m.role === "user"),
+    "text-only tool_result should not produce a following user message"
+  );
+});
+
+// ---------------------------------------------------------------------------
+// 4. string content tool_result → unchanged (regression guard)
+// ---------------------------------------------------------------------------
+test("string content tool_result is unchanged", () => {
+  const result = claudeToOpenAIRequest(
+    "gpt-4o",
+    {
+      messages: [
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "tool_use",
+              id: "tool-str",
+              name: "echo",
+              input: {},
+            },
+          ],
+        },
+        {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: "tool-str",
+              content: "Simple string result",
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const msgs = result.messages as any[];
+  const toolMsg = msgs.find((m) => m.role === "tool");
+  assert.ok(toolMsg, "expected a tool message");
+  assert.equal(toolMsg.content, "Simple string result");
+});