fix(translator): convert OpenAI tool-result images to Claude image blocks (#9692) (#10791)

Merged via merge-train (release/v3.8.50, batch1 2026-08-20) — static gates (typecheck/file-size/complexity/cognitive/changelog) green on the combined tree; test:unit reds observed in the boarded run were verified pre-existing on the pure release tip (unrelated flake), not caused by this PR. Thanks for the contribution!

changelog.d/fixes/9692-openai-to-claude-tool-images.md

@@ -0,0 +1 @@
+- **fix(translator):** convert OpenAI `image_url` blocks nested in `role: "tool"` / `tool_result` content to Claude `image` source blocks so OpenAI-compatible clients (Kimi Code CLI `ReadMediaFile`, and any other tool that returns media) no longer 400 the next Claude-format upstream turn ([#9692](https://github.com/diegosouzapw/OmniRoute/issues/9692))

open-sse/translator/request/openai-to-claude.ts

@@ -7,10 +7,17 @@ import { sanitizeToolId } from "../helpers/schemaCoercion.ts";
 import { safeParseJSON } from "../helpers/jsonUtil.ts";
 import { applyKimiCodingThinking } from "../helpers/claudeHelper.ts";
 import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../../config/defaultThinkingSignature.ts";
-import { getDefaultThinkingBudget, isAdaptiveThinkingOnly } from "../../../src/shared/constants/modelSpecs.ts";
+import {
+  getDefaultThinkingBudget,
+  isAdaptiveThinkingOnly,
+} from "../../../src/shared/constants/modelSpecs.ts";
 import { fitThinkingToMaxTokens } from "./openai-to-claude/thinkingBudget.ts";
 import { enforceToolResultAdjacency } from "./openai-to-claude/toolResultAdjacency.ts";
 import { sanitizeToolResultId } from "./openai-to-claude/sanitizeToolResultId.ts";
+import {
+  openAiImagePartToClaudeBlock,
+  normalizeToolResultImages,
+} from "./openai-to-claude/imageBlocks.ts";
 
 // Reasoning-effort levels Anthropic accepts on `output_config.effort`. Used to steer
 // adaptive-only Claude models (Opus 4.7+/Fable 5) without ever emitting a manual budget.
@@ -534,9 +541,10 @@ function getContentBlocksFromMessage(
     const sanitizedToolUseId = sanitizeToolResultId(msg.tool_call_id); // #7705
     if (!sanitizedToolUseId) return blocks;
     // T02: Strip empty text blocks from nested tool_result content to avoid Anthropic 400
-    const toolContent = Array.isArray(msg.content)
-      ? stripEmptyTextBlocks(msg.content)
-      : msg.content;
+    // #9692: rewrite OpenAI image_url parts to Claude image blocks (same as user turns)
+    const toolContent = normalizeToolResultImages(
+      Array.isArray(msg.content) ? stripEmptyTextBlocks(msg.content) : msg.content
+    );
     blocks.push({
       type: "tool_result",
       tool_use_id: sanitizedToolUseId,
@@ -555,43 +563,19 @@ function getContentBlocksFromMessage(
           // Skip tool_result with no tool_use_id (would be useless and may cause errors)
           if (!part.tool_use_id) continue;
           // T02: strip empty text blocks from nested content before passing to Anthropic
-          const resultContent = Array.isArray(part.content)
-            ? stripEmptyTextBlocks(part.content)
-            : part.content;
+          // #9692: convert OpenAI image_url nested in tool_result the same way
+          const resultContent = normalizeToolResultImages(
+            Array.isArray(part.content) ? stripEmptyTextBlocks(part.content) : part.content
+          );
           blocks.push({
             type: "tool_result",
             tool_use_id: sanitizeToolId(part.tool_use_id), // #7705
             content: resultContent,
             ...(part.is_error && { is_error: part.is_error }),
           });
-        } else if (part.type === "image_url") {
-          const url = part.image_url.url;
-          const match = url.match(/^data:([^;]+);base64,(.+)$/);
-          if (match) {
-            blocks.push({
-              type: "image",
-              source: { type: "base64", media_type: match[1], data: match[2] },
-            });
-          } else if (typeof url === "string" && url.trim()) {
-            blocks.push({
-              type: "image",
-              source: { type: "url", url },
-            });
-          }
-        } else if (part.type === "image" && part.source) {
-          blocks.push({ type: "image", source: part.source });
-        } else if (part.type === "image" && typeof part.image === "string") {
-          // AI SDK-style image part: { type: "image", image: "data:...;base64,..." } (#1330)
-          const url = part.image;
-          const match = url.match(/^data:([^;]+);base64,(.+)$/);
-          if (match) {
-            blocks.push({
-              type: "image",
-              source: { type: "base64", media_type: match[1], data: match[2] },
-            });
-          } else if (url.trim()) {
-            blocks.push({ type: "image", source: { type: "url", url } });
-          }
+        } else if (part.type === "image_url" || part.type === "image") {
+          const imageBlock = openAiImagePartToClaudeBlock(part);
+          if (imageBlock) blocks.push(imageBlock);
         } else if (part.type === "file" && (part.file?.file_data || part.file?.data)) {
           // OpenAI Chat Completions file block:
           // {type:"file", file:{filename, file_data:"data:<mime>;base64,..."}}.

open-sse/translator/request/openai-to-claude/imageBlocks.ts

@@ -0,0 +1,77 @@
+/**
+ * Convert OpenAI-style image parts (including those nested in tool results)
+ * into Claude Messages `image` blocks. User-message `image_url` already did
+ * this; `role: "tool"` and nested `tool_result` content previously forwarded
+ * the OpenAI shape unchanged, which Anthropic rejects with HTTP 400 (#9692).
+ */
+
+const DATA_URL_RE = /^data:([^;]+);base64,(.+)$/;
+
+type ClaudeImageBlock = {
+  type: "image";
+  source: { type: "base64"; media_type: string; data: string } | { type: "url"; url: string };
+};
+
+export function extractOpenAiImageUrl(imageUrl: unknown): string {
+  if (typeof imageUrl === "string") return imageUrl;
+  if (imageUrl && typeof imageUrl === "object" && !Array.isArray(imageUrl)) {
+    const url = (imageUrl as { url?: unknown }).url;
+    if (typeof url === "string") return url;
+  }
+  return "";
+}
+
+export function urlToClaudeImageBlock(url: string): ClaudeImageBlock | null {
+  if (typeof url !== "string") return null;
+  const trimmed = url.trim();
+  if (!trimmed) return null;
+  const match = trimmed.match(DATA_URL_RE);
+  if (match) {
+    return {
+      type: "image",
+      source: { type: "base64", media_type: match[1], data: match[2] },
+    };
+  }
+  return { type: "image", source: { type: "url", url: trimmed } };
+}
+
+/**
+ * Map one OpenAI / AI-SDK image-shaped part to a Claude image block.
+ * Returns null when the part is not an image (caller should keep it as-is).
+ */
+export function openAiImagePartToClaudeBlock(
+  part: Record<string, unknown>
+): ClaudeImageBlock | null {
+  const type = part.type;
+  if (type === "image_url") {
+    return urlToClaudeImageBlock(extractOpenAiImageUrl(part.image_url));
+  }
+  if (type === "image") {
+    if (part.source && typeof part.source === "object" && !Array.isArray(part.source)) {
+      return { type: "image", source: part.source as ClaudeImageBlock["source"] };
+    }
+    if (typeof part.image === "string") {
+      return urlToClaudeImageBlock(part.image);
+    }
+  }
+  return null;
+}
+
+/**
+ * Walk a tool_result content value and rewrite OpenAI `image_url` (and AI-SDK
+ * `image`) parts to Claude `image` blocks. Nested `tool_result` arrays recurse.
+ * Non-array content (plain strings) is left unchanged.
+ */
+export function normalizeToolResultImages(content: unknown): unknown {
+  if (!Array.isArray(content)) return content;
+  return content.map((block) => {
+    if (!block || typeof block !== "object" || Array.isArray(block)) return block;
+    const rec = block as Record<string, unknown>;
+    const image = openAiImagePartToClaudeBlock(rec);
+    if (image) return image;
+    if (rec.type === "tool_result" && Array.isArray(rec.content)) {
+      return { ...rec, content: normalizeToolResultImages(rec.content) };
+    }
+    return rec;
+  });
+}

tests/unit/openai-to-claude-tool-result-images-9692.test.ts

@@ -0,0 +1,222 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { openaiToClaudeRequest } =
+  await import("../../open-sse/translator/request/openai-to-claude.ts");
+const {
+  extractOpenAiImageUrl,
+  urlToClaudeImageBlock,
+  openAiImagePartToClaudeBlock,
+  normalizeToolResultImages,
+} = await import("../../open-sse/translator/request/openai-to-claude/imageBlocks.ts");
+
+const PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgo=";
+
+function findToolResult(translated: { messages: Array<{ content?: unknown[] }> }) {
+  for (const msg of translated.messages) {
+    if (!Array.isArray(msg.content)) continue;
+    const toolResult = msg.content.find(
+      (block) =>
+        block && typeof block === "object" && (block as { type?: string }).type === "tool_result"
+    );
+    if (toolResult) return toolResult as { type: string; content: unknown };
+  }
+  return null;
+}
+
+test("extractOpenAiImageUrl accepts both {url} objects and bare strings", () => {
+  assert.equal(extractOpenAiImageUrl({ url: PNG_DATA_URL }), PNG_DATA_URL);
+  assert.equal(extractOpenAiImageUrl(PNG_DATA_URL), PNG_DATA_URL);
+  assert.equal(extractOpenAiImageUrl({}), "");
+  assert.equal(extractOpenAiImageUrl(null), "");
+});
+
+test("urlToClaudeImageBlock splits data URLs and keeps http(s) as url sources", () => {
+  assert.deepEqual(urlToClaudeImageBlock(PNG_DATA_URL), {
+    type: "image",
+    source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+  });
+  assert.deepEqual(urlToClaudeImageBlock("https://example.com/shot.png"), {
+    type: "image",
+    source: { type: "url", url: "https://example.com/shot.png" },
+  });
+  assert.equal(urlToClaudeImageBlock("   "), null);
+});
+
+test("openAiImagePartToClaudeBlock maps image_url and AI-SDK image parts", () => {
+  assert.deepEqual(
+    openAiImagePartToClaudeBlock({
+      type: "image_url",
+      image_url: { url: PNG_DATA_URL },
+    }),
+    {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+    }
+  );
+  assert.deepEqual(openAiImagePartToClaudeBlock({ type: "image", image: PNG_DATA_URL }), {
+    type: "image",
+    source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+  });
+  assert.equal(openAiImagePartToClaudeBlock({ type: "text", text: "ok" }), null);
+});
+
+test("normalizeToolResultImages rewrites nested image_url inside tool_result content", () => {
+  const out = normalizeToolResultImages([
+    { type: "text", text: "caption" },
+    { type: "image_url", image_url: { url: PNG_DATA_URL } },
+    {
+      type: "tool_result",
+      content: [{ type: "image_url", image_url: "https://cdn.example/a.png" }],
+    },
+  ]);
+  assert.deepEqual(out, [
+    { type: "text", text: "caption" },
+    {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+    },
+    {
+      type: "tool_result",
+      content: [{ type: "image", source: { type: "url", url: "https://cdn.example/a.png" } }],
+    },
+  ]);
+});
+
+test("#9692: role=tool image_url content becomes a Claude image block, not a raw image_url", () => {
+  const translated = openaiToClaudeRequest(
+    "claude-sonnet-4",
+    {
+      messages: [
+        { role: "user", content: "describe the screenshot" },
+        {
+          role: "assistant",
+          content: "",
+          tool_calls: [
+            {
+              id: "tool_123",
+              type: "function",
+              function: { name: "ReadMediaFile", arguments: '{"path":"shot.png"}' },
+            },
+          ],
+        },
+        {
+          role: "tool",
+          tool_call_id: "tool_123",
+          content: [
+            {
+              type: "image_url",
+              image_url: { url: PNG_DATA_URL },
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const toolResult = findToolResult(translated);
+  assert.ok(toolResult, "expected a translated tool_result");
+  assert.ok(Array.isArray(toolResult.content), "tool_result.content must stay an array");
+  assert.equal(
+    (toolResult.content as Array<{ type?: string }>).some((block) => block.type === "image_url"),
+    false,
+    "OpenAI image_url must not leak into Claude tool_result content"
+  );
+  const image = (toolResult.content as Array<Record<string, unknown>>).find(
+    (block) => block.type === "image"
+  );
+  assert.ok(image, "expected a Claude image block inside tool_result");
+  assert.deepEqual(image.source, {
+    type: "base64",
+    media_type: "image/png",
+    data: "iVBORw0KGgo=",
+  });
+});
+
+test("#9692: mixed text + image_url in a tool result keeps text and converts the image", () => {
+  const translated = openaiToClaudeRequest(
+    "claude-sonnet-4",
+    {
+      messages: [
+        { role: "user", content: "read it" },
+        {
+          role: "assistant",
+          content: "",
+          tool_calls: [
+            {
+              id: "call_1",
+              type: "function",
+              function: { name: "ReadMediaFile", arguments: "{}" },
+            },
+          ],
+        },
+        {
+          role: "tool",
+          tool_call_id: "call_1",
+          content: [
+            { type: "text", text: "file: shot.png" },
+            { type: "image_url", image_url: PNG_DATA_URL },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const toolResult = findToolResult(translated);
+  assert.ok(toolResult);
+  assert.deepEqual(toolResult.content, [
+    { type: "text", text: "file: shot.png" },
+    {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
+    },
+  ]);
+});
+
+test("#9692: nested tool_result image_url on a user message is converted the same way", () => {
+  const translated = openaiToClaudeRequest(
+    "claude-sonnet-4",
+    {
+      messages: [
+        { role: "user", content: "describe" },
+        {
+          role: "assistant",
+          content: "",
+          tool_calls: [
+            {
+              id: "tool_123",
+              type: "function",
+              function: { name: "ReadMediaFile", arguments: "{}" },
+            },
+          ],
+        },
+        {
+          role: "user",
+          content: [
+            {
+              type: "tool_result",
+              tool_use_id: "tool_123",
+              content: [
+                {
+                  type: "image_url",
+                  image_url: { url: PNG_DATA_URL },
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    },
+    false
+  );
+
+  const toolResult = findToolResult(translated);
+  assert.ok(toolResult);
+  const image = (toolResult.content as Array<Record<string, unknown>>).find(
+    (block) => block.type === "image"
+  );
+  assert.ok(image);
+  assert.equal((image.source as { type?: string; media_type?: string }).media_type, "image/png");
+});