diff --git a/changelog.d/fixes/9692-openai-to-claude-tool-images.md b/changelog.d/fixes/9692-openai-to-claude-tool-images.md new file mode 100644 index 0000000000..c082d0dbc3 --- /dev/null +++ b/changelog.d/fixes/9692-openai-to-claude-tool-images.md @@ -0,0 +1 @@ +- **fix(translator):** convert OpenAI `image_url` blocks nested in `role: "tool"` / `tool_result` content to Claude `image` source blocks so OpenAI-compatible clients (Kimi Code CLI `ReadMediaFile`, and any other tool that returns media) no longer 400 the next Claude-format upstream turn ([#9692](https://github.com/diegosouzapw/OmniRoute/issues/9692)) diff --git a/open-sse/translator/request/openai-to-claude.ts b/open-sse/translator/request/openai-to-claude.ts index ff07029e61..8a5c115c2a 100644 --- a/open-sse/translator/request/openai-to-claude.ts +++ b/open-sse/translator/request/openai-to-claude.ts @@ -7,10 +7,17 @@ import { sanitizeToolId } from "../helpers/schemaCoercion.ts"; import { safeParseJSON } from "../helpers/jsonUtil.ts"; import { applyKimiCodingThinking } from "../helpers/claudeHelper.ts"; import { DEFAULT_THINKING_CLAUDE_SIGNATURE } from "../../config/defaultThinkingSignature.ts"; -import { getDefaultThinkingBudget, isAdaptiveThinkingOnly } from "../../../src/shared/constants/modelSpecs.ts"; +import { + getDefaultThinkingBudget, + isAdaptiveThinkingOnly, +} from "../../../src/shared/constants/modelSpecs.ts"; import { fitThinkingToMaxTokens } from "./openai-to-claude/thinkingBudget.ts"; import { enforceToolResultAdjacency } from "./openai-to-claude/toolResultAdjacency.ts"; import { sanitizeToolResultId } from "./openai-to-claude/sanitizeToolResultId.ts"; +import { + openAiImagePartToClaudeBlock, + normalizeToolResultImages, +} from "./openai-to-claude/imageBlocks.ts"; // Reasoning-effort levels Anthropic accepts on `output_config.effort`. Used to steer // adaptive-only Claude models (Opus 4.7+/Fable 5) without ever emitting a manual budget. @@ -534,9 +541,10 @@ function getContentBlocksFromMessage( const sanitizedToolUseId = sanitizeToolResultId(msg.tool_call_id); // #7705 if (!sanitizedToolUseId) return blocks; // T02: Strip empty text blocks from nested tool_result content to avoid Anthropic 400 - const toolContent = Array.isArray(msg.content) - ? stripEmptyTextBlocks(msg.content) - : msg.content; + // #9692: rewrite OpenAI image_url parts to Claude image blocks (same as user turns) + const toolContent = normalizeToolResultImages( + Array.isArray(msg.content) ? stripEmptyTextBlocks(msg.content) : msg.content + ); blocks.push({ type: "tool_result", tool_use_id: sanitizedToolUseId, @@ -555,43 +563,19 @@ function getContentBlocksFromMessage( // Skip tool_result with no tool_use_id (would be useless and may cause errors) if (!part.tool_use_id) continue; // T02: strip empty text blocks from nested content before passing to Anthropic - const resultContent = Array.isArray(part.content) - ? stripEmptyTextBlocks(part.content) - : part.content; + // #9692: convert OpenAI image_url nested in tool_result the same way + const resultContent = normalizeToolResultImages( + Array.isArray(part.content) ? stripEmptyTextBlocks(part.content) : part.content + ); blocks.push({ type: "tool_result", tool_use_id: sanitizeToolId(part.tool_use_id), // #7705 content: resultContent, ...(part.is_error && { is_error: part.is_error }), }); - } else if (part.type === "image_url") { - const url = part.image_url.url; - const match = url.match(/^data:([^;]+);base64,(.+)$/); - if (match) { - blocks.push({ - type: "image", - source: { type: "base64", media_type: match[1], data: match[2] }, - }); - } else if (typeof url === "string" && url.trim()) { - blocks.push({ - type: "image", - source: { type: "url", url }, - }); - } - } else if (part.type === "image" && part.source) { - blocks.push({ type: "image", source: part.source }); - } else if (part.type === "image" && typeof part.image === "string") { - // AI SDK-style image part: { type: "image", image: "data:...;base64,..." } (#1330) - const url = part.image; - const match = url.match(/^data:([^;]+);base64,(.+)$/); - if (match) { - blocks.push({ - type: "image", - source: { type: "base64", media_type: match[1], data: match[2] }, - }); - } else if (url.trim()) { - blocks.push({ type: "image", source: { type: "url", url } }); - } + } else if (part.type === "image_url" || part.type === "image") { + const imageBlock = openAiImagePartToClaudeBlock(part); + if (imageBlock) blocks.push(imageBlock); } else if (part.type === "file" && (part.file?.file_data || part.file?.data)) { // OpenAI Chat Completions file block: // {type:"file", file:{filename, file_data:"data:;base64,..."}}. diff --git a/open-sse/translator/request/openai-to-claude/imageBlocks.ts b/open-sse/translator/request/openai-to-claude/imageBlocks.ts new file mode 100644 index 0000000000..3157c500fa --- /dev/null +++ b/open-sse/translator/request/openai-to-claude/imageBlocks.ts @@ -0,0 +1,77 @@ +/** + * Convert OpenAI-style image parts (including those nested in tool results) + * into Claude Messages `image` blocks. User-message `image_url` already did + * this; `role: "tool"` and nested `tool_result` content previously forwarded + * the OpenAI shape unchanged, which Anthropic rejects with HTTP 400 (#9692). + */ + +const DATA_URL_RE = /^data:([^;]+);base64,(.+)$/; + +type ClaudeImageBlock = { + type: "image"; + source: { type: "base64"; media_type: string; data: string } | { type: "url"; url: string }; +}; + +export function extractOpenAiImageUrl(imageUrl: unknown): string { + if (typeof imageUrl === "string") return imageUrl; + if (imageUrl && typeof imageUrl === "object" && !Array.isArray(imageUrl)) { + const url = (imageUrl as { url?: unknown }).url; + if (typeof url === "string") return url; + } + return ""; +} + +export function urlToClaudeImageBlock(url: string): ClaudeImageBlock | null { + if (typeof url !== "string") return null; + const trimmed = url.trim(); + if (!trimmed) return null; + const match = trimmed.match(DATA_URL_RE); + if (match) { + return { + type: "image", + source: { type: "base64", media_type: match[1], data: match[2] }, + }; + } + return { type: "image", source: { type: "url", url: trimmed } }; +} + +/** + * Map one OpenAI / AI-SDK image-shaped part to a Claude image block. + * Returns null when the part is not an image (caller should keep it as-is). + */ +export function openAiImagePartToClaudeBlock( + part: Record +): ClaudeImageBlock | null { + const type = part.type; + if (type === "image_url") { + return urlToClaudeImageBlock(extractOpenAiImageUrl(part.image_url)); + } + if (type === "image") { + if (part.source && typeof part.source === "object" && !Array.isArray(part.source)) { + return { type: "image", source: part.source as ClaudeImageBlock["source"] }; + } + if (typeof part.image === "string") { + return urlToClaudeImageBlock(part.image); + } + } + return null; +} + +/** + * Walk a tool_result content value and rewrite OpenAI `image_url` (and AI-SDK + * `image`) parts to Claude `image` blocks. Nested `tool_result` arrays recurse. + * Non-array content (plain strings) is left unchanged. + */ +export function normalizeToolResultImages(content: unknown): unknown { + if (!Array.isArray(content)) return content; + return content.map((block) => { + if (!block || typeof block !== "object" || Array.isArray(block)) return block; + const rec = block as Record; + const image = openAiImagePartToClaudeBlock(rec); + if (image) return image; + if (rec.type === "tool_result" && Array.isArray(rec.content)) { + return { ...rec, content: normalizeToolResultImages(rec.content) }; + } + return rec; + }); +} diff --git a/tests/unit/openai-to-claude-tool-result-images-9692.test.ts b/tests/unit/openai-to-claude-tool-result-images-9692.test.ts new file mode 100644 index 0000000000..8950553f8c --- /dev/null +++ b/tests/unit/openai-to-claude-tool-result-images-9692.test.ts @@ -0,0 +1,222 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { openaiToClaudeRequest } = + await import("../../open-sse/translator/request/openai-to-claude.ts"); +const { + extractOpenAiImageUrl, + urlToClaudeImageBlock, + openAiImagePartToClaudeBlock, + normalizeToolResultImages, +} = await import("../../open-sse/translator/request/openai-to-claude/imageBlocks.ts"); + +const PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgo="; + +function findToolResult(translated: { messages: Array<{ content?: unknown[] }> }) { + for (const msg of translated.messages) { + if (!Array.isArray(msg.content)) continue; + const toolResult = msg.content.find( + (block) => + block && typeof block === "object" && (block as { type?: string }).type === "tool_result" + ); + if (toolResult) return toolResult as { type: string; content: unknown }; + } + return null; +} + +test("extractOpenAiImageUrl accepts both {url} objects and bare strings", () => { + assert.equal(extractOpenAiImageUrl({ url: PNG_DATA_URL }), PNG_DATA_URL); + assert.equal(extractOpenAiImageUrl(PNG_DATA_URL), PNG_DATA_URL); + assert.equal(extractOpenAiImageUrl({}), ""); + assert.equal(extractOpenAiImageUrl(null), ""); +}); + +test("urlToClaudeImageBlock splits data URLs and keeps http(s) as url sources", () => { + assert.deepEqual(urlToClaudeImageBlock(PNG_DATA_URL), { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + }); + assert.deepEqual(urlToClaudeImageBlock("https://example.com/shot.png"), { + type: "image", + source: { type: "url", url: "https://example.com/shot.png" }, + }); + assert.equal(urlToClaudeImageBlock(" "), null); +}); + +test("openAiImagePartToClaudeBlock maps image_url and AI-SDK image parts", () => { + assert.deepEqual( + openAiImagePartToClaudeBlock({ + type: "image_url", + image_url: { url: PNG_DATA_URL }, + }), + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + } + ); + assert.deepEqual(openAiImagePartToClaudeBlock({ type: "image", image: PNG_DATA_URL }), { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + }); + assert.equal(openAiImagePartToClaudeBlock({ type: "text", text: "ok" }), null); +}); + +test("normalizeToolResultImages rewrites nested image_url inside tool_result content", () => { + const out = normalizeToolResultImages([ + { type: "text", text: "caption" }, + { type: "image_url", image_url: { url: PNG_DATA_URL } }, + { + type: "tool_result", + content: [{ type: "image_url", image_url: "https://cdn.example/a.png" }], + }, + ]); + assert.deepEqual(out, [ + { type: "text", text: "caption" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + }, + { + type: "tool_result", + content: [{ type: "image", source: { type: "url", url: "https://cdn.example/a.png" } }], + }, + ]); +}); + +test("#9692: role=tool image_url content becomes a Claude image block, not a raw image_url", () => { + const translated = openaiToClaudeRequest( + "claude-sonnet-4", + { + messages: [ + { role: "user", content: "describe the screenshot" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + id: "tool_123", + type: "function", + function: { name: "ReadMediaFile", arguments: '{"path":"shot.png"}' }, + }, + ], + }, + { + role: "tool", + tool_call_id: "tool_123", + content: [ + { + type: "image_url", + image_url: { url: PNG_DATA_URL }, + }, + ], + }, + ], + }, + false + ); + + const toolResult = findToolResult(translated); + assert.ok(toolResult, "expected a translated tool_result"); + assert.ok(Array.isArray(toolResult.content), "tool_result.content must stay an array"); + assert.equal( + (toolResult.content as Array<{ type?: string }>).some((block) => block.type === "image_url"), + false, + "OpenAI image_url must not leak into Claude tool_result content" + ); + const image = (toolResult.content as Array>).find( + (block) => block.type === "image" + ); + assert.ok(image, "expected a Claude image block inside tool_result"); + assert.deepEqual(image.source, { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }); +}); + +test("#9692: mixed text + image_url in a tool result keeps text and converts the image", () => { + const translated = openaiToClaudeRequest( + "claude-sonnet-4", + { + messages: [ + { role: "user", content: "read it" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "ReadMediaFile", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "call_1", + content: [ + { type: "text", text: "file: shot.png" }, + { type: "image_url", image_url: PNG_DATA_URL }, + ], + }, + ], + }, + false + ); + + const toolResult = findToolResult(translated); + assert.ok(toolResult); + assert.deepEqual(toolResult.content, [ + { type: "text", text: "file: shot.png" }, + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" }, + }, + ]); +}); + +test("#9692: nested tool_result image_url on a user message is converted the same way", () => { + const translated = openaiToClaudeRequest( + "claude-sonnet-4", + { + messages: [ + { role: "user", content: "describe" }, + { + role: "assistant", + content: "", + tool_calls: [ + { + id: "tool_123", + type: "function", + function: { name: "ReadMediaFile", arguments: "{}" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool_123", + content: [ + { + type: "image_url", + image_url: { url: PNG_DATA_URL }, + }, + ], + }, + ], + }, + ], + }, + false + ); + + const toolResult = findToolResult(translated); + assert.ok(toolResult); + const image = (toolResult.content as Array>).find( + (block) => block.type === "image" + ); + assert.ok(image); + assert.equal((image.source as { type?: string; media_type?: string }).media_type, "image/png"); +});