From b2efa35982ed19e19e50684df143525f7f0666c2 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:56:40 -0300 Subject: [PATCH] =?UTF-8?q?fix(sse):=20CC=20bridge=20loses=20OpenAI-format?= =?UTF-8?q?=20image=20input=20(OpenCode/Kilo/Cline=20=E2=86=92=20AgentRout?= =?UTF-8?q?er)=20(#7888)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sse): convert OpenAI media parts to Claude blocks in the CC bridge OpenAI-format clients (OpenCode/Kilo/Cline) reach the Claude-Code-compatible bridge untranslated: chatCore skips the OpenAI->Claude translator when sourceFormat is OPENAI, so image_url / AI-SDK image / file parts either went upstream in OpenAI shape (silently ignored) or were dropped by the text-only extraction, and media-only user turns were removed by hasValidContent(). - claudeCodeCompatible: convertOpenAiMediaBlock() converts image_url (base64 + remote), AI-SDK string image and file parts (pdf->document, image mime->image) to Claude blocks in both bridge paths; Claude-native blocks pass through unchanged and the text-only wire image is preserved. - claudeHelper: hasValidContent() now counts image/document blocks so media-only user turns are not silently deleted. Reported-by: beingshafin Refs #7777 * refactor(sse): extract CC media-block conversion to ccOpenAiMediaBlocks.ts claudeCodeCompatible.ts is frozen at 1202 lines by check:file-size; the #7777 helpers pushed it to 1291. Move them to a dedicated module, no behavior change. * test(quality): register cc-bridge-openai-image-7777 test in stryker tap.testFiles --- .../7777-cc-bridge-openai-image-blocks.md | 1 + open-sse/services/ccOpenAiMediaBlocks.ts | 108 +++++++++ open-sse/services/claudeCodeCompatible.ts | 17 +- open-sse/translator/helpers/claudeHelper.ts | 6 +- stryker.conf.json | 1 + .../unit/cc-bridge-openai-image-7777.test.ts | 221 ++++++++++++++++++ 6 files changed, 345 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fixes/7777-cc-bridge-openai-image-blocks.md create mode 100644 open-sse/services/ccOpenAiMediaBlocks.ts create mode 100644 tests/unit/cc-bridge-openai-image-7777.test.ts diff --git a/changelog.d/fixes/7777-cc-bridge-openai-image-blocks.md b/changelog.d/fixes/7777-cc-bridge-openai-image-blocks.md new file mode 100644 index 0000000000..619cf10692 --- /dev/null +++ b/changelog.d/fixes/7777-cc-bridge-openai-image-blocks.md @@ -0,0 +1 @@ +- **fix(sse):** Claude-Code-compatible bridge (AgentRouter and any CC provider) no longer loses vision input from OpenAI-format clients (OpenCode/Kilo/Cline): `image_url`, AI-SDK `image` and Chat Completions `file` parts are now converted to Claude `image`/`document` blocks before dispatch, and media-only user turns are no longer dropped by the empty-content filter. (thanks @beingshafin) diff --git a/open-sse/services/ccOpenAiMediaBlocks.ts b/open-sse/services/ccOpenAiMediaBlocks.ts new file mode 100644 index 0000000000..753cde7ca0 --- /dev/null +++ b/open-sse/services/ccOpenAiMediaBlocks.ts @@ -0,0 +1,108 @@ +/** + * OpenAI-format media parts reach the Claude-Code-compatible bridge + * untranslated when the source client speaks OpenAI (chatCore skips the + * OpenAI→Claude translator for CC-compatible providers), so `image_url` / + * AI SDK `image` / Chat Completions `file` parts must become Claude blocks + * before dispatch or the upstream silently ignores them (#7777). + * Claude-native blocks (`image`/`document` carrying `source`) are left for + * the caller to pass through unchanged. + */ + +const DATA_URL_BASE64_PATTERN = /^data:([^;]+);base64,(.+)$/; + +/** Returns a Claude block for an OpenAI media part, or null for non-media blocks. */ +export function convertOpenAiMediaBlock( + record: Record +): Record | null { + if (record.type === "image_url") { + const rawUrl = + typeof record.image_url === "string" ? record.image_url : readRecord(record.image_url)?.url; + return claudeImageBlockFromUrl(rawUrl); + } + + if (record.type === "image" && !readRecord(record.source) && typeof record.image === "string") { + return claudeImageBlockFromUrl(record.image); + } + + if (record.type === "file") { + const file = readRecord(record.file); + const fileData = toNonEmptyString(file?.file_data) || toNonEmptyString(file?.data); + if (!fileData) return null; + const title = toNonEmptyString(file?.filename); + const match = fileData.match(DATA_URL_BASE64_PATTERN); + if (match) { + const mediaType = match[1]; + const source = { type: "base64", media_type: mediaType, data: match[2] }; + if (mediaType === "application/pdf") { + return { type: "document", source, ...(title ? { title } : {}) }; + } + if (mediaType.startsWith("image/")) { + return { type: "image", source }; + } + return null; + } + if (/^https?:\/\//i.test(fileData)) { + return { + type: "document", + source: { type: "url", url: fileData }, + ...(title ? { title } : {}), + }; + } + return null; + } + + return null; +} + +/** + * Collects the Claude-shaped media blocks of a message's content array: + * OpenAI media parts are converted, Claude-native `image`/`document` blocks + * are cloned through as-is, everything else is ignored. + */ +export function collectClaudeMediaBlocks(content: unknown): Array> { + if (!Array.isArray(content)) return []; + + const blocks: Array> = []; + for (const part of content) { + const record = readRecord(cloneValue(part)); + if (!record) continue; + const media = convertOpenAiMediaBlock(record); + if (media) { + blocks.push(media); + continue; + } + if ((record.type === "image" || record.type === "document") && readRecord(record.source)) { + blocks.push(record); + } + } + return blocks; +} + +function claudeImageBlockFromUrl(rawUrl: unknown): Record | null { + const url = toNonEmptyString(rawUrl); + if (!url) return null; + const match = url.match(DATA_URL_BASE64_PATTERN); + if (match) { + return { type: "image", source: { type: "base64", media_type: match[1], data: match[2] } }; + } + return { type: "image", source: { type: "url", url } }; +} + +function readRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function toNonEmptyString(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed || null; +} + +function cloneValue(value: T): T { + if (typeof structuredClone === "function") { + return structuredClone(value); + } + return JSON.parse(JSON.stringify(value)) as T; +} diff --git a/open-sse/services/claudeCodeCompatible.ts b/open-sse/services/claudeCodeCompatible.ts index a33e4cd09e..5e3f5c7ae8 100644 --- a/open-sse/services/claudeCodeCompatible.ts +++ b/open-sse/services/claudeCodeCompatible.ts @@ -16,6 +16,7 @@ import { applyClaudeCodeCompatibleThinkingDisplay } from "./claudeCodeCompatible import { obfuscateInBody } from "./claudeCodeObfuscation.ts"; import { applySystemTransformPipeline, PROVIDER_CC_BRIDGE } from "./systemTransforms.ts"; import { usesCcWireImage } from "./ccWireImageBuiltins.ts"; +import { collectClaudeMediaBlocks, convertOpenAiMediaBlock } from "./ccOpenAiMediaBlocks.ts"; import { fixToolPairs, fixToolAdjacency, @@ -517,13 +518,13 @@ function buildClaudeCodeCompatibleMessages(messages: MessageLike[]) { message ): message is { role: "user" | "assistant"; - content: Array<{ type: string; text: string }>; + content: Array>; } => !!message && message.content.length > 0 ); const merged: Array<{ role: "user" | "assistant"; - content: Array<{ type: string; text: string }>; + content: Array>; }> = []; for (const message of converted) { @@ -719,12 +720,12 @@ function convertClaudeCodeCompatibleMessage(message: MessageLike | null | undefi if (!role) return null; const text = contentToText(message?.content); - if (!text) return null; + // #7777: keep the user-turn media parts that contentToText() above drops. + const media = role === "user" ? collectClaudeMediaBlocks(message?.content) : []; + const content = [...(text ? [{ type: "text", text }] : []), ...media]; + if (content.length === 0) return null; - return { - role, - content: [{ type: "text", text }], - }; + return { role, content }; } function buildClaudeCodeCompatibleTools( @@ -977,7 +978,7 @@ function normalizeClaudeContentBlock(block: unknown) { }; } - return record; + return convertOpenAiMediaBlock(record) ?? record; } function convertClaudeCodeCompatibleClaudeMessage( diff --git a/open-sse/translator/helpers/claudeHelper.ts b/open-sse/translator/helpers/claudeHelper.ts index ce4abf4808..20a91b0931 100644 --- a/open-sse/translator/helpers/claudeHelper.ts +++ b/open-sse/translator/helpers/claudeHelper.ts @@ -87,7 +87,11 @@ export function hasValidContent(msg: ClaudeMessage): boolean { (block) => (block.type === "text" && block.text?.trim()) || block.type === "tool_use" || - block.type === "tool_result" + block.type === "tool_result" || + // #7777: media-only user turns are real content — dropping them + // silently deletes vision input on the CC bridge / Claude paths. + block.type === "image" || + block.type === "document" ); } return false; diff --git a/stryker.conf.json b/stryker.conf.json index 6066be870d..900186627e 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -69,6 +69,7 @@ "tests/unit/auto-combo-scoring-clamp.test.ts", "tests/unit/build/check-circular-deps.test.ts", "tests/unit/cache-sweeps.test.ts", + "tests/unit/cc-bridge-openai-image-7777.test.ts", "tests/unit/cc-compatible-provider.test.ts", "tests/unit/chat-combo-live-test.test.ts", "tests/unit/chat-context-relay.test.ts", diff --git a/tests/unit/cc-bridge-openai-image-7777.test.ts b/tests/unit/cc-bridge-openai-image-7777.test.ts new file mode 100644 index 0000000000..0c2071f1c8 --- /dev/null +++ b/tests/unit/cc-bridge-openai-image-7777.test.ts @@ -0,0 +1,221 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { buildClaudeCodeCompatibleRequest } = await import( + "../../open-sse/services/claudeCodeCompatible.ts" +); + +// #7777 — OpenAI-format clients (OpenCode/Kilo/Cline) reach the CC bridge +// untranslated: chatCore skips the OpenAI→Claude translator when +// sourceFormat === OPENAI, so `image_url` / AI-SDK `image` / `file` parts used +// to arrive at the upstream in OpenAI shape (silently ignored) or be dropped +// by the text-only extraction. These tests pin the bridge-level conversion. + +type Block = Record; + +const PNG_DATA_URL = "data:image/png;base64,iVBORw0KGgoTESTPNG"; +const JPEG_DATA_URL = "data:image/jpeg;base64,/9j/4AAQTESTJPEG"; +const PDF_DATA_URL = "data:application/pdf;base64,JVBERi0xLjQKTESTPDF"; + +function buildRequest(messages: unknown[]) { + const body = { model: "claude-opus-4-8", messages, max_tokens: 1024 }; + return buildClaudeCodeCompatibleRequest({ + sourceBody: body as unknown as Record, + normalizedBody: { ...body } as unknown as Record, + claudeBody: null, + model: "claude-opus-4-8", + stream: false, + sessionId: "cc-bridge-7777-session", + }); +} + +function allContentBlocks(request: { messages: unknown }): Block[] { + const messages = Array.isArray(request.messages) ? request.messages : []; + return messages.flatMap((message) => { + const content = (message as { content?: unknown }).content; + return Array.isArray(content) ? (content as Block[]) : []; + }); +} + +function findImageBlocks(blocks: Block[]): Block[] { + return blocks.filter((block) => block.type === "image"); +} + +const SYSTEM_MESSAGE = { role: "system", content: "You are a vision assistant." }; + +test("CC bridge converts OpenAI image_url data URLs to Claude base64 image blocks (system present, #7777)", () => { + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [ + { type: "text", text: "Describe this image" }, + { type: "image_url", image_url: { url: PNG_DATA_URL } }, + ], + }, + ]); + + const blocks = allContentBlocks(request); + assert.equal( + blocks.some((block) => block.type === "image_url"), + false, + "raw OpenAI image_url blocks must not reach the upstream payload" + ); + const [image] = findImageBlocks(blocks); + assert.ok(image, "expected a Claude image block in the upstream messages"); + assert.deepEqual(image.source, { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoTESTPNG", + }); + assert.ok( + blocks.some((block) => block.type === "text" && block.text === "Describe this image"), + "the text part must survive alongside the image" + ); +}); + +test("CC bridge converts remote image_url references to Claude url image blocks (#7777)", () => { + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [ + { type: "text", text: "What is in this picture?" }, + { type: "image_url", image_url: { url: "https://example.com/cat.png" } }, + ], + }, + ]); + + const [image] = findImageBlocks(allContentBlocks(request)); + assert.ok(image, "expected a Claude image block in the upstream messages"); + assert.deepEqual(image.source, { type: "url", url: "https://example.com/cat.png" }); +}); + +test("CC bridge preserves images when the request has no system message (#7777)", () => { + const request = buildRequest([ + { + role: "user", + content: [ + { type: "text", text: "Describe this image" }, + { type: "image_url", image_url: { url: PNG_DATA_URL } }, + ], + }, + ]); + + const blocks = allContentBlocks(request); + const [image] = findImageBlocks(blocks); + assert.ok(image, "expected a Claude image block on the no-system path"); + assert.deepEqual(image.source, { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoTESTPNG", + }); + assert.ok( + blocks.some((block) => block.type === "text" && block.text === "Describe this image"), + "the text part must survive alongside the image" + ); +}); + +test("CC bridge converts AI SDK-style string image parts (#7777)", () => { + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [ + { type: "text", text: "Inspect the attachment" }, + { type: "image", image: JPEG_DATA_URL }, + ], + }, + ]); + + const [image] = findImageBlocks(allContentBlocks(request)); + assert.ok(image, "expected a Claude image block for the AI SDK image part"); + assert.deepEqual(image.source, { + type: "base64", + media_type: "image/jpeg", + data: "/9j/4AAQTESTJPEG", + }); +}); + +test("CC bridge maps OpenAI file parts with PDF data to Claude document blocks (#7777)", () => { + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [ + { type: "text", text: "Summarize the report" }, + { type: "file", file: { filename: "report.pdf", file_data: PDF_DATA_URL } }, + ], + }, + ]); + + const blocks = allContentBlocks(request); + const document = blocks.find((block) => block.type === "document"); + assert.ok(document, "expected a Claude document block for the PDF file part"); + assert.deepEqual(document.source, { + type: "base64", + media_type: "application/pdf", + data: "JVBERi0xLjQKTESTPDF", + }); + assert.equal(document.title, "report.pdf"); +}); + +test("CC bridge keeps image-only user messages instead of dropping them (#7777)", () => { + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [{ type: "image_url", image_url: { url: PNG_DATA_URL } }], + }, + ]); + + const messages = Array.isArray(request.messages) ? request.messages : []; + assert.ok(messages.length >= 1, "the image-only user message must not be dropped"); + const [image] = findImageBlocks(allContentBlocks(request)); + assert.ok(image, "expected the image block of an image-only message to survive"); + assert.deepEqual(image.source, { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoTESTPNG", + }); +}); + +test("CC bridge passes Claude-native image blocks through unchanged", () => { + const nativeSource = { type: "base64", media_type: "image/png", data: "NATIVEDATA" }; + const request = buildRequest([ + SYSTEM_MESSAGE, + { + role: "user", + content: [ + { type: "text", text: "Already Claude-shaped" }, + { type: "image", source: nativeSource }, + ], + }, + ]); + + const [image] = findImageBlocks(allContentBlocks(request)); + assert.ok(image, "expected the Claude-native image block to be preserved"); + assert.deepEqual(image.source, nativeSource); +}); + +test("CC bridge keeps the legacy text-only wire image intact (regression guard)", () => { + const request = buildRequest([ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi there" }, + { + role: "user", + content: [ + { type: "text", text: "first" }, + { type: "text", text: "second" }, + ], + }, + ]); + + const messages = Array.isArray(request.messages) + ? (request.messages as Array<{ role: string; content: Block[] }>) + : []; + assert.equal(messages.length, 3); + assert.deepEqual(messages[0].content, [{ type: "text", text: "hello" }]); + assert.deepEqual(messages[1].content, [{ type: "text", text: "hi there" }]); + assert.deepEqual(messages[2].content, [{ type: "text", text: "first\nsecond" }]); +});