From c9251f932681f052aec01de8f84e70943f2c9936 Mon Sep 17 00:00:00 2001 From: Tushar Agarwal <76201310+Tushar49@users.noreply.github.com> Date: Fri, 29 May 2026 10:14:01 +0530 Subject: [PATCH] fix(geminiHelper): support rec.image content shape + warn on dropped remote URLs (refs #2807) (#2855) Integrated into release/v3.8.6. --- open-sse/translator/helpers/geminiHelper.ts | 23 ++++ .../unit/translator-openai-to-gemini.test.ts | 108 +++++++++++++++--- 2 files changed, 116 insertions(+), 15 deletions(-) diff --git a/open-sse/translator/helpers/geminiHelper.ts b/open-sse/translator/helpers/geminiHelper.ts index beff4fb525..ff93ed0a68 100644 --- a/open-sse/translator/helpers/geminiHelper.ts +++ b/open-sse/translator/helpers/geminiHelper.ts @@ -149,13 +149,17 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] { // 4. Standard OpenAI Data URIs const imageUrl = toRecord(rec.image_url); + const imageObj = toRecord(rec.image); const fileUrl = toRecord(rec.file_url); const fileObj = toRecord(rec.file); const docObj = toRecord(rec.document); // `file_url` is a top-level string on the Responses-API input_file shape (#2515). + // `rec.image` (with nested {url}) is emitted by some MCP tool wrappers and + // translation layers as an alternative to `rec.image_url` (#2807). const fileData = (typeof rec.file_url === "string" ? rec.file_url : undefined) || imageUrl?.url || + imageObj?.url || fileUrl?.url || fileObj?.url || docObj?.url; @@ -170,6 +174,25 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] { inlineData: { mimeType, data }, }); } + } else if (typeof fileData === "string" && /^https?:\/\//i.test(fileData)) { + // Remote URLs cannot be passed directly to Gemini's inlineData (which + // requires base64). Fetching + encoding would require making this + // function async, which is a breaking change for sync callers (#2807). + // Until that refactor lands, warn loudly instead of silently dropping + // so users can see WHY their vision request failed. + // Strip query string before logging to avoid leaking auth tokens + // (signed URLs, SAS tokens, etc.) embedded in query parameters. + let safeUrl: string; + try { + const parsed = new URL(fileData); + safeUrl = parsed.origin + parsed.pathname; + } catch { + safeUrl = fileData.split("?")[0]; + } + console.warn( + `[geminiHelper] Dropped remote image URL (Gemini inlineData requires base64): ${safeUrl}` + + ` - encode the image as a data: URI client-side until #2807 async fetch lands.` + ); } } } diff --git a/tests/unit/translator-openai-to-gemini.test.ts b/tests/unit/translator-openai-to-gemini.test.ts index 8e85c5d8af..5c28e9f042 100644 --- a/tests/unit/translator-openai-to-gemini.test.ts +++ b/tests/unit/translator-openai-to-gemini.test.ts @@ -47,22 +47,31 @@ function getFunctionDeclarationParameters(parameters: unknown) { } test("OpenAI -> Gemini helper converts text, images and files into Gemini parts", () => { - const parts = convertOpenAIContentToParts([ - { type: "text", text: "Hello" }, - { type: "image_url", image_url: { url: "data:image/png;base64,abc" } }, - { type: "file_url", file_url: { url: "data:application/pdf;base64,Zm9v" } }, - { type: "document", document: { url: "data:text/plain;base64,YmFy" } }, - { type: "image_url", image_url: { url: "https://example.com/skip.png" } }, - { type: "file_url", file_url: { url: "not-a-data-url" } }, - ]); + // Suppress warn emitted for the remote https://example.com/skip.png URL in the + // fixture below — that warn is expected and tested separately. Suppressing here + // keeps stderr clean so CI does not flag spurious output. + const originalWarn = console.warn; + console.warn = () => {}; + try { + const parts = convertOpenAIContentToParts([ + { type: "text", text: "Hello" }, + { type: "image_url", image_url: { url: "data:image/png;base64,abc" } }, + { type: "file_url", file_url: { url: "data:application/pdf;base64,Zm9v" } }, + { type: "document", document: { url: "data:text/plain;base64,YmFy" } }, + { type: "image_url", image_url: { url: "https://example.com/skip.png" } }, + { type: "file_url", file_url: { url: "not-a-data-url" } }, + ]); - assert.deepEqual(parts, [ - { text: "Hello" }, - { inlineData: { mimeType: "image/png", data: "abc" } }, - { inlineData: { mimeType: "application/pdf", data: "Zm9v" } }, - { inlineData: { mimeType: "text/plain", data: "YmFy" } }, - ]); - assert.deepEqual(convertOpenAIContentToParts("raw text"), [{ text: "raw text" }]); + assert.deepEqual(parts, [ + { text: "Hello" }, + { inlineData: { mimeType: "image/png", data: "abc" } }, + { inlineData: { mimeType: "application/pdf", data: "Zm9v" } }, + { inlineData: { mimeType: "text/plain", data: "YmFy" } }, + ]); + assert.deepEqual(convertOpenAIContentToParts("raw text"), [{ text: "raw text" }]); + } finally { + console.warn = originalWarn; + } }); test("OpenAI -> Gemini helper cleans complex JSON Schema structures for Gemini compatibility", () => { @@ -936,6 +945,75 @@ test("convertOpenAIContentToParts handles input_file file_url data URI (#2515)", assert.equal((inline as any).inlineData.mimeType, "application/pdf"); }); +test("convertOpenAIContentToParts handles rec.image with nested {url} as base64 data URI (#2807)", () => { + const parts = convertOpenAIContentToParts([ + { type: "text", text: "What's this?" }, + { type: "image", image: { url: "data:image/png;base64,iVBORw0KGgo=" } }, + ]); + const inline = parts.find((p) => (p as any).inlineData); + assert.ok( + inline, + "rec.image with nested {url} must produce an inlineData part (was previously silently dropped)" + ); + assert.equal((inline as any).inlineData.data, "iVBORw0KGgo="); + assert.equal((inline as any).inlineData.mimeType, "image/png"); +}); + +test("convertOpenAIContentToParts warns and drops remote http(s) URLs (#2807 - until async refactor)", () => { + const originalWarn = console.warn; + const warnings: string[] = []; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + const parts = convertOpenAIContentToParts([ + { type: "image_url", image_url: { url: "https://example.com/cat.png" } }, + ]); + const inline = parts.find((p) => (p as any).inlineData); + assert.equal( + inline, + undefined, + "remote URL still cannot be encoded into inlineData (sync function) - that's expected" + ); + assert.ok( + warnings.some((w) => /Dropped remote image URL/i.test(w) && /example\.com\/cat\.png/.test(w)), + `expected a warning naming the dropped URL, got: ${JSON.stringify(warnings)}` + ); + } finally { + console.warn = originalWarn; + } +}); + +test("convertOpenAIContentToParts warns and drops rec.image remote http(s) URLs (#2807)", () => { + // rec.image is the alternative content shape emitted by MCP tool wrappers and + // LangChain shim layers. Remote URLs in this shape must also hit the warn-and-drop + // branch rather than being silently ignored. + const originalWarn = console.warn; + const warnings: string[] = []; + console.warn = (...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }; + try { + const parts = convertOpenAIContentToParts([ + { type: "image", image: { url: "https://example.com/remote.png" } }, + ]); + const inline = parts.find((p) => (p as any).inlineData); + assert.equal( + inline, + undefined, + "rec.image remote URL must not produce an inlineData part (sync function cannot fetch)" + ); + assert.ok( + warnings.some( + (w) => /Dropped remote image URL/i.test(w) && /example\.com\/remote\.png/.test(w) + ), + `expected a warning naming the dropped rec.image URL, got: ${JSON.stringify(warnings)}` + ); + } finally { + console.warn = originalWarn; + } +}); + // Regression for #2504: with credentials._signatureNamespace set, a previously-cached // Gemini thoughtSignature must be re-attached to the functionCall on the follow-up turn. test("openaiToGeminiRequest re-attaches cached thoughtSignature for FORMATS.GEMINI (#2504)", async () => {