fix(geminiHelper): support rec.image content shape + warn on dropped remote URLs (refs #2807) (#2855)

Integrated into release/v3.8.6.

open-sse/translator/helpers/geminiHelper.ts

@@ -149,13 +149,17 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
 
         // 4. Standard OpenAI Data URIs
         const imageUrl = toRecord(rec.image_url);
+        const imageObj = toRecord(rec.image);
         const fileUrl = toRecord(rec.file_url);
         const fileObj = toRecord(rec.file);
         const docObj = toRecord(rec.document);
         // `file_url` is a top-level string on the Responses-API input_file shape (#2515).
+        // `rec.image` (with nested {url}) is emitted by some MCP tool wrappers and
+        // translation layers as an alternative to `rec.image_url` (#2807).
         const fileData =
           (typeof rec.file_url === "string" ? rec.file_url : undefined) ||
           imageUrl?.url ||
+          imageObj?.url ||
           fileUrl?.url ||
           fileObj?.url ||
           docObj?.url;
@@ -170,6 +174,25 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
               inlineData: { mimeType, data },
             });
           }
+        } else if (typeof fileData === "string" && /^https?:\/\//i.test(fileData)) {
+          // Remote URLs cannot be passed directly to Gemini's inlineData (which
+          // requires base64). Fetching + encoding would require making this
+          // function async, which is a breaking change for sync callers (#2807).
+          // Until that refactor lands, warn loudly instead of silently dropping
+          // so users can see WHY their vision request failed.
+          // Strip query string before logging to avoid leaking auth tokens
+          // (signed URLs, SAS tokens, etc.) embedded in query parameters.
+          let safeUrl: string;
+          try {
+            const parsed = new URL(fileData);
+            safeUrl = parsed.origin + parsed.pathname;
+          } catch {
+            safeUrl = fileData.split("?")[0];
+          }
+          console.warn(
+            `[geminiHelper] Dropped remote image URL (Gemini inlineData requires base64): ${safeUrl}` +
+              ` - encode the image as a data: URI client-side until #2807 async fetch lands.`
+          );
         }
       }
     }

tests/unit/translator-openai-to-gemini.test.ts

@@ -47,22 +47,31 @@ function getFunctionDeclarationParameters(parameters: unknown) {
 }
 
 test("OpenAI -> Gemini helper converts text, images and files into Gemini parts", () => {
-  const parts = convertOpenAIContentToParts([
-    { type: "text", text: "Hello" },
-    { type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
-    { type: "file_url", file_url: { url: "data:application/pdf;base64,Zm9v" } },
-    { type: "document", document: { url: "data:text/plain;base64,YmFy" } },
-    { type: "image_url", image_url: { url: "https://example.com/skip.png" } },
-    { type: "file_url", file_url: { url: "not-a-data-url" } },
-  ]);
+  // Suppress warn emitted for the remote https://example.com/skip.png URL in the
+  // fixture below — that warn is expected and tested separately. Suppressing here
+  // keeps stderr clean so CI does not flag spurious output.
+  const originalWarn = console.warn;
+  console.warn = () => {};
+  try {
+    const parts = convertOpenAIContentToParts([
+      { type: "text", text: "Hello" },
+      { type: "image_url", image_url: { url: "data:image/png;base64,abc" } },
+      { type: "file_url", file_url: { url: "data:application/pdf;base64,Zm9v" } },
+      { type: "document", document: { url: "data:text/plain;base64,YmFy" } },
+      { type: "image_url", image_url: { url: "https://example.com/skip.png" } },
+      { type: "file_url", file_url: { url: "not-a-data-url" } },
+    ]);
 
-  assert.deepEqual(parts, [
-    { text: "Hello" },
-    { inlineData: { mimeType: "image/png", data: "abc" } },
-    { inlineData: { mimeType: "application/pdf", data: "Zm9v" } },
-    { inlineData: { mimeType: "text/plain", data: "YmFy" } },
-  ]);
-  assert.deepEqual(convertOpenAIContentToParts("raw text"), [{ text: "raw text" }]);
+    assert.deepEqual(parts, [
+      { text: "Hello" },
+      { inlineData: { mimeType: "image/png", data: "abc" } },
+      { inlineData: { mimeType: "application/pdf", data: "Zm9v" } },
+      { inlineData: { mimeType: "text/plain", data: "YmFy" } },
+    ]);
+    assert.deepEqual(convertOpenAIContentToParts("raw text"), [{ text: "raw text" }]);
+  } finally {
+    console.warn = originalWarn;
+  }
 });
 
 test("OpenAI -> Gemini helper cleans complex JSON Schema structures for Gemini compatibility", () => {
@@ -936,6 +945,75 @@ test("convertOpenAIContentToParts handles input_file file_url data URI (#2515)",
   assert.equal((inline as any).inlineData.mimeType, "application/pdf");
 });
 
+test("convertOpenAIContentToParts handles rec.image with nested {url} as base64 data URI (#2807)", () => {
+  const parts = convertOpenAIContentToParts([
+    { type: "text", text: "What's this?" },
+    { type: "image", image: { url: "data:image/png;base64,iVBORw0KGgo=" } },
+  ]);
+  const inline = parts.find((p) => (p as any).inlineData);
+  assert.ok(
+    inline,
+    "rec.image with nested {url} must produce an inlineData part (was previously silently dropped)"
+  );
+  assert.equal((inline as any).inlineData.data, "iVBORw0KGgo=");
+  assert.equal((inline as any).inlineData.mimeType, "image/png");
+});
+
+test("convertOpenAIContentToParts warns and drops remote http(s) URLs (#2807 - until async refactor)", () => {
+  const originalWarn = console.warn;
+  const warnings: string[] = [];
+  console.warn = (...args: unknown[]) => {
+    warnings.push(args.map(String).join(" "));
+  };
+  try {
+    const parts = convertOpenAIContentToParts([
+      { type: "image_url", image_url: { url: "https://example.com/cat.png" } },
+    ]);
+    const inline = parts.find((p) => (p as any).inlineData);
+    assert.equal(
+      inline,
+      undefined,
+      "remote URL still cannot be encoded into inlineData (sync function) - that's expected"
+    );
+    assert.ok(
+      warnings.some((w) => /Dropped remote image URL/i.test(w) && /example\.com\/cat\.png/.test(w)),
+      `expected a warning naming the dropped URL, got: ${JSON.stringify(warnings)}`
+    );
+  } finally {
+    console.warn = originalWarn;
+  }
+});
+
+test("convertOpenAIContentToParts warns and drops rec.image remote http(s) URLs (#2807)", () => {
+  // rec.image is the alternative content shape emitted by MCP tool wrappers and
+  // LangChain shim layers. Remote URLs in this shape must also hit the warn-and-drop
+  // branch rather than being silently ignored.
+  const originalWarn = console.warn;
+  const warnings: string[] = [];
+  console.warn = (...args: unknown[]) => {
+    warnings.push(args.map(String).join(" "));
+  };
+  try {
+    const parts = convertOpenAIContentToParts([
+      { type: "image", image: { url: "https://example.com/remote.png" } },
+    ]);
+    const inline = parts.find((p) => (p as any).inlineData);
+    assert.equal(
+      inline,
+      undefined,
+      "rec.image remote URL must not produce an inlineData part (sync function cannot fetch)"
+    );
+    assert.ok(
+      warnings.some(
+        (w) => /Dropped remote image URL/i.test(w) && /example\.com\/remote\.png/.test(w)
+      ),
+      `expected a warning naming the dropped rec.image URL, got: ${JSON.stringify(warnings)}`
+    );
+  } finally {
+    console.warn = originalWarn;
+  }
+});
+
 // Regression for #2504: with credentials._signatureNamespace set, a previously-cached
 // Gemini thoughtSignature must be re-attached to the functionCall on the follow-up turn.
 test("openaiToGeminiRequest re-attaches cached thoughtSignature for FORMATS.GEMINI (#2504)", async () => {