fix(translator): read PDF/video file attachments for Gemini/Antigravity and Claude (#6790)

Reconstructed onto release/v3.8.47 to drop unrelated main-drift (deps/electron/proxy
files belong to #6620, not this PR); keeps only the author's translator + test changes.

Co-authored-by: Wital <witalorocha216@gmail.com>
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>

changelog.d/fixes/6790-gemini-pdf-video-attachments.md

@@ -0,0 +1 @@
+- **fix(translator):** read PDF/video `file_data` attachments on the OpenAI→Gemini/Antigravity and OpenAI→Claude paths so multimodal documents (not just images) reach the upstream — PDFs map to `document`/`inlineData` and videos keep their `video/mp4` mime instead of being dropped (#6790 — thanks @Witroch4).

open-sse/translator/helpers/geminiHelper.ts

@@ -172,17 +172,35 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] {
 
         // 3. Handle raw data strings (e.g. {"type": "file", "data": "JVBER...", "mime_type": "..."}).
         //    Also accept the Responses-API shape {"type":"input_file","file_data":"JVBER...","filename":...}
-        //    so PDFs sent as `input_file` reach Gemini instead of being silently dropped (#2515).
+        //    AND the OpenAI Chat Completions shape
+        //    {"type":"file","file":{"filename":...,"file_data":"data:<mime>;base64,..."}} so PDFs and
+        //    videos reach Gemini instead of being silently dropped (#2515). Gemini reads
+        //    application/pdf and video/* natively via inlineData, exactly like images.
         const file = toRecord(rec.file);
         const doc = toRecord(rec.document);
-        const rawDataStr = rec.data || rec.file_data || file?.data || doc?.data;
-        const mimeTypeFallback =
-          rec.mime_type || rec.media_type || file?.mime_type || doc?.mime_type || "application/pdf";
+        const rawDataStr =
+          rec.data || rec.file_data || file?.data || file?.file_data || doc?.data || doc?.file_data;
         if (typeof rawDataStr === "string" && !rawDataStr.startsWith("http")) {
+          // Prefer the mime embedded in the data: URI (e.g. application/pdf, video/mp4) so
+          // documents and videos are not mislabeled as the fallback; the fallback applies
+          // only to bare base64 that carries no data: prefix.
+          let mimeType =
+            rec.mime_type ||
+            rec.media_type ||
+            file?.mime_type ||
+            doc?.mime_type ||
+            "application/pdf";
+          if (rawDataStr.startsWith("data:")) {
+            const commaIndex = rawDataStr.indexOf(",");
+            if (commaIndex !== -1) {
+              const parsedMime = rawDataStr.substring(5, commaIndex).split(";")[0];
+              if (parsedMime) mimeType = parsedMime;
+            }
+          }
           const rawData = rawDataStr.replace(/^data:[a-zA-Z0-9/+-]+;base64,/, "");
           parts.push({
             inlineData: {
-              mimeType: String(mimeTypeFallback),
+              mimeType: String(mimeType),
               data: rawData,
             },
           });

open-sse/translator/request/openai-to-claude.ts

@@ -545,6 +545,36 @@ function getContentBlocksFromMessage(
           } else if (url.trim()) {
             blocks.push({ type: "image", source: { type: "url", url } });
           }
+        } else if (part.type === "file" && (part.file?.file_data || part.file?.data)) {
+          // OpenAI Chat Completions file block:
+          // {type:"file", file:{filename, file_data:"data:<mime>;base64,..."}}.
+          // Map PDFs to a Claude document block and image mimes to an image block so the
+          // attachment reaches the model instead of being silently dropped. Claude has no
+          // native video input, so non-pdf/non-image files are skipped here.
+          const fileData = part.file.file_data || part.file.data;
+          const fmatch =
+            typeof fileData === "string" ? fileData.match(/^data:([^;]+);base64,(.+)$/) : null;
+          if (fmatch) {
+            const mediaType = fmatch[1];
+            if (mediaType === "application/pdf") {
+              blocks.push({
+                type: "document",
+                source: { type: "base64", media_type: mediaType, data: fmatch[2] },
+                ...(part.file.filename ? { title: part.file.filename } : {}),
+              });
+            } else if (mediaType.startsWith("image/")) {
+              blocks.push({
+                type: "image",
+                source: { type: "base64", media_type: mediaType, data: fmatch[2] },
+              });
+            }
+          } else if (typeof fileData === "string" && /^https?:\/\//i.test(fileData)) {
+            blocks.push({
+              type: "document",
+              source: { type: "url", url: fileData },
+              ...(part.file.filename ? { title: part.file.filename } : {}),
+            });
+          }
         }
       }
     }
@@ -622,7 +652,12 @@ function getContentBlocksFromMessage(
       (b) => b.type === "thinking" || b.type === "redacted_thinking"
     );
     const hasToolUseBlock = blocks.some((b) => b.type === "tool_use");
-    if (msg.reasoning_content && thinkingEnabledForRequest && hasToolUseBlock && !hasThinkingBlock) {
+    if (
+      msg.reasoning_content &&
+      thinkingEnabledForRequest &&
+      hasToolUseBlock &&
+      !hasThinkingBlock
+    ) {
       blocks.unshift({
         type: "redacted_thinking",
         data: DEFAULT_THINKING_CLAUDE_SIGNATURE,

tests/unit/gemini-helper.test.ts

@@ -20,7 +20,7 @@ test("DEFAULT_SAFETY_SETTINGS is an array", () => {
 
 test("tryParseJSON parses valid JSON", () => {
   assert.deepEqual(gemini.tryParseJSON('{"a":1}'), { a: 1 });
-  assert.deepEqual(gemini.tryParseJSON('[1,2,3]'), [1, 2, 3]);
+  assert.deepEqual(gemini.tryParseJSON("[1,2,3]"), [1, 2, 3]);
   assert.equal(gemini.tryParseJSON('"hello"'), "hello");
   assert.equal(gemini.tryParseJSON("42"), 42);
   assert.equal(gemini.tryParseJSON("true"), true);
@@ -130,3 +130,37 @@ test("cleanJSONSchemaForAntigravity handles nested schema", () => {
   const result = gemini.cleanJSONSchemaForAntigravity(schema);
   assert.ok(typeof result === "object");
 });
+
+test("convertOpenAIContentToParts maps OpenAI Chat Completions file (PDF) to inlineData", () => {
+  const content = [
+    { type: "text", text: "read this" },
+    {
+      type: "file",
+      file: { filename: "doc.pdf", file_data: "data:application/pdf;base64,JVBERiAtMQ==" },
+    },
+  ];
+  const parts = gemini.convertOpenAIContentToParts(content);
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "PDF file part must be converted to inlineData, not dropped");
+  assert.equal(inline.inlineData.mimeType, "application/pdf");
+  assert.equal(inline.inlineData.data, "JVBERiAtMQ==");
+});
+
+test("convertOpenAIContentToParts keeps the real mime for a video file_data", () => {
+  const content = [
+    { type: "file", file: { filename: "clip.mp4", file_data: "data:video/mp4;base64,AAAAIGZ0" } },
+  ];
+  const parts = gemini.convertOpenAIContentToParts(content);
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "video file part must be converted to inlineData");
+  assert.equal(inline.inlineData.mimeType, "video/mp4");
+  assert.equal(inline.inlineData.data, "AAAAIGZ0");
+});
+
+test("convertOpenAIContentToParts still maps image_url data URIs (regression)", () => {
+  const content = [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } }];
+  const parts = gemini.convertOpenAIContentToParts(content);
+  const inline = parts.find((p) => p.inlineData);
+  assert.ok(inline, "image_url must still convert to inlineData");
+  assert.equal(inline.inlineData.mimeType, "image/png");
+});

tests/unit/openai-to-claude-file-attachments.test.ts

@@ -0,0 +1,70 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const { openaiToClaudeRequest } =
+  await import("../../open-sse/translator/request/openai-to-claude.ts");
+
+function userBlocks(model: string, content: unknown) {
+  const translated = openaiToClaudeRequest(model, { messages: [{ role: "user", content }] }, false);
+  const userMsg = translated.messages.find((m) => m.role === "user");
+  assert.ok(userMsg && Array.isArray(userMsg.content), "expected a translated user message");
+  return userMsg.content;
+}
+
+test("openaiToClaudeRequest maps an OpenAI file (PDF) block to a Claude document block", () => {
+  const blocks = userBlocks("claude-sonnet-4", [
+    { type: "text", text: "summarize" },
+    {
+      type: "file",
+      file: { filename: "edital.pdf", file_data: "data:application/pdf;base64,JVBERiAtMQ==" },
+    },
+  ]);
+  const doc = blocks.find((b) => b.type === "document");
+  assert.ok(doc, "PDF file block must become a Claude document block, not be dropped");
+  assert.equal(doc.source.type, "base64");
+  assert.equal(doc.source.media_type, "application/pdf");
+  assert.equal(doc.source.data, "JVBERiAtMQ==");
+  assert.equal(doc.title, "edital.pdf");
+});
+
+test("openaiToClaudeRequest maps an OpenAI file (image mime) block to a Claude image block", () => {
+  const blocks = userBlocks("claude-sonnet-4", [
+    {
+      type: "file",
+      file: { filename: "shot.png", file_data: "data:image/png;base64,iVBORw0KGgo=" },
+    },
+  ]);
+  const img = blocks.find((b) => b.type === "image");
+  assert.ok(img, "image-mime file block must become a Claude image block");
+  assert.equal(img.source.type, "base64");
+  assert.equal(img.source.media_type, "image/png");
+  assert.equal(img.source.data, "iVBORw0KGgo=");
+});
+
+test("openaiToClaudeRequest maps a remote file (PDF url) block to a Claude document url block", () => {
+  const blocks = userBlocks("claude-sonnet-4", [
+    { type: "file", file: { filename: "remote.pdf", file_data: "https://example.com/a.pdf" } },
+  ]);
+  const doc = blocks.find((b) => b.type === "document");
+  assert.ok(doc, "remote PDF file block must become a Claude document url block");
+  assert.equal(doc.source.type, "url");
+  assert.equal(doc.source.url, "https://example.com/a.pdf");
+});
+
+test("openaiToClaudeRequest skips a video file block (Claude has no native video input)", () => {
+  const blocks = userBlocks("claude-sonnet-4", [
+    { type: "text", text: "watch this" },
+    { type: "file", file: { filename: "clip.mp4", file_data: "data:video/mp4;base64,AAAAIGZ0" } },
+  ]);
+  const doc = blocks.find((b) => b.type === "document");
+  const img = blocks.find((b) => b.type === "image");
+  assert.ok(
+    !doc && !img,
+    "a video file must not be mislabeled as a Claude document or image block"
+  );
+  // the text part is still forwarded
+  assert.ok(
+    blocks.some((b) => b.type === "text"),
+    "the accompanying text part must still be forwarded"
+  );
+});