diff --git a/changelog.d/fixes/6790-gemini-pdf-video-attachments.md b/changelog.d/fixes/6790-gemini-pdf-video-attachments.md new file mode 100644 index 0000000000..9875a41091 --- /dev/null +++ b/changelog.d/fixes/6790-gemini-pdf-video-attachments.md @@ -0,0 +1 @@ +- **fix(translator):** read PDF/video `file_data` attachments on the OpenAI→Gemini/Antigravity and OpenAI→Claude paths so multimodal documents (not just images) reach the upstream — PDFs map to `document`/`inlineData` and videos keep their `video/mp4` mime instead of being dropped (#6790 — thanks @Witroch4). diff --git a/open-sse/translator/helpers/geminiHelper.ts b/open-sse/translator/helpers/geminiHelper.ts index 5fd500b2da..7f45cf329d 100644 --- a/open-sse/translator/helpers/geminiHelper.ts +++ b/open-sse/translator/helpers/geminiHelper.ts @@ -172,17 +172,35 @@ export function convertOpenAIContentToParts(content: unknown): JsonRecord[] { // 3. Handle raw data strings (e.g. {"type": "file", "data": "JVBER...", "mime_type": "..."}). // Also accept the Responses-API shape {"type":"input_file","file_data":"JVBER...","filename":...} - // so PDFs sent as `input_file` reach Gemini instead of being silently dropped (#2515). + // AND the OpenAI Chat Completions shape + // {"type":"file","file":{"filename":...,"file_data":"data:;base64,..."}} so PDFs and + // videos reach Gemini instead of being silently dropped (#2515). Gemini reads + // application/pdf and video/* natively via inlineData, exactly like images. const file = toRecord(rec.file); const doc = toRecord(rec.document); - const rawDataStr = rec.data || rec.file_data || file?.data || doc?.data; - const mimeTypeFallback = - rec.mime_type || rec.media_type || file?.mime_type || doc?.mime_type || "application/pdf"; + const rawDataStr = + rec.data || rec.file_data || file?.data || file?.file_data || doc?.data || doc?.file_data; if (typeof rawDataStr === "string" && !rawDataStr.startsWith("http")) { + // Prefer the mime embedded in the data: URI (e.g. application/pdf, video/mp4) so + // documents and videos are not mislabeled as the fallback; the fallback applies + // only to bare base64 that carries no data: prefix. + let mimeType = + rec.mime_type || + rec.media_type || + file?.mime_type || + doc?.mime_type || + "application/pdf"; + if (rawDataStr.startsWith("data:")) { + const commaIndex = rawDataStr.indexOf(","); + if (commaIndex !== -1) { + const parsedMime = rawDataStr.substring(5, commaIndex).split(";")[0]; + if (parsedMime) mimeType = parsedMime; + } + } const rawData = rawDataStr.replace(/^data:[a-zA-Z0-9/+-]+;base64,/, ""); parts.push({ inlineData: { - mimeType: String(mimeTypeFallback), + mimeType: String(mimeType), data: rawData, }, }); diff --git a/open-sse/translator/request/openai-to-claude.ts b/open-sse/translator/request/openai-to-claude.ts index 1c380b5f33..ec75bb5834 100644 --- a/open-sse/translator/request/openai-to-claude.ts +++ b/open-sse/translator/request/openai-to-claude.ts @@ -545,6 +545,36 @@ function getContentBlocksFromMessage( } else if (url.trim()) { blocks.push({ type: "image", source: { type: "url", url } }); } + } else if (part.type === "file" && (part.file?.file_data || part.file?.data)) { + // OpenAI Chat Completions file block: + // {type:"file", file:{filename, file_data:"data:;base64,..."}}. + // Map PDFs to a Claude document block and image mimes to an image block so the + // attachment reaches the model instead of being silently dropped. Claude has no + // native video input, so non-pdf/non-image files are skipped here. + const fileData = part.file.file_data || part.file.data; + const fmatch = + typeof fileData === "string" ? fileData.match(/^data:([^;]+);base64,(.+)$/) : null; + if (fmatch) { + const mediaType = fmatch[1]; + if (mediaType === "application/pdf") { + blocks.push({ + type: "document", + source: { type: "base64", media_type: mediaType, data: fmatch[2] }, + ...(part.file.filename ? { title: part.file.filename } : {}), + }); + } else if (mediaType.startsWith("image/")) { + blocks.push({ + type: "image", + source: { type: "base64", media_type: mediaType, data: fmatch[2] }, + }); + } + } else if (typeof fileData === "string" && /^https?:\/\//i.test(fileData)) { + blocks.push({ + type: "document", + source: { type: "url", url: fileData }, + ...(part.file.filename ? { title: part.file.filename } : {}), + }); + } } } } @@ -622,7 +652,12 @@ function getContentBlocksFromMessage( (b) => b.type === "thinking" || b.type === "redacted_thinking" ); const hasToolUseBlock = blocks.some((b) => b.type === "tool_use"); - if (msg.reasoning_content && thinkingEnabledForRequest && hasToolUseBlock && !hasThinkingBlock) { + if ( + msg.reasoning_content && + thinkingEnabledForRequest && + hasToolUseBlock && + !hasThinkingBlock + ) { blocks.unshift({ type: "redacted_thinking", data: DEFAULT_THINKING_CLAUDE_SIGNATURE, diff --git a/tests/unit/gemini-helper.test.ts b/tests/unit/gemini-helper.test.ts index 479a9d3fa9..e7a14cd858 100644 --- a/tests/unit/gemini-helper.test.ts +++ b/tests/unit/gemini-helper.test.ts @@ -20,7 +20,7 @@ test("DEFAULT_SAFETY_SETTINGS is an array", () => { test("tryParseJSON parses valid JSON", () => { assert.deepEqual(gemini.tryParseJSON('{"a":1}'), { a: 1 }); - assert.deepEqual(gemini.tryParseJSON('[1,2,3]'), [1, 2, 3]); + assert.deepEqual(gemini.tryParseJSON("[1,2,3]"), [1, 2, 3]); assert.equal(gemini.tryParseJSON('"hello"'), "hello"); assert.equal(gemini.tryParseJSON("42"), 42); assert.equal(gemini.tryParseJSON("true"), true); @@ -130,3 +130,37 @@ test("cleanJSONSchemaForAntigravity handles nested schema", () => { const result = gemini.cleanJSONSchemaForAntigravity(schema); assert.ok(typeof result === "object"); }); + +test("convertOpenAIContentToParts maps OpenAI Chat Completions file (PDF) to inlineData", () => { + const content = [ + { type: "text", text: "read this" }, + { + type: "file", + file: { filename: "doc.pdf", file_data: "data:application/pdf;base64,JVBERiAtMQ==" }, + }, + ]; + const parts = gemini.convertOpenAIContentToParts(content); + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "PDF file part must be converted to inlineData, not dropped"); + assert.equal(inline.inlineData.mimeType, "application/pdf"); + assert.equal(inline.inlineData.data, "JVBERiAtMQ=="); +}); + +test("convertOpenAIContentToParts keeps the real mime for a video file_data", () => { + const content = [ + { type: "file", file: { filename: "clip.mp4", file_data: "data:video/mp4;base64,AAAAIGZ0" } }, + ]; + const parts = gemini.convertOpenAIContentToParts(content); + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "video file part must be converted to inlineData"); + assert.equal(inline.inlineData.mimeType, "video/mp4"); + assert.equal(inline.inlineData.data, "AAAAIGZ0"); +}); + +test("convertOpenAIContentToParts still maps image_url data URIs (regression)", () => { + const content = [{ type: "image_url", image_url: { url: "data:image/png;base64,iVBORw0KGgo=" } }]; + const parts = gemini.convertOpenAIContentToParts(content); + const inline = parts.find((p) => p.inlineData); + assert.ok(inline, "image_url must still convert to inlineData"); + assert.equal(inline.inlineData.mimeType, "image/png"); +}); diff --git a/tests/unit/openai-to-claude-file-attachments.test.ts b/tests/unit/openai-to-claude-file-attachments.test.ts new file mode 100644 index 0000000000..727044251c --- /dev/null +++ b/tests/unit/openai-to-claude-file-attachments.test.ts @@ -0,0 +1,70 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { openaiToClaudeRequest } = + await import("../../open-sse/translator/request/openai-to-claude.ts"); + +function userBlocks(model: string, content: unknown) { + const translated = openaiToClaudeRequest(model, { messages: [{ role: "user", content }] }, false); + const userMsg = translated.messages.find((m) => m.role === "user"); + assert.ok(userMsg && Array.isArray(userMsg.content), "expected a translated user message"); + return userMsg.content; +} + +test("openaiToClaudeRequest maps an OpenAI file (PDF) block to a Claude document block", () => { + const blocks = userBlocks("claude-sonnet-4", [ + { type: "text", text: "summarize" }, + { + type: "file", + file: { filename: "edital.pdf", file_data: "data:application/pdf;base64,JVBERiAtMQ==" }, + }, + ]); + const doc = blocks.find((b) => b.type === "document"); + assert.ok(doc, "PDF file block must become a Claude document block, not be dropped"); + assert.equal(doc.source.type, "base64"); + assert.equal(doc.source.media_type, "application/pdf"); + assert.equal(doc.source.data, "JVBERiAtMQ=="); + assert.equal(doc.title, "edital.pdf"); +}); + +test("openaiToClaudeRequest maps an OpenAI file (image mime) block to a Claude image block", () => { + const blocks = userBlocks("claude-sonnet-4", [ + { + type: "file", + file: { filename: "shot.png", file_data: "data:image/png;base64,iVBORw0KGgo=" }, + }, + ]); + const img = blocks.find((b) => b.type === "image"); + assert.ok(img, "image-mime file block must become a Claude image block"); + assert.equal(img.source.type, "base64"); + assert.equal(img.source.media_type, "image/png"); + assert.equal(img.source.data, "iVBORw0KGgo="); +}); + +test("openaiToClaudeRequest maps a remote file (PDF url) block to a Claude document url block", () => { + const blocks = userBlocks("claude-sonnet-4", [ + { type: "file", file: { filename: "remote.pdf", file_data: "https://example.com/a.pdf" } }, + ]); + const doc = blocks.find((b) => b.type === "document"); + assert.ok(doc, "remote PDF file block must become a Claude document url block"); + assert.equal(doc.source.type, "url"); + assert.equal(doc.source.url, "https://example.com/a.pdf"); +}); + +test("openaiToClaudeRequest skips a video file block (Claude has no native video input)", () => { + const blocks = userBlocks("claude-sonnet-4", [ + { type: "text", text: "watch this" }, + { type: "file", file: { filename: "clip.mp4", file_data: "data:video/mp4;base64,AAAAIGZ0" } }, + ]); + const doc = blocks.find((b) => b.type === "document"); + const img = blocks.find((b) => b.type === "image"); + assert.ok( + !doc && !img, + "a video file must not be mislabeled as a Claude document or image block" + ); + // the text part is still forwarded + assert.ok( + blocks.some((b) => b.type === "text"), + "the accompanying text part must still be forwarded" + ); +});