From a280bfc11229e4afb71e45b3197621e79ed7553b Mon Sep 17 00:00:00 2001 From: Nguyen Thanh Dat Date: Fri, 21 Aug 2026 01:19:59 +0700 Subject: [PATCH] fix(context): budget base64 file payloads instead of counting them as text (#10858) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obrigado — um PDF de ~1MB enviado como file/document base64 (OpenAI ou Claude) era medido caractere-a-caractere, estimando 350.022 tokens (o mesmo documento pelo path Gemini inlineData já estimava 1.209). Corrige a inconsistência reconhecendo os shapes que faltavam, sem introduzir constante nova. Validação (worktree combinado a partir de origin/release/v3.8.50, 0 conflitos): - typecheck:core limpo, complexity/cognitive-complexity dentro do baseline - tests/unit/10840-file-token-context.test.ts — 5/5 passando - Suítes de contexto relacionadas — 59/59 (5 arquivos) passando --- .../fixes/10858-base64-file-token-estimate.md | 1 + open-sse/services/contextManager.ts | 56 ++++++++++ tests/unit/10840-file-token-context.test.ts | 101 ++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 changelog.d/fixes/10858-base64-file-token-estimate.md create mode 100644 tests/unit/10840-file-token-context.test.ts diff --git a/changelog.d/fixes/10858-base64-file-token-estimate.md b/changelog.d/fixes/10858-base64-file-token-estimate.md new file mode 100644 index 0000000000..18d8104b10 --- /dev/null +++ b/changelog.d/fixes/10858-base64-file-token-estimate.md @@ -0,0 +1 @@ +- **fix(context):** Base64 file payloads (OpenAI `file` parts, Responses `input_file`, Claude `document` blocks) are budgeted like the Gemini `inlineData` path instead of being counted as prompt text — a ~1MB PDF estimated at 350k tokens and was rejected on the context limit before reaching the provider's document pipeline ([#10840](https://github.com/diegosouzapw/OmniRoute/issues/10840), [#10858](https://github.com/diegosouzapw/OmniRoute/pull/10858)) — thanks @ntdat812 diff --git a/open-sse/services/contextManager.ts b/open-sse/services/contextManager.ts index c231759a07..a2d678f107 100644 --- a/open-sse/services/contextManager.ts +++ b/open-sse/services/contextManager.ts @@ -75,6 +75,13 @@ const CHARS_PER_TOKEN = 4; // see #8368 research notes. const IMAGE_TOKEN_ESTIMATE = 1200; +// #10840: same budget, deliberately. The Gemini `inlineData` matcher does not +// inspect media type, so a base64 PDF arriving in that shape is ALREADY measured +// at IMAGE_TOKEN_ESTIMATE today. Reusing it makes the OpenAI `file` and Claude +// `document` shapes agree with the estimate the same document already receives, +// rather than introducing a second constant with no grounding in this repo. +const DOCUMENT_TOKEN_ESTIMATE = IMAGE_TOKEN_ESTIMATE; + // Matches inline base64 data URLs, e.g. "data:image/png;base64,AAAA...". // Deliberately scoped to `data:image/...;base64,` so remote (http/https) // URLs and generic long base64 text strings stay on the text-estimation path. @@ -117,6 +124,45 @@ function matchesGeminiInlineDataShape(node: Record): boolean { return typeof (inlineData as Record).data === "string"; } +// Any inline base64 data URL, regardless of media type — file parts legitimately +// carry application/pdf, text/csv, and so on. +const INLINE_BASE64_DATA_RE = /^data:[^;,]+;base64,/; + +function isInlineBase64DataUrl(value: unknown): boolean { + return typeof value === "string" && INLINE_BASE64_DATA_RE.test(value); +} + +// OpenAI chat.completions: { type: 'file', file: { file_data | data: 'data:...;base64,...' } } +// Responses API: { type: 'input_file', file_data: 'data:...;base64,...' } +// Shapes mirror services/ccOpenAiMediaBlocks.ts::convertOpenAiMediaBlock. +function matchesOpenAIFileShape(node: Record): boolean { + if (node.type === "input_file") return isInlineBase64DataUrl(node.file_data); + if (node.type !== "file") return false; + const file = node.file; + if (!file || typeof file !== "object") return false; + const f = file as Record; + return isInlineBase64DataUrl(f.file_data) || isInlineBase64DataUrl(f.data); +} + +// Claude: { type: 'document', source: { type: 'base64', data: '...' } } +function matchesClaudeDocumentShape(node: Record): boolean { + if (node.type !== "document") return false; + const source = node.source; + if (!source || typeof source !== "object") return false; + const src = source as Record; + return src.type === "base64" && typeof src.data === "string"; +} + +/** + * Detect inline-base64 *document* blocks (#10840). Deliberately separate from + * {@link isInlineBase64ImageBlock}: that predicate also drives + * pruneOlderInlineImages, and dropping a user's attached PDF is not the same + * decision as dropping an old screenshot. This one only feeds token estimation. + */ +export function isInlineBase64DocumentBlock(node: Record): boolean { + return matchesOpenAIFileShape(node) || matchesClaudeDocumentShape(node); +} + /** * Detect the 5 documented inline-base64 image content-block shapes (see the * shape-specific matchers above). @@ -224,6 +270,10 @@ function extractImageTokens(node: unknown, seen: Set): { node: unknown; tokens += IMAGE_TOKEN_ESTIMATE; return { __image_token_estimate__: IMAGE_TOKEN_ESTIMATE }; } + if (record && isInlineBase64DocumentBlock(record)) { + tokens += DOCUMENT_TOKEN_ESTIMATE; + return { __document_token_estimate__: DOCUMENT_TOKEN_ESTIMATE }; + } const result = extractImageTokens(item, seen); tokens += result.tokens; return result.node; @@ -238,6 +288,12 @@ function extractImageTokens(node: unknown, seen: Set): { node: unknown; tokens: IMAGE_TOKEN_ESTIMATE, }; } + if (isInlineBase64DocumentBlock(record)) { + return { + node: { __document_token_estimate__: DOCUMENT_TOKEN_ESTIMATE }, + tokens: DOCUMENT_TOKEN_ESTIMATE, + }; + } let tokens = 0; const out: Record = {}; diff --git a/tests/unit/10840-file-token-context.test.ts b/tests/unit/10840-file-token-context.test.ts new file mode 100644 index 0000000000..68bd0f0eb3 --- /dev/null +++ b/tests/unit/10840-file-token-context.test.ts @@ -0,0 +1,101 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + estimateTokens, + isInlineBase64DocumentBlock, + isInlineBase64ImageBlock, + pruneOlderInlineImages, +} from "../../open-sse/services/contextManager.ts"; + +/** + * #10840 — a base64 file payload (PDF and friends) was measured as ordinary + * prompt text, so large documents were rejected on the context limit before + * ever reaching a provider's native document pipeline. + * + * The estimate must not depend on which wire shape the document arrived in: + * the Gemini `inlineData` matcher never inspected media type, so the SAME PDF + * was already budgeted at the bounded image estimate there while the OpenAI + * `file` and Claude `document` shapes were measured character by character. + */ + +function base64Payload(approxBytes: number): string { + return Buffer.alloc(approxBytes, 65).toString("base64"); +} + +const PDF_B64 = base64Payload(1_000_000); // ~1 MB document +const PDF_DATA_URL = `data:application/pdf;base64,${PDF_B64}`; + +const SHAPES: Array<[string, Record]> = [ + ["gemini inlineData", { inlineData: { mimeType: "application/pdf", data: PDF_B64 } }], + [ + "claude document", + { type: "document", source: { type: "base64", media_type: "application/pdf", data: PDF_B64 } }, + ], + ["openai file.file_data", { type: "file", file: { filename: "d.pdf", file_data: PDF_DATA_URL } }], + ["openai file.data", { type: "file", file: { filename: "d.pdf", data: PDF_DATA_URL } }], + ["responses input_file", { type: "input_file", filename: "d.pdf", file_data: PDF_DATA_URL }], +]; + +test("#10840: a base64 document is never measured as raw prompt text", () => { + for (const [name, block] of SHAPES) { + const tokens = estimateTokens({ + messages: [{ role: "user", content: [{ type: "text", text: "Summarise this." }, block] }], + }); + assert.ok( + tokens < 5_000, + `${name}: expected a bounded document estimate, got ${tokens} tokens for a ~1MB file` + ); + } +}); + +test("#10840: every wire shape of the same document agrees", () => { + const counts = SHAPES.map(([, block]) => estimateTokens(block)); + const unique = [...new Set(counts)]; + assert.equal( + unique.length, + 1, + `the same document must cost the same regardless of shape, got ${JSON.stringify( + SHAPES.map(([n], i) => `${n}=${counts[i]}`) + )}` + ); +}); + +test("#10840: a remote file URL still flows through the text path", () => { + // Not base64 transport — nothing to exclude, and it is short anyway. + const block = { type: "file", file: { filename: "d.pdf", file_data: "https://x.test/d.pdf" } }; + assert.equal(isInlineBase64DocumentBlock(block), false); +}); + +test("#10840: document detection stays separate from image detection", () => { + const doc = SHAPES[2][1]; + const img = { type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }; + + assert.equal(isInlineBase64DocumentBlock(doc), true); + assert.equal(isInlineBase64ImageBlock(doc), false, "a document must not register as an image"); + assert.equal(isInlineBase64ImageBlock(img), true); + assert.equal(isInlineBase64DocumentBlock(img), false); +}); + +test("#10840: pruneOlderInlineImages still ignores documents", () => { + // Dropping an attached PDF is not the same decision as dropping an old + // screenshot, so the pruner must keep its image-only scope. + const messages = [ + { + role: "user", + content: [{ type: "file", file: { filename: "a.pdf", file_data: PDF_DATA_URL } }], + }, + { + role: "user", + content: [{ type: "file", file: { filename: "b.pdf", file_data: PDF_DATA_URL } }], + }, + { + role: "user", + content: [{ type: "file", file: { filename: "c.pdf", file_data: PDF_DATA_URL } }], + }, + ]; + + const { pruned, messages: after } = pruneOlderInlineImages(messages, { keepLatest: 1 }); + + assert.equal(pruned, 0, "documents must not be pruned by the image pruner"); + assert.deepEqual(after, messages); +});