From b84c915b230181c80960fe0ef722be055e415041 Mon Sep 17 00:00:00 2001 From: tombii Date: Sat, 28 Mar 2026 16:23:40 +0100 Subject: [PATCH] fix(sse): preserve cache_control in Claude passthrough mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Claude Code routes through OmniRoute (Claude → OmniRoute → Claude), OmniRoute was stripping all cache_control markers and replacing them with its own generic caching strategy. This broke Claude Code's carefully placed cache breakpoints for plans and other features. Changes: - Add preserveCacheControl parameter to prepareClaudeRequest() - Detect Claude passthrough mode (sourceFormat === targetFormat === CLAUDE) - Skip cache_control normalization when preserveCacheControl=true - Preserve client's cache_control markers in system, messages, and tools This ensures Claude Code's prompt caching optimization works correctly while maintaining OmniRoute's caching strategy for translation scenarios. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 4 + open-sse/translator/helpers/claudeHelper.ts | 37 ++-- open-sse/translator/index.ts | 4 +- .../claude-cache-control-passthrough.test.mjs | 175 ++++++++++++++++++ 4 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 tests/unit/claude-cache-control-passthrough.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 56eaa57200..c9d723e61a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### 🐛 Bug Fixes + +- **Claude Prompt Caching Passthrough** — Fixed cache_control markers being stripped in Claude passthrough mode (Claude → OmniRoute → Claude), which caused Claude Code users to deplete their Anthropic API quota 5-10x faster than direct connections. OmniRoute now preserves client's cache_control markers when sourceFormat and targetFormat are both Claude, ensuring prompt caching works correctly and dramatically reducing token consumption. + ## [3.1.8] - 2026-03-27 ### 🐛 Bug Fixes & Features diff --git a/open-sse/translator/helpers/claudeHelper.ts b/open-sse/translator/helpers/claudeHelper.ts index e03d21e584..0b1a614ecb 100644 --- a/open-sse/translator/helpers/claudeHelper.ts +++ b/open-sse/translator/helpers/claudeHelper.ts @@ -105,13 +105,14 @@ function markMessageCacheControl(msg, ttl) { } // Prepare request for Claude format endpoints -// - Cleanup cache_control +// - Cleanup cache_control (unless preserveCacheControl=true for passthrough) // - Filter empty messages // - Add thinking block for Anthropic endpoint (provider === "claude") // - Fix tool_use/tool_result ordering -export function prepareClaudeRequest(body, provider = null) { +export function prepareClaudeRequest(body, provider = null, preserveCacheControl = false) { // 1. System: remove all cache_control, add only to last block with ttl 1h - if (body.system && Array.isArray(body.system)) { + // In passthrough mode, preserve existing cache_control markers + if (body.system && Array.isArray(body.system) && !preserveCacheControl) { body.system = body.system.map((block, i) => { const { cache_control, ...rest } = block; if (i === body.system.length - 1) { @@ -127,11 +128,12 @@ export function prepareClaudeRequest(body, provider = null) { let filtered = []; // Pass 1: remove cache_control + filter empty messages + // In passthrough mode, preserve existing cache_control markers for (let i = 0; i < len; i++) { const msg = body.messages[i]; - // Remove cache_control from content blocks - if (Array.isArray(msg.content)) { + // Remove cache_control from content blocks (skip in passthrough mode) + if (Array.isArray(msg.content) && !preserveCacheControl) { for (const block of msg.content) { delete block.cache_control; } @@ -177,14 +179,17 @@ export function prepareClaudeRequest(body, provider = null) { // Claude Code-style prompt caching: // - cache the second-to-last user turn for conversation reuse // - cache the last assistant turn so the next user turn can reuse it - const userMessageIndexes = filtered.reduce((indexes, msg, index) => { - if (msg?.role === "user") indexes.push(index); - return indexes; - }, []); - const secondToLastUserIndex = - userMessageIndexes.length >= 2 ? userMessageIndexes[userMessageIndexes.length - 2] : -1; - if (secondToLastUserIndex >= 0) { - markMessageCacheControl(filtered[secondToLastUserIndex]); + // Skip in passthrough mode to preserve client's cache_control markers + if (!preserveCacheControl) { + const userMessageIndexes = filtered.reduce((indexes, msg, index) => { + if (msg?.role === "user") indexes.push(index); + return indexes; + }, []); + const secondToLastUserIndex = + userMessageIndexes.length >= 2 ? userMessageIndexes[userMessageIndexes.length - 2] : -1; + if (secondToLastUserIndex >= 0) { + markMessageCacheControl(filtered[secondToLastUserIndex]); + } } // Pass 2 (reverse): add cache_control to last assistant + handle thinking for Anthropic @@ -194,7 +199,8 @@ export function prepareClaudeRequest(body, provider = null) { if (msg.role === "assistant" && Array.isArray(ensureMessageContentArray(msg))) { // Add cache_control to last block of first (from end) assistant with content - if (!lastAssistantProcessed && markMessageCacheControl(msg)) { + // Skip in passthrough mode to preserve client's cache_control markers + if (!preserveCacheControl && !lastAssistantProcessed && markMessageCacheControl(msg)) { lastAssistantProcessed = true; } @@ -227,7 +233,8 @@ export function prepareClaudeRequest(body, provider = null) { // 3. Tools: remove all cache_control, add only to last non-deferred tool with ttl 1h // Tools with defer_loading=true cannot have cache_control (API rejects it) - if (body.tools && Array.isArray(body.tools)) { + // In passthrough mode, preserve existing cache_control markers + if (body.tools && Array.isArray(body.tools) && !preserveCacheControl) { body.tools = body.tools.map((tool) => { const { cache_control, ...rest } = tool; return rest; diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 25f70900fe..9dfb4b31ce 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -144,8 +144,10 @@ export function translateRequest( } // Final step: prepare request for Claude format endpoints + // In Claude passthrough mode (Claude → Claude), preserve cache_control markers if (targetFormat === FORMATS.CLAUDE) { - result = prepareClaudeRequest(result, provider); + const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE; + result = prepareClaudeRequest(result, provider, isClaudePassthrough); } // Normalize openai-responses input shape for providers that require list input. diff --git a/tests/unit/claude-cache-control-passthrough.test.mjs b/tests/unit/claude-cache-control-passthrough.test.mjs new file mode 100644 index 0000000000..5014d04992 --- /dev/null +++ b/tests/unit/claude-cache-control-passthrough.test.mjs @@ -0,0 +1,175 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { prepareClaudeRequest } from "../../open-sse/translator/helpers/claudeHelper.ts"; + +describe("Claude cache_control passthrough", () => { + test("preserveCacheControl=true preserves cache_control in system blocks", () => { + const body = { + system: [ + { type: "text", text: "System prompt 1" }, + { type: "text", text: "System prompt 2", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.system.length, 2); + assert.equal(result.system[0].cache_control, undefined); + assert.deepEqual(result.system[1].cache_control, { type: "ephemeral", ttl: "5m" }); + }); + + test("preserveCacheControl=false replaces cache_control in system blocks", () => { + const body = { + system: [ + { type: "text", text: "System prompt 1" }, + { type: "text", text: "System prompt 2", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + assert.equal(result.system.length, 2); + assert.equal(result.system[0].cache_control, undefined); + assert.deepEqual(result.system[1].cache_control, { type: "ephemeral", ttl: "1h" }); + }); + + test("preserveCacheControl=true preserves cache_control in message content blocks", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "User message 1" }, + { type: "text", text: "User message 2", cache_control: { type: "ephemeral" } }, + ], + }, + { + role: "assistant", + content: [ + { type: "text", text: "Assistant response", cache_control: { type: "ephemeral", ttl: "10m" } }, + ], + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.messages.length, 2); + assert.equal(result.messages[0].content[0].cache_control, undefined); + assert.deepEqual(result.messages[0].content[1].cache_control, { type: "ephemeral" }); + assert.deepEqual(result.messages[1].content[0].cache_control, { type: "ephemeral", ttl: "10m" }); + }); + + test("preserveCacheControl=false strips and re-adds cache_control in messages", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "User message 1" }, + { type: "text", text: "User message 2", cache_control: { type: "ephemeral" } }, + ], + }, + { + role: "assistant", + content: [ + { type: "text", text: "Assistant response", cache_control: { type: "ephemeral", ttl: "10m" } }, + ], + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + // Original cache_control should be stripped and OmniRoute's strategy applied + assert.equal(result.messages.length, 2); + // User message should not have cache_control (only second-to-last user gets it) + assert.equal(result.messages[0].content[0].cache_control, undefined); + assert.equal(result.messages[0].content[1].cache_control, undefined); + // Last assistant should have cache_control added by OmniRoute + assert.deepEqual(result.messages[1].content[0].cache_control, { type: "ephemeral" }); + }); + + test("preserveCacheControl=true preserves cache_control in tools", () => { + const body = { + messages: [], + tools: [ + { name: "tool1", description: "Tool 1", input_schema: { type: "object" } }, + { + name: "tool2", + description: "Tool 2", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "30m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.tools.length, 2); + assert.equal(result.tools[0].cache_control, undefined); + assert.deepEqual(result.tools[1].cache_control, { type: "ephemeral", ttl: "30m" }); + }); + + test("preserveCacheControl=false replaces cache_control in tools", () => { + const body = { + messages: [], + tools: [ + { name: "tool1", description: "Tool 1", input_schema: { type: "object" } }, + { + name: "tool2", + description: "Tool 2", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "30m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + assert.equal(result.tools.length, 2); + assert.equal(result.tools[0].cache_control, undefined); + assert.deepEqual(result.tools[1].cache_control, { type: "ephemeral", ttl: "1h" }); + }); + + test("preserveCacheControl=true with Claude Code-style caching", () => { + const body = { + system: [ + { type: "text", text: "System", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [ + { + role: "user", + content: [{ type: "text", text: "Turn 1", cache_control: { type: "ephemeral" } }], + }, + { + role: "assistant", + content: [{ type: "text", text: "Response 1" }], + }, + { + role: "user", + content: [{ type: "text", text: "Turn 2" }], + }, + ], + tools: [ + { + name: "bash", + description: "Execute bash", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + // All original cache_control should be preserved + assert.deepEqual(result.system[0].cache_control, { type: "ephemeral", ttl: "5m" }); + assert.deepEqual(result.messages[0].content[0].cache_control, { type: "ephemeral" }); + assert.equal(result.messages[1].content[0].cache_control, undefined); + assert.equal(result.messages[2].content[0].cache_control, undefined); + assert.deepEqual(result.tools[0].cache_control, { type: "ephemeral", ttl: "5m" }); + }); +});