diff --git a/open-sse/services/claudeCodeCompatible.ts b/open-sse/services/claudeCodeCompatible.ts index de5de80369..0472fe5840 100644 --- a/open-sse/services/claudeCodeCompatible.ts +++ b/open-sse/services/claudeCodeCompatible.ts @@ -166,12 +166,18 @@ export function buildClaudeCodeCompatibleRequest({ systemBlocks: preparedClaudeBody?.system as Record[] | undefined, cwd, now, + preserveCacheControl, }); const resolvedSessionId = sessionId || randomUUID(); const effort = resolveClaudeCodeCompatibleEffort(sourceBody, normalizedBody, model); const maxTokens = resolveClaudeCodeCompatibleMaxTokens(sourceBody, normalizedBody); const tools = preparedClaudeBody?.tools - ? (preparedClaudeBody.tools as Record[]) + ? applyClaudeCodeCompatibleToolCacheStrategy( + preparedClaudeBody.tools as Record[], + { + preserveExisting: preserveCacheControl, + } + ) : buildClaudeCodeCompatibleTools(normalizedBody, sourceBody); const toolChoice = tools.length > 0 @@ -351,8 +357,10 @@ function buildClaudeCodeCompatibleMessagesFromClaude( for (const message of merged) { stripCacheControlFromContentBlocks(message.content); } - applyClaudeCodeCompatibleMessageCacheStrategy(merged); } + applyClaudeCodeCompatibleMessageCacheStrategy(merged, { + preserveExisting: preserveCacheControl, + }); if (merged.length === 0) { const fallbackText = converted @@ -379,11 +387,13 @@ function buildClaudeCodeCompatibleSystemBlocks({ systemBlocks, cwd, now, + preserveCacheControl, }: { messages: MessageLike[] | undefined; systemBlocks?: Array> | undefined; cwd: string; now: Date; + preserveCacheControl: boolean; }) { const customSystemBlocks = Array.isArray(systemBlocks) && systemBlocks.length > 0 @@ -412,6 +422,15 @@ function buildClaudeCodeCompatibleSystemBlocks({ blocks.push(systemBlock); } + if ( + preserveCacheControl && + customSystemBlocks.length > 0 && + !customSystemBlocks.some((block) => hasCacheControl(block)) + ) { + const lastCustomSystemBlock = customSystemBlocks[customSystemBlocks.length - 1]; + lastCustomSystemBlock.cache_control = { type: "ephemeral", ttl: "1h" }; + } + return blocks; } @@ -448,12 +467,10 @@ function buildClaudeCodeCompatibleTools( return rawTools .map((tool) => convertClaudeCodeCompatibleTool(tool)) .filter((tool): tool is Record => !!tool) + .map((tool) => ({ ...tool })) .map((tool, index, tools) => { if (index !== findLastCacheableToolIndex(tools)) return tool; - return { - ...tool, - cache_control: { type: "ephemeral", ttl: "1h" }, - }; + return { ...tool, cache_control: { type: "ephemeral", ttl: "1h" } }; }); } @@ -633,7 +650,7 @@ function convertClaudeCodeCompatibleClaudeMessage( function extractCustomSystemBlocks(messages: MessageLike[] | undefined) { if (!Array.isArray(messages)) return []; - return messages + const blocks = messages .filter((message) => { const role = String(message?.role || "").toLowerCase(); return role === "system" || role === "developer"; @@ -645,11 +662,21 @@ function extractCustomSystemBlocks(messages: MessageLike[] | undefined) { text, cache_control: { type: "ephemeral" }, })); + + if (blocks.length > 0) { + blocks[blocks.length - 1].cache_control = { type: "ephemeral", ttl: "1h" }; + } + + return blocks; } function applyClaudeCodeCompatibleMessageCacheStrategy( - messages: Array<{ role: "user" | "assistant"; content: Array> }> + messages: Array<{ role: "user" | "assistant"; content: Array> }>, + options: { + preserveExisting?: boolean; + } = {} ) { + const preserveExisting = options.preserveExisting === true; const userIndexes = messages.reduce((indexes, message, index) => { if (message.role === "user") indexes.push(index); return indexes; @@ -658,14 +685,28 @@ function applyClaudeCodeCompatibleMessageCacheStrategy( const secondToLastUserIndex = userIndexes.length >= 2 ? userIndexes[userIndexes.length - 2] : -1; if (secondToLastUserIndex >= 0) { - markLastContentCacheControl(messages[secondToLastUserIndex].content); + markLastContentCacheControl( + messages[secondToLastUserIndex].content, + undefined, + preserveExisting + ); } else if (!hasAssistant && userIndexes.length > 0) { - markLastContentCacheControl(messages[userIndexes[userIndexes.length - 1]].content); + markLastContentCacheControl( + messages[userIndexes[userIndexes.length - 1]].content, + undefined, + preserveExisting + ); + } + + const lastUserIndex = userIndexes.length > 0 ? userIndexes[userIndexes.length - 1] : -1; + const endsOnUser = lastUserIndex === messages.length - 1; + if (endsOnUser && lastUserIndex >= 0) { + markLastContentCacheControl(messages[lastUserIndex].content, undefined, preserveExisting); } for (let i = messages.length - 1; i >= 0; i--) { if (messages[i].role !== "assistant") continue; - if (markLastContentCacheControl(messages[i].content)) break; + if (markLastContentCacheControl(messages[i].content, undefined, preserveExisting)) break; } } @@ -675,14 +716,52 @@ function stripCacheControlFromContentBlocks(content: Array>, ttl?: string) { +function applyClaudeCodeCompatibleToolCacheStrategy( + tools: Array>, + options: { + preserveExisting?: boolean; + } = {} +) { + const preparedTools = tools.map((tool) => ({ ...tool })); + const lastCacheableToolIndex = findLastCacheableToolIndex(preparedTools); + if (lastCacheableToolIndex < 0) return preparedTools; + + if (options.preserveExisting && preparedTools.some((tool) => hasCacheControl(tool))) { + return preparedTools; + } + + preparedTools[lastCacheableToolIndex].cache_control = { type: "ephemeral", ttl: "1h" }; + return preparedTools; +} + +function markLastContentCacheControl( + content: Array>, + ttl?: string, + preserveExisting = false +) { if (!Array.isArray(content) || content.length === 0) return false; const lastBlock = content[content.length - 1]; if (!lastBlock) return false; + if ( + preserveExisting && + content.some( + (block) => + !!block && + typeof block === "object" && + !!readRecord(block)?.cache_control && + typeof readRecord(block)?.cache_control === "object" + ) + ) { + return true; + } lastBlock.cache_control = ttl ? { type: "ephemeral", ttl } : { type: "ephemeral" }; return true; } +function hasCacheControl(value: unknown) { + return !!readRecord(value)?.cache_control && typeof readRecord(value)?.cache_control === "object"; +} + function findLastCacheableToolIndex(tools: Array>) { for (let i = tools.length - 1; i >= 0; i--) { if (!tools[i].defer_loading) { diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index 0810674bb1..cd538e2ebf 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -148,6 +148,13 @@ export function shouldPreserveCacheControl({ return false; } + // CC-compatible bridges should default to OmniRoute-managed cache markers. + // Their request shape differs from native Claude Messages payloads, so + // preserving client markers in auto mode weakens cache coverage. + if (typeof targetProvider === "string" && targetProvider.startsWith("anthropic-compatible-cc-")) { + return false; + } + // Auto mode: use automatic detection (existing logic) // Must be a caching-aware client if (!isClaudeCodeClient(userAgent)) { diff --git a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx index 774552da93..f6b1fe35f7 100644 --- a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx @@ -277,7 +277,7 @@ export default function RoutingTab() { { value: "auto", label: "Auto (Recommended)", - desc: "Preserve cache_control only for caching-aware clients (Claude Code) with deterministic routing", + desc: "Preserve cache_control for native Claude-compatible flows with deterministic routing; CC-compatible bridges use OmniRoute-managed markers", }, { value: "always", diff --git a/tests/unit/cache-control-claude-providers.test.mjs b/tests/unit/cache-control-claude-providers.test.mjs index ae437cdf89..820a1676cd 100644 --- a/tests/unit/cache-control-claude-providers.test.mjs +++ b/tests/unit/cache-control-claude-providers.test.mjs @@ -138,4 +138,19 @@ describe("Cache Control Policy - Claude Protocol Providers", () => { false ); }); + + test("shouldPreserveCacheControl defaults CC-compatible providers to OmniRoute-managed cache in auto mode", () => { + const claudeCodeUA = "Claude-Code/1.0.0"; + + assert.equal( + shouldPreserveCacheControl({ + userAgent: claudeCodeUA, + isCombo: false, + targetProvider: "anthropic-compatible-cc-cm", + targetFormat: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + false + ); + }); }); diff --git a/tests/unit/cc-compatible-provider.test.mjs b/tests/unit/cc-compatible-provider.test.mjs index 022668a1e9..4060f825d5 100644 --- a/tests/unit/cc-compatible-provider.test.mjs +++ b/tests/unit/cc-compatible-provider.test.mjs @@ -106,7 +106,7 @@ test("buildClaudeCodeCompatibleRequest keeps prior role history while dropping t ); assert.deepEqual(payload.messages[0].content.at(-1).cache_control, { type: "ephemeral" }); assert.deepEqual(payload.messages[1].content.at(-1).cache_control, { type: "ephemeral" }); - assert.equal(payload.messages[2].content.at(-1).cache_control, undefined); + assert.deepEqual(payload.messages[2].content.at(-1).cache_control, { type: "ephemeral" }); assert.equal(payload.system.length, 4); assert.equal(payload.system.at(-1).text, "sys"); assert.equal(payload.tools.length, 1); @@ -182,10 +182,90 @@ test("buildClaudeCodeCompatibleRequest preserves Claude cache markers when reque type: "ephemeral", ttl: "10m", }); - assert.equal(payload.messages[2].content[0].cache_control, undefined); + assert.deepEqual(payload.messages[2].content[0].cache_control, { type: "ephemeral" }); assert.deepEqual(payload.tools[0].cache_control, { type: "ephemeral", ttl: "30m" }); }); +test("buildClaudeCodeCompatibleRequest supplements missing Claude cache markers in preserve mode", () => { + const payload = buildClaudeCodeCompatibleRequest({ + sourceBody: { + max_tokens: 64, + }, + normalizedBody: { + max_tokens: 64, + messages: [{ role: "user", content: "fallback" }], + }, + claudeBody: { + system: [{ type: "text", text: "sys" }], + messages: [ + { + role: "user", + content: [{ type: "text", text: "u1" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "a1" }], + }, + { role: "user", content: [{ type: "text", text: "u2" }] }, + ], + tools: [ + { + name: "lookup_weather", + description: "Fetch weather", + input_schema: { + type: "object", + properties: { + city: { type: "string" }, + }, + required: ["city"], + }, + }, + ], + }, + model: "claude-sonnet-4-6", + sessionId: "session-preserve-defaults", + preserveCacheControl: true, + }); + + assert.deepEqual(payload.messages[0].content[0].cache_control, { type: "ephemeral" }); + assert.deepEqual(payload.messages[1].content[0].cache_control, { type: "ephemeral" }); + assert.deepEqual(payload.messages[2].content[0].cache_control, { type: "ephemeral" }); + assert.deepEqual(payload.tools[0].cache_control, { type: "ephemeral", ttl: "1h" }); +}); + +test("buildClaudeCodeCompatibleRequest marks final user turn and 1h system cache in non-preserve mode", () => { + const largeUserPrompt = Array.from( + { length: 200 }, + (_, index) => `Context line ${index + 1}: repeated stable context for cache testing.` + ).join("\n"); + + const payload = buildClaudeCodeCompatibleRequest({ + sourceBody: { + max_tokens: 64, + }, + normalizedBody: { + max_tokens: 64, + messages: [ + { role: "system", content: "Follow the house style exactly." }, + { role: "user", content: "[Start a new chat]" }, + { role: "assistant", content: "Hello short ack" }, + { role: "user", content: largeUserPrompt }, + ], + }, + model: "claude-sonnet-4-6", + sessionId: "session-last-user-cache", + preserveCacheControl: false, + }); + + assert.deepEqual(payload.system.at(-1).cache_control, { + type: "ephemeral", + ttl: "1h", + }); + assert.deepEqual(payload.messages[0].content[0].cache_control, { type: "ephemeral" }); + assert.deepEqual(payload.messages[1].content[0].cache_control, { type: "ephemeral" }); + assert.deepEqual(payload.messages[2].content[0].cache_control, { type: "ephemeral" }); +}); + test("buildClaudeCodeCompatibleRequest falls back to a user turn when the source only has assistant/model text", () => { const payload = buildClaudeCodeCompatibleRequest({ sourceBody: { @@ -419,7 +499,7 @@ test("handleChatCore forces upstream streaming for CC compatible while returning assert.equal(payload.usage.completion_tokens, 5); }); -test("handleChatCore preserves Claude cache_control when CC-compatible requests originate from Claude", async () => { +test("handleChatCore applies OmniRoute-managed cache strategy for CC-compatible requests in auto mode", async () => { const calls = []; globalThis.fetch = async (url, init = {}) => { calls.push({ @@ -524,18 +604,20 @@ test("handleChatCore preserves Claude cache_control when CC-compatible requests assert.equal(calls.length, 1); assert.deepEqual(calls[0].body.system.at(-1).cache_control, { type: "ephemeral", - ttl: "5m", + ttl: "1h", }); assert.deepEqual(calls[0].body.messages[0].content[0].cache_control, { type: "ephemeral", }); assert.deepEqual(calls[0].body.messages[1].content[0].cache_control, { type: "ephemeral", - ttl: "10m", + }); + assert.deepEqual(calls[0].body.messages[2].content[0].cache_control, { + type: "ephemeral", }); assert.deepEqual(calls[0].body.tools[0].cache_control, { type: "ephemeral", - ttl: "30m", + ttl: "1h", }); });