From de9fe1a231d45a26cdbd4431decf8354ac5cc4f8 Mon Sep 17 00:00:00 2001 From: Aaron Scherer <896295+cryptiklemur@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:41:54 -0500 Subject: [PATCH] fix(sse): preserve Claude Code cache breakpoints (#8934) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated in local merge-train (devbox-vm-06-dev002) @ combined-tip (FAST gates green: static + changed tests + vitest — only pre-existing audit.test.ts flake). Evidence: /home/diegosouzapw/dev/proxys/OmniRoute/.claude/worktrees/merge-train-20260805-213228-suite.log --- open-sse/executors/claudeIdentity.ts | 20 +++- open-sse/handlers/chatCore.ts | 21 ++-- open-sse/services/claudeCodeConstraints.ts | 14 +++ tests/unit/chatcore-translation-paths.test.ts | 100 ++++++++++++++++++ tests/unit/claude-code-parity.test.ts | 48 ++++++++- 5 files changed, 191 insertions(+), 12 deletions(-) diff --git a/open-sse/executors/claudeIdentity.ts b/open-sse/executors/claudeIdentity.ts index 8fed8fa597..68cc708e4a 100644 --- a/open-sse/executors/claudeIdentity.ts +++ b/open-sse/executors/claudeIdentity.ts @@ -323,6 +323,23 @@ function isContext1mModel(model: unknown): boolean { ); } +export function shouldUseMidConversationSystem( + body: Record | null | undefined, + model?: string | null +): boolean { + const payload = body || {}; + const hasSystem = + !!payload.system && + (typeof payload.system === "string" || + (Array.isArray(payload.system) && payload.system.length > 0)); + const hasTools = Array.isArray(payload.tools) && payload.tools.length > 0; + const effectiveModel = model ?? (typeof payload.model === "string" ? payload.model : ""); + + return ( + hasSystem && hasTools && matchesModelPrefix(effectiveModel, CONTEXT_1M_BETA_MODEL_PREFIXES) + ); +} + /** * Pick the anthropic-beta flag set that matches the request shape. Real CLI * uses three patterns: minimal probe, structured-output, and full agent. @@ -375,8 +392,7 @@ export function selectBetaFlags( const isFullAgent = hasTools && hasSystem; const effectiveModel = model ?? (typeof b.model === "string" ? b.model : ""); const isHeavyAgent = isFullAgent && isHeavyAgentModel(effectiveModel); - const isOpusAgent = - isFullAgent && matchesModelPrefix(effectiveModel, CONTEXT_1M_BETA_MODEL_PREFIXES); + const isOpusAgent = shouldUseMidConversationSystem(b, effectiveModel); const isContext1m = isFullAgent && isContext1mModel(effectiveModel); const flags: string[] = []; diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 028590a09e..4eaa24ff57 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -79,6 +79,7 @@ import { FORMATS } from "../translator/formats.ts"; import { collectCustomToolNamesForSourceFormat } from "../translator/request/openai-responses/additionalTools.ts"; import { sanitizeKiroTools } from "../utils/kiroSanitizer.ts"; import { splitMisplacedToolResults } from "../translator/helpers/claudeHelper.ts"; +import { ensureCacheControlOnLastUserMessage } from "../services/claudeCodeConstraints.ts"; import { createSSETransformStreamWithLogger, createPassthroughStreamWithLogger, @@ -119,6 +120,7 @@ import { normalizeClaudeAdaptiveThinking, normalizeClaudeDisabledThinkingEffort, } from "../services/claudeAdaptiveThinking.ts"; +import { shouldUseMidConversationSystem } from "../executors/claudeIdentity.ts"; import { normalizeClaudeHaikuConstraints } from "../services/claudeHaikuConstraints.ts"; import { applyDefaultReasoningEffort } from "../services/defaultReasoningEffort.ts"; import { echoModelInObject } from "../services/responseModelEcho.ts"; @@ -2065,20 +2067,23 @@ export async function handleChatCore({ } } - // Fix #2468: always extract role:"system" → top-level system. - // The semantic passthrough correctly skips the Claude→OpenAI→Claude - // round-trip, but even pure Claude bodies may carry system content as - // role:"system" messages rather than the top-level system field, which - // Anthropic's Messages API now rejects with a 400. + // Legacy models reject role:"system" messages. Opus accepts them behind + // its beta, and hoisting them breaks the prompt cache prefix. if (isClaudeCodeSemanticPassthrough) { - // Only lift system/developer messages — preserves Claude Code's - // native payload structure (documents, tool chains, thinking, etc.) - extractSystemRoleMessages(translatedBody); + if ( + provider !== "claude" || + !shouldUseMidConversationSystem(translatedBody, effectiveModel) + ) { + extractSystemRoleMessages(translatedBody); + } if (Array.isArray(translatedBody.messages)) { translatedBody.messages = splitMisplacedToolResults( translatedBody.messages as ClaudeMessage[] ) as typeof translatedBody.messages; } + if (provider === "claude") { + ensureCacheControlOnLastUserMessage(translatedBody); + } } else { normalizeClaudeUpstreamMessages(translatedBody, { preserveToolResultBlocks: true }); } diff --git a/open-sse/services/claudeCodeConstraints.ts b/open-sse/services/claudeCodeConstraints.ts index af17a3657c..ddac177b32 100644 --- a/open-sse/services/claudeCodeConstraints.ts +++ b/open-sse/services/claudeCodeConstraints.ts @@ -128,6 +128,20 @@ export function ensureCacheControlOnLastUserMessage(body: Record> | undefined; if (!Array.isArray(messages) || messages.length === 0) return; + const system = body.system as Array> | undefined; + const systemCacheControlCount = Array.isArray(system) + ? system.filter((block) => block.cache_control).length + : 0; + + for (const message of messages) { + const content = message.content as Array> | undefined; + if (Array.isArray(content) && content.some((block) => block.cache_control)) { + return; + } + } + + if (systemCacheControlCount >= MAX_CACHE_CONTROL_BLOCKS) return; + // Find the last user message for (let i = messages.length - 1; i >= 0; i--) { if (String(messages[i].role) === "user") { diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index e5dd816f4a..a3bea6c861 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -762,6 +762,60 @@ test("chatCore normalizes native Claude Code messages for native Claude OAuth pa assert.equal(call.body.messages[2].content[0].type, "tool_result"); }); +test("chatCore preserves Opus 5 mid-conversation system cache breakpoints", async () => { + await settingsDb.updateSettings({ alwaysPreserveClientCache: "auto" }); + invalidateCacheControlSettingsCache(); + + const { call, result } = await invokeChatCore({ + provider: "claude", + model: "claude-opus-5", + endpoint: "/v1/messages", + credentials: { apiKey: "claude-key", providerSpecificData: {} }, + body: { + model: "claude-opus-5", + max_tokens: 64, + system: [ + { + type: "text", + text: "stable system prompt", + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + ], + messages: [ + { role: "user", content: [{ type: "text", text: "first turn" }] }, + { role: "assistant", content: [{ type: "text", text: "first response" }] }, + { + role: "system", + content: [ + { + type: "text", + text: "compact continuation", + cache_control: { type: "ephemeral" }, + }, + ], + }, + { role: "user", content: [{ type: "text", text: "latest turn" }] }, + ], + tools: [{ name: "Bash", input_schema: { type: "object", properties: {} } }], + }, + userAgent: "Claude-Code/2.1.220", + requestHeaders: { "x-app": "cli", "x-claude-code-session-id": "session-123" }, + responseFormat: "claude", + }); + + assert.equal(result.success, true); + assert.deepEqual( + call.body.messages.map((message: { role: string }) => message.role), + ["user", "assistant", "system", "user"] + ); + assert.deepEqual(call.body.messages[2].content[0].cache_control, { type: "ephemeral" }); + assert.equal( + call.body.system.some((block: { text?: string }) => block.text === "compact continuation"), + false + ); + assert.equal(call.body.messages[3].content[0].cache_control, undefined); +}); + test("chatCore keeps Claude normalization for non-Claude-Code Claude passthrough", async () => { const { call, result } = await invokeChatCore({ provider: "claude", @@ -937,6 +991,52 @@ test("chatCore preserves cache_control automatically for Claude Code single-mode assert.equal(call.body.tools[0].cache_control, undefined); }); +test("chatCore supplements a missing message cache breakpoint for native Claude Code requests", async () => { + await settingsDb.updateSettings({ alwaysPreserveClientCache: "auto" }); + invalidateCacheControlSettingsCache(); + + const { call } = await invokeChatCore({ + provider: "claude", + model: "claude-sonnet-4-6", + endpoint: "/v1/messages", + credentials: { apiKey: "claude-key", providerSpecificData: {} }, + body: { + model: "claude-sonnet-4-6", + max_tokens: 64, + system: [ + { + type: "text", + text: "stable system prompt", + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + { + type: "text", + text: "stable project instructions", + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + ], + messages: [ + { role: "user", content: [{ type: "text", text: "first turn" }] }, + { role: "assistant", content: [{ type: "text", text: "first response" }] }, + { role: "user", content: [{ type: "text", text: "latest turn" }] }, + ], + tools: [ + { + name: "lookup_weather", + description: "Fetch weather", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + ], + }, + userAgent: "Claude-Code/1.0.0", + responseFormat: "claude", + }); + + assert.deepEqual(call.body.messages[2].content[0].cache_control, { type: "ephemeral" }); + assert.equal(call.body.tools[0].cache_control, undefined); +}); + test("chatCore auto cache policy becomes false for nondeterministic combos", async () => { await settingsDb.updateSettings({ alwaysPreserveClientCache: "auto" }); invalidateCacheControlSettingsCache(); diff --git a/tests/unit/claude-code-parity.test.ts b/tests/unit/claude-code-parity.test.ts index 42022464ad..480f76a177 100644 --- a/tests/unit/claude-code-parity.test.ts +++ b/tests/unit/claude-code-parity.test.ts @@ -341,15 +341,59 @@ describe("enforceCacheControlLimit", () => { }); describe("ensureCacheControlOnLastUserMessage", () => { - it("does not throw on a valid messages array", () => { + it("adds a breakpoint to the last user message when messages have none", () => { const body = { + system: [ + { type: "text", text: "s1", cache_control: { type: "ephemeral" } }, + { type: "text", text: "s2", cache_control: { type: "ephemeral" } }, + ], messages: [ { role: "user", content: [{ type: "text", text: "Hello" }] }, { role: "assistant", content: [{ type: "text", text: "Hi!" }] }, { role: "user", content: [{ type: "text", text: "Follow up" }] }, ], }; - assert.doesNotThrow(() => ensureCacheControlOnLastUserMessage(body)); + + ensureCacheControlOnLastUserMessage(body); + + assert.deepEqual(body.messages[2].content[0].cache_control, { type: "ephemeral" }); + }); + + it("keeps an existing message breakpoint without adding another", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "Hello", + cache_control: { type: "ephemeral" }, + }, + ], + }, + { role: "user", content: [{ type: "text", text: "Follow up" }] }, + ], + }; + + ensureCacheControlOnLastUserMessage(body); + + assert.equal(body.messages[1].content[0].cache_control, undefined); + }); + + it("does not exceed four surviving system and message breakpoints", () => { + const body = { + system: Array.from({ length: 4 }, (_, index) => ({ + type: "text", + text: `s${index}`, + cache_control: { type: "ephemeral" }, + })), + messages: [{ role: "user", content: [{ type: "text", text: "Hello" }] }], + }; + + ensureCacheControlOnLastUserMessage(body); + + assert.equal(body.messages[0].content[0].cache_control, undefined); }); it("handles body without messages without throwing", () => {