From 0bc557fb8b5ca640626793ce474e22f3948b34cf Mon Sep 17 00:00:00 2001 From: tombii Date: Sun, 29 Mar 2026 12:24:44 +0200 Subject: [PATCH] feat(sse): preserve client cache_control for Claude Code with deterministic routing Adds intelligent cache control preservation for Claude Code clients: - New cacheControlPolicy.ts module with detection logic: - isClaudeCodeClient(): Detects Claude Code via User-Agent - providerSupportsCaching(): Checks provider (claude, anthropic, zai, qwen) - isDeterministicStrategy(): Identifies priority/cost-optimized strategies - shouldPreserveCacheControl(): Main policy decision - Cache control is preserved when: 1. Client is Claude Code (detected via User-Agent) 2. Provider supports prompt caching 3. Request routing is deterministic: - Single model requests (always) - Combo with priority or cost-optimized strategy only - Updated translator to accept preserveCacheControl option - Updated chatCore and chat handler to propagate combo strategy - Added comprehensive unit tests (24 tests) Non-deterministic combo strategies (weighted, round-robin, random, etc.) continue to use OmniRoute's managed caching strategy. Refs: #cache-control-preservation Co-Authored-By: Claude Sonnet 4.6 --- open-sse/handlers/chatCore.ts | 29 ++- open-sse/translator/index.ts | 10 +- open-sse/utils/cacheControlPolicy.ts | 100 +++++++++ src/sse/handlers/chat.ts | 24 ++- tests/unit/cache-control-policy.test.mjs | 252 +++++++++++++++++++++++ 5 files changed, 404 insertions(+), 11 deletions(-) create mode 100644 open-sse/utils/cacheControlPolicy.ts create mode 100644 tests/unit/cache-control-policy.test.mjs diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 4cdbf8d139..2e49e0cca5 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2,6 +2,7 @@ import { getCorsOrigin } from "../utils/cors.ts"; import { detectFormatFromEndpoint, getTargetFormat } from "../services/provider.ts"; import { translateRequest, needsTranslation } from "../translator/index.ts"; import { FORMATS } from "../translator/formats.ts"; +import { shouldPreserveCacheControl } from "../utils/cacheControlPolicy.ts"; import { createSSETransformStreamWithLogger, createPassthroughStreamWithLogger, @@ -306,6 +307,10 @@ function attachLogMeta( * @param {function} options.onDisconnect - Callback when client disconnects * @param {string} options.connectionId - Connection ID for usage tracking * @param {object} options.apiKeyInfo - API key metadata for usage attribution + * @param {string} options.userAgent - Client user agent for caching decisions + * @param {string} options.comboName - Combo name if this is a combo request + * @param {string} options.comboStrategy - Combo routing strategy (e.g., 'priority', 'cost-optimized') + * @param {boolean} options.isCombo - Whether this request is from a combo */ export async function handleChatCore({ body, @@ -320,6 +325,8 @@ export async function handleChatCore({ apiKeyInfo = null, userAgent, comboName, + comboStrategy = null, + isCombo = false, }) { let { provider, model, extendedContext } = modelInfo; const requestedModel = @@ -674,6 +681,22 @@ export async function handleChatCore({ // Translate request (pass reqLogger for intermediate logging) let translatedBody = body; const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; + + // Determine if we should preserve client-side cache_control headers + const preserveCacheControl = shouldPreserveCacheControl({ + userAgent, + isCombo, + comboStrategy, + targetProvider: provider, + }); + + if (preserveCacheControl) { + log?.debug?.( + "CACHE", + `Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})` + ); + } + try { if (nativeCodexPassthrough) { translatedBody = { ...body, _nativeCodexPassthrough: true }; @@ -701,7 +724,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); translatedBody = translateRequest( FORMATS.OPENAI, @@ -712,7 +735,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); log?.debug?.("FORMAT", "claude->openai->claude normalized passthrough"); } else { @@ -816,7 +839,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); } } catch (error) { diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 10f3675272..79be284b99 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -73,6 +73,7 @@ function normalizeOpenAIResponsesRequest(body) { /** @param options.normalizeToolCallId - When true, use 9-char tool call ids (e.g. Mistral); when false, leave ids as-is */ /** @param options.preserveDeveloperRole - undefined/true: keep developer for OpenAI format (default); false: map to system */ +/** @param options.preserveCacheControl - When true, preserve client-side cache_control markers (for Claude Code, etc.) */ // Translate request: source -> openai -> target export function translateRequest( sourceFormat, @@ -83,7 +84,7 @@ export function translateRequest( credentials = null, provider = null, reqLogger = null, - options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean } + options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean; preserveCacheControl?: boolean } ) { let result = body; const use9CharId = options?.normalizeToolCallId === true; @@ -149,10 +150,13 @@ export function translateRequest( } // Final step: prepare request for Claude format endpoints - // In Claude passthrough mode (Claude → Claude), preserve cache_control markers + // Preserve cache_control when: + // 1. Claude passthrough mode (Claude → Claude), OR + // 2. Explicitly requested via options (for caching-aware clients like Claude Code) if (targetFormat === FORMATS.CLAUDE) { const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE; - result = prepareClaudeRequest(result, provider, isClaudePassthrough); + const preserveCache = isClaudePassthrough || options?.preserveCacheControl === true; + result = prepareClaudeRequest(result, provider, preserveCache); } // Normalize openai-responses input shape for providers that require list input. diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts new file mode 100644 index 0000000000..a84cfabd9f --- /dev/null +++ b/open-sse/utils/cacheControlPolicy.ts @@ -0,0 +1,100 @@ +/** + * Cache Control Policy + * + * Determines when to preserve client-side prompt caching headers (cache_control) + * vs. applying OmniRoute's own caching strategy. + * + * Client-side caching (e.g., Claude Code) should be preserved when: + * 1. Client is Claude Code or similar caching-aware client + * 2. Request will hit a deterministic target (single model or deterministic combo strategy) + * 3. Provider supports prompt caching (Anthropic, Alibaba Qwen, etc.) + */ + +import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies"; + +/** + * Routing strategies that are deterministic (same request → same provider) + */ +const DETERMINISTIC_STRATEGIES: Set = new Set([ + "priority", + "cost-optimized", +]); + +/** + * Providers that support prompt caching + */ +const CACHING_PROVIDERS = new Set([ + "claude", + "anthropic", + "zai", + "qwen", // Alibaba Qwen Coding Plan International +]); + +/** + * Detect if the client is Claude Code or another caching-aware client + */ +export function isClaudeCodeClient(userAgent: string | null | undefined): boolean { + if (!userAgent) return false; + const ua = userAgent.toLowerCase(); + + // Claude Code user agents + if (ua.includes("claude-code") || ua.includes("claude_code")) return true; + if (ua.includes("anthropic") && ua.includes("cli")) return true; + + return false; +} + +/** + * Check if a provider supports prompt caching + */ +export function providerSupportsCaching(provider: string | null | undefined): boolean { + if (!provider) return false; + return CACHING_PROVIDERS.has(provider.toLowerCase()); +} + +/** + * Check if a routing strategy is deterministic + */ +export function isDeterministicStrategy(strategy: RoutingStrategyValue | null | undefined): boolean { + if (!strategy) return false; + return DETERMINISTIC_STRATEGIES.has(strategy); +} + +/** + * Determine if client-side cache_control headers should be preserved + * + * @param userAgent - User-Agent header from the request + * @param isCombo - Whether this is a combo model + * @param comboStrategy - The combo's routing strategy (if applicable) + * @param targetProvider - The target provider for the request + * @returns true if cache_control should be preserved, false if OmniRoute should manage it + */ +export function shouldPreserveCacheControl({ + userAgent, + isCombo, + comboStrategy, + targetProvider, +}: { + userAgent: string | null | undefined; + isCombo: boolean; + comboStrategy?: RoutingStrategyValue | null; + targetProvider: string | null | undefined; +}): boolean { + // Must be a caching-aware client + if (!isClaudeCodeClient(userAgent)) { + return false; + } + + // Target provider must support caching + if (!providerSupportsCaching(targetProvider)) { + return false; + } + + // Single model: always preserve (deterministic) + if (!isCombo) { + return true; + } + + // Combo: only preserve if strategy is deterministic + return isDeterministicStrategy(comboStrategy); +} diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 0fbf689550..7013b1d86f 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -275,7 +275,7 @@ export async function handleChat(request: any, clientRawRequest: any = null) { handleSingleModel: (b: any, m: string) => handleSingleModelChat(b, m, clientRawRequest, request, combo.name, apiKeyInfo, telemetry, { sessionId, - }), + }, combo.strategy, true), isModelAvailable: checkModelAvailable, log, settings, @@ -304,7 +304,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) { combo.name, apiKeyInfo, telemetry, - { sessionId, emergencyFallbackTried: true } + { sessionId, emergencyFallbackTried: true }, + combo.strategy, + true ); if (fallbackResponse.ok) { log.info("GLOBAL_FALLBACK", `Global fallback ${fallbackModel} succeeded`); @@ -336,7 +338,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) { null, apiKeyInfo, telemetry, - { sessionId } + { sessionId }, + null, + false ); recordTelemetry(telemetry); return withSessionHeader(response, sessionId); @@ -366,7 +370,9 @@ async function handleSingleModelChat( comboName: string | null = null, apiKeyInfo: any = null, telemetry: any = null, - runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {} + runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {}, + comboStrategy: string | null = null, + isCombo: boolean = false ) { // 1. Resolve model → provider/model const resolved = await resolveModelOrError(modelStr, body, clientRawRequest?.endpoint); @@ -443,6 +449,8 @@ async function handleSingleModelChat( apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, extendedContext, }); if (telemetry) telemetry.endPhase(); @@ -512,7 +520,9 @@ async function handleSingleModelChat( comboName, apiKeyInfo, telemetry, - { ...runtimeOptions, emergencyFallbackTried: true } + { ...runtimeOptions, emergencyFallbackTried: true }, + null, // no strategy for emergency fallback + Boolean(comboName) // isCombo if comboName exists ); if (fallbackResponse.ok) { @@ -648,6 +658,8 @@ async function executeChatWithBreaker({ apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, extendedContext, }: any): Promise<{ result: any; tlsFingerprintUsed: boolean }> { let tlsFingerprintUsed = false; @@ -665,6 +677,8 @@ async function executeChatWithBreaker({ apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, onCredentialsRefreshed: async (newCreds: any) => { await updateProviderCredentials(credentials.connectionId, { accessToken: newCreds.accessToken, diff --git a/tests/unit/cache-control-policy.test.mjs b/tests/unit/cache-control-policy.test.mjs new file mode 100644 index 0000000000..efea6c9c99 --- /dev/null +++ b/tests/unit/cache-control-policy.test.mjs @@ -0,0 +1,252 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { + isClaudeCodeClient, + providerSupportsCaching, + isDeterministicStrategy, + shouldPreserveCacheControl, +} from "../../open-sse/utils/cacheControlPolicy.ts"; + +describe("Cache Control Policy", () => { + describe("isClaudeCodeClient", () => { + test("detects claude-code user agent", () => { + assert.equal(isClaudeCodeClient("claude-code/0.1.0"), true); + assert.equal(isClaudeCodeClient("claude_code/0.1.0"), true); + assert.equal(isClaudeCodeClient("Anthropic CLI/1.0"), true); + }); + + test("rejects non-Claude clients", () => { + assert.equal(isClaudeCodeClient("curl/7.68.0"), false); + assert.equal(isClaudeCodeClient("OpenAI/1.0"), false); + assert.equal(isClaudeCodeClient(null), false); + assert.equal(isClaudeCodeClient(undefined), false); + assert.equal(isClaudeCodeClient(""), false); + }); + + test("is case-insensitive", () => { + assert.equal(isClaudeCodeClient("Claude-Code/0.1.0"), true); + assert.equal(isClaudeCodeClient("CLAUDE-CODE/0.1.0"), true); + }); + }); + + describe("providerSupportsCaching", () => { + test("detects caching providers", () => { + assert.equal(providerSupportsCaching("claude"), true); + assert.equal(providerSupportsCaching("anthropic"), true); + assert.equal(providerSupportsCaching("zai"), true); + assert.equal(providerSupportsCaching("qwen"), true); + }); + + test("rejects non-caching providers", () => { + assert.equal(providerSupportsCaching("openai"), false); + assert.equal(providerSupportsCaching("gemini"), false); + assert.equal(providerSupportsCaching("unknown"), false); + assert.equal(providerSupportsCaching(null), false); + assert.equal(providerSupportsCaching(undefined), false); + }); + + test("is case-insensitive", () => { + assert.equal(providerSupportsCaching("Claude"), true); + assert.equal(providerSupportsCaching("ANTHROPIC"), true); + }); + }); + + describe("isDeterministicStrategy", () => { + test("identifies deterministic strategies", () => { + assert.equal(isDeterministicStrategy("priority"), true); + assert.equal(isDeterministicStrategy("cost-optimized"), true); + }); + + test("identifies non-deterministic strategies", () => { + assert.equal(isDeterministicStrategy("weighted"), false); + assert.equal(isDeterministicStrategy("round-robin"), false); + assert.equal(isDeterministicStrategy("random"), false); + assert.equal(isDeterministicStrategy("fill-first"), false); + assert.equal(isDeterministicStrategy("p2c"), false); + assert.equal(isDeterministicStrategy("least-used"), false); + assert.equal(isDeterministicStrategy("strict-random"), false); + }); + + test("handles null/undefined", () => { + assert.equal(isDeterministicStrategy(null), false); + assert.equal(isDeterministicStrategy(undefined), false); + }); + }); + + describe("shouldPreserveCacheControl", () => { + test("preserves for single model + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "claude", + }), + true + ); + }); + + test("preserves for combo with priority strategy + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "priority", + targetProvider: "claude", + }), + true + ); + }); + + test("preserves for combo with cost-optimized strategy + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "cost-optimized", + targetProvider: "anthropic", + }), + true + ); + }); + + test("rejects non-Claude clients", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "curl/7.68.0", + isCombo: false, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects non-caching providers", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "openai", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (weighted)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "weighted", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (round-robin)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "round-robin", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (random)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "random", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with fill-first strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "fill-first", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with p2c strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "p2c", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with least-used strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "least-used", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with strict-random strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "strict-random", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with null strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: null, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects when userAgent is null", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: null, + isCombo: false, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects when targetProvider is null", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: null, + }), + false + ); + }); + }); +});