From 7db3ba615bbf1f116fcd37f6f484bbd1a4247072 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Mon, 27 Jul 2026 17:24:43 -0300 Subject: [PATCH] fix(sse): stop thrashing the provider prompt cache for caching-aware clients (#8705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sse): stop thrashing the provider prompt cache for caching-aware clients - shouldPreserveCacheControl: preserve client cache_control markers for every combo strategy. The deterministic-strategy gate forced marker rewrites whose per-request breakpoint positions are not stable turn-over-turn, thrashing the upstream prompt cache (observed in production as ~200k cache_write tokens per turn on quota-share combos). Preserving is never worse: on a stable target the client's breakpoints advance deterministically; on a target switch both approaches miss equally. - prepareClaudeRequest: translator-path opt-in fallback — when preserve-mode has nothing to preserve (client sent no cache_control anywhere), apply the standard heuristic so requests never ship with zero cache breakpoints. The claude-code-compatible relay path keeps its no-supplement contract. - anthropic-beta: forward the client-negotiated context-1m-2025-08-07 through the allowlist merge so a /model [1m] client keeps its long-context negotiation behind the proxy (never forced when the client did not send it). * fix(sse): surface cache tokens in non-streaming OpenAI usage + finalize pending on quota-share block - translateNonStreamingResponse (claude→openai): fold cache_read into prompt_tokens and expose prompt_tokens_details.cached_tokens / cache_creation_tokens, mirroring the streaming contract (#1426/#2215). Non-streaming OpenAI clients behind a cached Claude upstream previously saw prompt_tokens= (e.g. 23 for a ~9k request) with no cache visibility. - chatCore quota-share block: finalize the pending-request slot before returning the policy 429 — the path never reaches upstream and the orphaned pending lingered as a status-0 call-log row until the reaper swept it. Validated live on the staging box (repro: blocked key → orphan row; after: clean). --------- Co-authored-by: diegosouzapw --- open-sse/config/anthropicHeaders.ts | 12 +- open-sse/handlers/chatCore.ts | 10 ++ open-sse/handlers/responseTranslator.ts | 17 ++- open-sse/translator/helpers/claudeHelper.ts | 44 +++++- open-sse/translator/index.ts | 8 +- open-sse/utils/cacheControlPolicy.ts | 46 +++--- tests/unit/cache-control-policy.test.ts | 140 +++++++----------- ...ude-context-1m-client-beta-forward.test.ts | 74 +++++++++ ...-prepare-request-preserve-fallback.test.ts | 114 ++++++++++++++ .../unit/nonstream-usage-cache-tokens.test.ts | 68 +++++++++ 10 files changed, 422 insertions(+), 111 deletions(-) create mode 100644 tests/unit/claude-context-1m-client-beta-forward.test.ts create mode 100644 tests/unit/claude-prepare-request-preserve-fallback.test.ts create mode 100644 tests/unit/nonstream-usage-cache-tokens.test.ts diff --git a/open-sse/config/anthropicHeaders.ts b/open-sse/config/anthropicHeaders.ts index 7b04eb6c45..6a98e4aa98 100644 --- a/open-sse/config/anthropicHeaders.ts +++ b/open-sse/config/anthropicHeaders.ts @@ -43,9 +43,17 @@ export const ANTHROPIC_BETA_CLAUDE_OAUTH = [ * claude.ai backend on top of OmniRoute's own set. Kept to betas the backend * actually accepts and that OmniRoute does not otherwise emit — so a blind * passthrough cannot reintroduce the over-sending fingerprint/rejection bugs - * (#3415, #2454). Currently: deferred-tool negotiation (#3974). + * (#3415, #2454). Currently: deferred-tool negotiation (#3974) and the + * client's own `[1m]` long-context negotiation (context-1m). selectBetaFlags + * deliberately emits context-1m only for Opus (never FORCE it — long-context + * credit gate); forwarding it when the CLIENT negotiated it matches what real + * Claude Code sends for `/model [1m]` and is required for >200K-context + * requests on models/accounts where the beta is enforced. */ -export const FORWARDABLE_CLIENT_BETAS = Object.freeze(["tool-search-tool-2025-10-19"]); +export const FORWARDABLE_CLIENT_BETAS = Object.freeze([ + "tool-search-tool-2025-10-19", + "context-1m-2025-08-07", +]); /** * Union an `anthropic-beta` comma-list with the allowlisted client-negotiated diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 75924ddbaa..6b023e0ce9 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -2512,6 +2512,16 @@ export async function handleChatCore({ "QUOTA_SHARE", `[quotaShare] blocked apiKeyId=${apiKeyInfo.id} provider=${provider ?? "unknown"}: ${decision.reason}` ); + // Finalize the pending-request slot registered at handler entry — this + // return path never reaches the upstream, and without the decrement the + // pending detail lingers as an orphaned status-0 call-log row until the + // reaper sweeps it (mirrors the other pre-upstream error returns). + trackPendingRequest( + model, + provider, + connectionId || credentials?.connectionId || null, + false + ); const headers: Record = { "Content-Type": "application/json" }; if (decision.retryAfterSeconds) { headers["Retry-After"] = String(decision.retryAfterSeconds); diff --git a/open-sse/handlers/responseTranslator.ts b/open-sse/handlers/responseTranslator.ts index 92fb47b7b6..7c03f1f623 100644 --- a/open-sse/handlers/responseTranslator.ts +++ b/open-sse/handlers/responseTranslator.ts @@ -539,13 +539,26 @@ export function translateNonStreamingResponse( const usage = toRecord(root.usage); if (Object.keys(usage).length > 0) { - const promptTokens = toNumber(usage.input_tokens, 0); + // Mirror the streaming translator's usage contract (#1426/#2215): + // cache_read folds into prompt_tokens (it is billed prompt input); + // cache_creation stays out of prompt_tokens and is exposed via + // prompt_tokens_details, alongside cached_tokens (OpenAI field name). + const cachedTokens = toNumber(usage.cache_read_input_tokens, 0); + const cacheCreationTokens = toNumber(usage.cache_creation_input_tokens, 0); + const promptTokens = toNumber(usage.input_tokens, 0) + cachedTokens; const completionTokens = toNumber(usage.output_tokens, 0); - result.usage = { + const usageOut: JsonRecord = { prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens, }; + if (cachedTokens > 0 || cacheCreationTokens > 0) { + const details: JsonRecord = {}; + if (cachedTokens > 0) details.cached_tokens = cachedTokens; + if (cacheCreationTokens > 0) details.cache_creation_tokens = cacheCreationTokens; + usageOut.prompt_tokens_details = details; + } + result.usage = usageOut; } intermediateOpenAI = result; diff --git a/open-sse/translator/helpers/claudeHelper.ts b/open-sse/translator/helpers/claudeHelper.ts index 446cea4828..e34704651c 100644 --- a/open-sse/translator/helpers/claudeHelper.ts +++ b/open-sse/translator/helpers/claudeHelper.ts @@ -239,6 +239,32 @@ function markMessageCacheControl(msg: ClaudeMessage, ttl?: string): boolean { return true; } +/** True when the body carries at least one cache_control marker anywhere + * (system blocks, message content blocks, or tools). Used to decide whether + * preserve-mode has anything to preserve. */ +function bodyHasAnyCacheControl(body: ClaudeRequestBody): boolean { + if (Array.isArray(body.system)) { + for (const block of body.system) { + if (block && typeof block === "object" && block.cache_control) return true; + } + } + if (Array.isArray(body.messages)) { + for (const msg of body.messages) { + if (!Array.isArray(msg?.content)) continue; + for (const block of msg.content) { + if (block && typeof block === "object" && block.cache_control) return true; + } + } + } + if (Array.isArray(body.tools)) { + for (const tool of body.tools) { + if (tool && typeof tool === "object" && (tool as { cache_control?: unknown }).cache_control) + return true; + } + } + return false; +} + // Prepare request for Claude format endpoints // - Cleanup cache_control (unless preserveCacheControl=true for passthrough) // - Filter empty messages @@ -248,7 +274,8 @@ export function prepareClaudeRequest( body: ClaudeRequestBody, provider: string | null = null, preserveCacheControl = false, - model: string | null = null + model: string | null = null, + opts: { fallbackToHeuristicWhenNoMarkers?: boolean } = {} ): ClaudeRequestBody { // 0. Strip Anthropic `output_config` for providers that reject it on their // Claude-compatible endpoints (MiniMax). Must run before any downstream @@ -257,6 +284,21 @@ export function prepareClaudeRequest( delete body.output_config; } + // preserveCacheControl means "the client manages its own cache markers". + // A body with NO cache_control anywhere has nothing to preserve — shipping + // it untouched would mean zero prompt-cache breakpoints (every token billed + // uncached on every turn). The translator path opts into falling back to the + // standard heuristic in that case; the claude-code-compatible relay path + // keeps its long-standing "never supplement missing markers" contract + // (cc-compatible-provider.test.ts) and does not pass the flag. + if ( + preserveCacheControl && + opts.fallbackToHeuristicWhenNoMarkers === true && + !bodyHasAnyCacheControl(body) + ) { + preserveCacheControl = false; + } + // 1. System: remove all cache_control, add only to last block with ttl 1h // In passthrough mode, preserve existing cache_control markers const supportsPromptCaching = diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 0e14c105ed..d7f107fbb7 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -368,10 +368,16 @@ export function translateRequest( // Preserve cache_control when: // 1. Claude passthrough mode (Claude → Claude), OR // 2. Explicitly requested via options (for caching-aware clients like Claude Code) + // When preserve-mode has nothing to preserve (the client sent no cache_control + // anywhere), fall back to the standard heuristic so the request never ships + // with zero prompt-cache breakpoints. Translator-path only — the CC relay + // path keeps its "never supplement" contract. if (targetFormat === FORMATS.CLAUDE) { const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE; const preserveCache = isClaudePassthrough || options?.preserveCacheControl === true; - result = prepareClaudeRequest(result, provider, preserveCache, model); + result = prepareClaudeRequest(result, provider, preserveCache, model, { + fallbackToHeuristicWhenNoMarkers: true, + }); } // Normalize openai-responses input shape for providers that require list input. diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index 23e4eac4eb..7e1abe6c54 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -4,10 +4,14 @@ * Determines when to preserve client-side prompt caching headers (cache_control) * vs. applying OmniRoute's own caching strategy. * - * Client-side caching (e.g., Claude Code) should be preserved when: + * Client-side caching (e.g., Claude Code) is preserved when: * 1. Client is Claude Code or similar caching-aware client - * 2. Request will hit a deterministic target (single model or deterministic combo strategy) - * 3. Provider supports prompt caching (Anthropic, etc.) + * 2. Provider supports prompt caching (Anthropic, etc.) + * + * Combo membership / routing strategy no longer gates preservation: rewriting a + * caching-aware client's markers produced per-request breakpoint positions that + * thrashed the provider prompt cache, while preserving them is never worse + * (see shouldPreserveCacheControl). */ import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies"; @@ -231,17 +235,28 @@ export function isDeterministicStrategy( /** * Determine if client-side cache_control headers should be preserved * + * Auto mode preserves for every caching-aware client talking to a + * caching-capable provider — regardless of combo membership or routing + * strategy. The old gate (combos only preserved on "deterministic" + * strategies) forced OmniRoute to strip the client's markers and re-derive + * breakpoints per request; the re-derived positions are not stable + * turn-over-turn, which thrashed the provider prompt cache (observed in + * production as ~200k cache_write tokens per turn on quota-share combos). + * Preserving the client's markers is never worse than rewriting them: on a + * stable target the client's breakpoints advance deterministically, and on a + * target switch both approaches miss equally. + * * @param userAgent - User-Agent header from the request - * @param isCombo - Whether this is a combo model - * @param comboStrategy - The combo's routing strategy (if applicable) + * @param isCombo - Whether this is a combo model (kept for callers/telemetry; + * no longer gates preservation) + * @param comboStrategy - The combo's routing strategy (kept for + * callers/telemetry; no longer gates preservation) * @param targetProvider - The target provider for the request * @param settings - Cache control settings from database (optional) * @returns true if cache_control should be preserved, false if OmniRoute should manage it */ export function shouldPreserveCacheControl({ userAgent, - isCombo, - comboStrategy, targetProvider, targetFormat, settings, @@ -263,24 +278,13 @@ export function shouldPreserveCacheControl({ return false; } - // Auto mode: use automatic detection (existing logic) - // Must be a caching-aware client + // Auto mode: must be a caching-aware client… if (!isClaudeCodeClient(userAgent)) { return false; } - // Target provider must support caching - if (!providerSupportsCaching(targetProvider, targetFormat, connectionCacheOverride)) { - return false; - } - - // Single model: always preserve (deterministic) - if (!isCombo) { - return true; - } - - // Combo: only preserve if strategy is deterministic - return isDeterministicStrategy(comboStrategy); + // …talking to a provider that supports prompt caching. + return providerSupportsCaching(targetProvider, targetFormat, connectionCacheOverride); } /** diff --git a/tests/unit/cache-control-policy.test.ts b/tests/unit/cache-control-policy.test.ts index 558e768f28..934c39a8c7 100644 --- a/tests/unit/cache-control-policy.test.ts +++ b/tests/unit/cache-control-policy.test.ts @@ -155,91 +155,39 @@ describe("Cache Control Policy", () => { ); }); - test("rejects combo with non-deterministic strategy (weighted)", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "weighted", - targetProvider: "claude", - }), - false - ); - }); + // Client cache_control markers are preserved for EVERY combo strategy, not + // only the "deterministic" ones. Rewriting a caching-aware client's markers + // never beats preserving them: on a stable target the client's breakpoints + // advance deterministically turn-over-turn (the proxy heuristic recomputes + // positions and thrashes the provider prompt cache — observed in production + // as ~200k cache_write per turn on qtSd/ quota-share combos), and on a + // target switch both approaches miss equally. + for (const strategy of [ + "weighted", + "round-robin", + "random", + "fill-first", + "p2c", + "least-used", + "strict-random", + "quota-share", + ]) { + test(`preserves for combo with ${strategy} strategy (markers win over routing strategy)`, () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: strategy as Parameters< + typeof shouldPreserveCacheControl + >[0]["comboStrategy"], + targetProvider: "claude", + }), + true + ); + }); + } - test("rejects combo with non-deterministic strategy (round-robin)", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "round-robin", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with non-deterministic strategy (random)", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "random", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with fill-first strategy", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "fill-first", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with p2c strategy", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "p2c", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with least-used strategy", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "least-used", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with strict-random strategy", () => { - assert.equal( - shouldPreserveCacheControl({ - userAgent: "claude-code/0.1.0", - isCombo: true, - comboStrategy: "strict-random", - targetProvider: "claude", - }), - false - ); - }); - - test("rejects combo with null strategy", () => { + test("preserves for combo with null strategy (strategy is irrelevant to preservation)", () => { assert.equal( shouldPreserveCacheControl({ userAgent: "claude-code/0.1.0", @@ -247,6 +195,30 @@ describe("Cache Control Policy", () => { comboStrategy: null, targetProvider: "claude", }), + true + ); + }); + + test("still rejects combo when the provider does not support caching", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "quota-share", + targetProvider: "gemini", + }), + false + ); + }); + + test("still rejects combo for non-caching-aware clients", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "curl/7.68.0", + isCombo: true, + comboStrategy: "quota-share", + targetProvider: "claude", + }), false ); }); diff --git a/tests/unit/claude-context-1m-client-beta-forward.test.ts b/tests/unit/claude-context-1m-client-beta-forward.test.ts new file mode 100644 index 0000000000..ff0eaefce9 --- /dev/null +++ b/tests/unit/claude-context-1m-client-beta-forward.test.ts @@ -0,0 +1,74 @@ +/** + * Client-negotiated `context-1m-2025-08-07` must be forwarded upstream. + * + * When Claude Code runs with a `[1m]` model (e.g. `/model sonnet[1m]` or + * `ANTHROPIC_DEFAULT_SONNET_MODEL='[1m]'`), the client sizes its context + * window at 1M AND negotiates `anthropic-beta: context-1m-2025-08-07` itself. + * Behind OmniRoute that beta was silently dropped for non-Opus models: + * selectBetaFlags only emits context-1m for `claude-opus*` (deliberate — never + * FORCE it, see the long-context credit-gate note there), and the + * FORWARDABLE_CLIENT_BETAS allowlist did not include it, so the client's own + * negotiation was stripped. Real Claude Code sends the beta in this setup, so + * forwarding what the client asked for is the fingerprint-faithful behavior — + * and it is required for >200K-context requests on models/accounts where the + * beta is enforced. + * + * Guard preserved: the beta is only APPENDED when the client sent it — a + * client without `[1m]` keeps the exact same beta set as before. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { selectBetaFlags } = await import("../../open-sse/executors/claudeIdentity.ts"); +const { mergeClientAnthropicBeta, FORWARDABLE_CLIENT_BETAS } = + await import("../../open-sse/config/anthropicHeaders.ts"); + +const CONTEXT_1M = "context-1m-2025-08-07"; + +function fullAgentBody(model: string) { + return { + model, + system: "You are a coding agent.", + tools: [{ name: "read_file", description: "x", input_schema: { type: "object" } }], + }; +} + +test("mergeClientAnthropicBeta forwards a client-negotiated context-1m beta", () => { + const out = mergeClientAnthropicBeta( + "claude-code-20250219,oauth-2025-04-20", + `oauth-2025-04-20,${CONTEXT_1M}` + ); + assert.ok( + out.split(",").includes(CONTEXT_1M), + "client [1m] negotiation must survive the allowlist merge" + ); + assert.ok(FORWARDABLE_CLIENT_BETAS.includes(CONTEXT_1M)); +}); + +test("selectBetaFlags + merge keeps context-1m for a Sonnet [1m] client", () => { + const clientBeta = `claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14,${CONTEXT_1M}`; + const out = mergeClientAnthropicBeta( + selectBetaFlags(fullAgentBody("claude-sonnet-5"), null, clientBeta), + clientBeta + ); + assert.ok( + out.split(",").includes(CONTEXT_1M), + "Sonnet client that negotiated [1m] must reach upstream with context-1m" + ); +}); + +test("context-1m is never forced when the client did not negotiate it", () => { + const clientBeta = "claude-code-20250219,oauth-2025-04-20,interleaved-thinking-2025-05-14"; + const out = mergeClientAnthropicBeta( + selectBetaFlags(fullAgentBody("claude-sonnet-5"), null, clientBeta), + clientBeta + ); + assert.ok( + !out.split(",").includes(CONTEXT_1M), + "a client without [1m] keeps the exact same beta set as before" + ); +}); + +test("tool-search forwarding keeps working (regression #3974)", () => { + assert.ok(FORWARDABLE_CLIENT_BETAS.includes("tool-search-tool-2025-10-19")); +}); diff --git a/tests/unit/claude-prepare-request-preserve-fallback.test.ts b/tests/unit/claude-prepare-request-preserve-fallback.test.ts new file mode 100644 index 0000000000..193885d0a0 --- /dev/null +++ b/tests/unit/claude-prepare-request-preserve-fallback.test.ts @@ -0,0 +1,114 @@ +/** + * prepareClaudeRequest preserve-mode fallback. + * + * preserveCacheControl=true must mean "the client manages its own cache + * markers" — but when the client body carries NO cache_control anywhere, + * preserving "nothing" would ship a request with zero prompt-cache + * breakpoints (every token billed uncached on every turn). In that case the + * standard heuristic must still apply, exactly as if preserveCacheControl + * were false. When the client DID place markers, they are preserved verbatim + * and the heuristic must not add any extra ones. + */ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { prepareClaudeRequest } from "../../open-sse/translator/helpers/claudeHelper.ts"; + +type Block = Record; + +function baseBody(): Record { + return { + model: "claude-sonnet-5", + system: [ + { type: "text", text: "You are a helpful assistant." }, + { type: "text", text: "Project context goes here." }, + ], + messages: [ + { role: "user", content: [{ type: "text", text: "First question" }] }, + { role: "assistant", content: [{ type: "text", text: "First answer" }] }, + { role: "user", content: [{ type: "text", text: "Second question" }] }, + ], + max_tokens: 128, + }; +} + +function collectCacheControls(body: Record): Block[] { + const found: Block[] = []; + for (const block of (body.system as Block[]) ?? []) { + if (block.cache_control) found.push(block); + } + for (const msg of (body.messages as Array<{ content?: Block[] }>) ?? []) { + for (const block of msg.content ?? []) { + if (block.cache_control) found.push(block); + } + } + return found; +} + +describe("prepareClaudeRequest preserve-mode fallback", () => { + test("falls back to the heuristic when the client sent no markers at all (translator opt-in)", () => { + const body = baseBody(); + const result = prepareClaudeRequest(body, "claude", true, "claude-sonnet-5", { + fallbackToHeuristicWhenNoMarkers: true, + }); + + const system = result.system as Block[]; + const lastSystem = system[system.length - 1]; + assert.ok( + lastSystem.cache_control, + "last system block should carry the heuristic cache_control when the client sent none" + ); + assert.ok( + collectCacheControls(result as Record).length > 0, + "request must not go upstream with zero cache breakpoints" + ); + }); + + test("without the opt-in flag, preserve-mode never supplements markers (CC relay contract)", () => { + const body = baseBody(); + const result = prepareClaudeRequest(body, "claude", true, "claude-sonnet-5"); + assert.equal( + collectCacheControls(result as Record).length, + 0, + "the claude-code-compatible relay path keeps its no-supplement contract" + ); + }); + + test("preserves client markers verbatim and adds none (system marker)", () => { + const body = baseBody(); + (body.system as Block[])[0].cache_control = { type: "ephemeral" }; + const result = prepareClaudeRequest(body, "claude", true, "claude-sonnet-5"); + + const system = result.system as Block[]; + assert.deepEqual( + system[0].cache_control, + { type: "ephemeral" }, + "client marker must survive untouched" + ); + assert.equal( + collectCacheControls(result as Record).length, + 1, + "no extra heuristic markers may be added when the client manages its own" + ); + }); + + test("preserves client markers verbatim and adds none (message marker)", () => { + const body = baseBody(); + const messages = body.messages as Array<{ content: Block[] }>; + messages[2].content[0].cache_control = { type: "ephemeral" }; + const result = prepareClaudeRequest(body, "claude", true, "claude-sonnet-5"); + + const controls = collectCacheControls(result as Record); + assert.equal(controls.length, 1, "only the client's own marker may remain"); + const resultMessages = result.messages as Array<{ content: Block[] }>; + assert.deepEqual(resultMessages[resultMessages.length - 1].content[0].cache_control, { + type: "ephemeral", + }); + }); + + test("non-preserve mode keeps applying the heuristic (regression)", () => { + const body = baseBody(); + const result = prepareClaudeRequest(body, "claude", false, "claude-sonnet-5"); + const system = result.system as Block[]; + assert.ok(system[system.length - 1].cache_control, "heuristic marker on last system block"); + }); +}); diff --git a/tests/unit/nonstream-usage-cache-tokens.test.ts b/tests/unit/nonstream-usage-cache-tokens.test.ts new file mode 100644 index 0000000000..7fafbc4db1 --- /dev/null +++ b/tests/unit/nonstream-usage-cache-tokens.test.ts @@ -0,0 +1,68 @@ +/** + * Non-streaming claude→openai usage must surface prompt-cache tokens. + * + * The streaming translator already folds cache_read into prompt_tokens and + * exposes prompt_tokens_details (#1426/#2215). The non-streaming path built the + * usage object from input_tokens/output_tokens only, so an OpenAI-format client + * behind a cached Claude upstream saw prompt_tokens= (e.g. + * 23 for a ~9k-token request) and no cache visibility at all — billing/telemetry + * built on the OpenAI usage contract under-reported by orders of magnitude. + * + * Contract mirrored from the streaming path: + * prompt_tokens = input_tokens + cache_read_input_tokens + * prompt_tokens_details.cached_tokens = cache_read_input_tokens + * prompt_tokens_details.cache_creation_tokens = cache_creation_input_tokens + * (cache_creation stays OUT of prompt_tokens — #2215: providers price it + * differently and folding it in broke min-token accounting.) + */ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { translateNonStreamingResponse } from "../../open-sse/handlers/responseTranslator.ts"; +import { FORMATS } from "../../open-sse/translator/formats.ts"; + +function claudeResponse(usage: Record) { + return { + id: "msg_test", + model: "claude-sonnet-5", + stop_reason: "end_turn", + content: [{ type: "text", text: "ok" }], + usage, + }; +} + +describe("non-streaming claude→openai usage cache tokens", () => { + test("folds cache_read into prompt_tokens and exposes details", () => { + const out = translateNonStreamingResponse( + claudeResponse({ + input_tokens: 2, + output_tokens: 6, + cache_read_input_tokens: 8941, + cache_creation_input_tokens: 27, + }), + FORMATS.CLAUDE, + FORMATS.OPENAI + ) as Record; + + const usage = out.usage as Record; + assert.equal(usage.prompt_tokens, 8943, "prompt_tokens = input + cache_read"); + assert.equal(usage.completion_tokens, 6); + assert.equal(usage.total_tokens, 8949); + const details = usage.prompt_tokens_details as Record; + assert.ok(details, "prompt_tokens_details must be present when cache fields exist"); + assert.equal(details.cached_tokens, 8941); + assert.equal(details.cache_creation_tokens, 27); + }); + + test("no cache fields → plain usage without details (unchanged shape)", () => { + const out = translateNonStreamingResponse( + claudeResponse({ input_tokens: 10, output_tokens: 5 }), + FORMATS.CLAUDE, + FORMATS.OPENAI + ) as Record; + + const usage = out.usage as Record; + assert.equal(usage.prompt_tokens, 10); + assert.equal(usage.total_tokens, 15); + assert.equal(usage.prompt_tokens_details, undefined); + }); +});