diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb05060ad..1b6439d88c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ _In development — bullets added per PR; finalized at release._ - **fix(sse):** prevent non-JSON SSE lines and duplicate `[DONE]` from breaking clients. (thanks @qianze0628) - **fix(sse):** dedupe case-variant Anthropic headers in the executor `buildHeaders` path — Node/undici's `fetch` merges `anthropic-version` and `Anthropic-Version` into a single `"v, v"` value that the Anthropic API rejects, so both case variants are now collapsed to one canonical lowercase header (same for `anthropic-beta`). (thanks @Delcado19) - **oauth(kiro):** support Kiro IDC (organization) token import — when the `~/.aws/sso/cache` token carries a `clientIdHash`, auto-import now reads the linked client registration file to obtain `clientId`/`clientSecret`, probes the Kiro IDE `profile.json` for `profileArn` (ARN region normalized to `us-east-1` for the runtime gateway), and refreshes via the regional AWS OIDC endpoint instead of the social path; the import schema and modal forward these credentials so manual imports also work for IDC tokens. (thanks @enjoyer-hub) +- **fix(translator):** preserve client `cache_control` breakpoints when routing Claude-format requests (e.g. Claude Code) to Alibaba DashScope's OpenAI-compatible providers (`alibaba` / `alibaba-cn`). The Claude→OpenAI translation previously stripped the markers from the system and message text blocks, so DashScope's explicit caching never engaged and every request was a cache miss. Cache hints now survive when preservation is requested for caching-capable OpenAI-format providers. (thanks @sacrtap) --- diff --git a/open-sse/translator/helpers/openaiHelper.ts b/open-sse/translator/helpers/openaiHelper.ts index b6a9bbfb32..cb92b2c427 100644 --- a/open-sse/translator/helpers/openaiHelper.ts +++ b/open-sse/translator/helpers/openaiHelper.ts @@ -31,7 +31,12 @@ const CLAUDE_TOOL_CHOICE_REQUIRED = "an" + "y"; // Filter messages to OpenAI standard format // Remove: redacted_thinking, and other non-OpenAI blocks // Convert: thinking blocks → reasoning_content on the message -export function filterToOpenAIFormat(body) { +export function filterToOpenAIFormat(body, opts = {}) { + // #2069 — when the routed provider honors OpenAI-format cache_control + // breakpoints (DashScope/alibaba, Xiaomi MiMo, etc.) and preservation was + // requested upstream, keep the `cache_control` field on each content block + // instead of destructuring it away. `signature` is always stripped. + const preserveCacheControl = opts?.preserveCacheControl === true; if (!body.messages || !Array.isArray(body.messages)) return body; body.messages = body.messages.map((msg) => { @@ -72,8 +77,13 @@ export function filterToOpenAIFormat(body) { // Only keep valid OpenAI content types if (VALID_OPENAI_CONTENT_TYPES.includes(block.type)) { - // Remove signature and cache_control fields - const { signature, cache_control, ...cleanBlock } = block; + // Strip `signature` always; strip `cache_control` unless the provider + // honors OpenAI-format cache breakpoints and preservation was requested (#2069). + const { signature, cache_control, ...rest } = block; + const cleanBlock = + preserveCacheControl && cache_control !== undefined + ? { ...rest, cache_control } + : rest; if ( cleanBlock.type === "text" && typeof cleanBlock.text === "string" && diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index d0f00967aa..c43cbaa399 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -5,6 +5,7 @@ import { prepareClaudeRequest, } from "./helpers/claudeHelper.ts"; import { filterToOpenAIFormat } from "./helpers/openaiHelper.ts"; +import { providerHonorsOpenAIFormatCacheControl } from "../utils/cacheControlPolicy.ts"; import { coerceToolSchemas, injectEmptyReasoningContentForToolCalls, @@ -213,12 +214,22 @@ export function translateRequest( if (toOpenAI) { // Forward Copilot UA marker to source→openai translators only. const hasTargetHint = targetFormat != null; + // #2069 — forward the cache_control-preservation intent so the + // source→openai translator (e.g. claudeToOpenAIRequest) keeps the + // client's breakpoints — but ONLY for providers that honor explicit + // OpenAI-format cache_control (DashScope/alibaba, Xiaomi MiMo). Generic + // / implicit-cache OpenAI providers (openai/codex/azure) must still be + // stripped. + const preserveCacheControl = + options?.preserveCacheControl === true && + providerHonorsOpenAIFormatCacheControl(provider); const step1Credentials = - options?.copilotClient || hasTargetHint + options?.copilotClient || hasTargetHint || preserveCacheControl ? { ...(credentials && typeof credentials === "object" ? credentials : {}), ...(options?.copilotClient ? { _copilotClient: true } : {}), ...(hasTargetHint ? { _targetFormat: targetFormat } : {}), + ...(preserveCacheControl ? { _preserveCacheControl: true } : {}), } : credentials; result = toOpenAI(model, result, stream, step1Credentials); @@ -256,7 +267,14 @@ export function translateRequest( // Always normalize to clean OpenAI format when target is OpenAI // This handles hybrid requests (e.g., OpenAI messages + Claude tools) if (targetFormat === FORMATS.OPENAI) { - result = filterToOpenAIFormat(result); + // #2069 — preserve client cache_control breakpoints only for providers that + // honor explicit OpenAI-format markers (DashScope/alibaba, Xiaomi MiMo) when + // requested upstream; generic/implicit-cache OpenAI providers stay stripped. + result = filterToOpenAIFormat(result, { + preserveCacheControl: + options?.preserveCacheControl === true && + providerHonorsOpenAIFormatCacheControl(provider), + }); } // Final step: prepare request for Claude format endpoints diff --git a/open-sse/translator/request/claude-to-openai.ts b/open-sse/translator/request/claude-to-openai.ts index 94b2a19b92..770f1d3fd3 100644 --- a/open-sse/translator/request/claude-to-openai.ts +++ b/open-sse/translator/request/claude-to-openai.ts @@ -91,6 +91,16 @@ function shouldUseNativeResponsesWebSearch(credentials: unknown): boolean { // Convert Claude request to OpenAI format export function claudeToOpenAIRequest(model, body, stream, credentials: unknown = null) { + // #2069 — when the routed provider honors OpenAI-format cache_control breakpoints + // (DashScope/alibaba, etc.) and the upstream caller requested preservation, keep + // the client's cache_control markers on system + message text blocks instead of + // collapsing them away during the Claude→OpenAI conversion. + const preserveCacheControl = + credentials !== null && + typeof credentials === "object" && + !Array.isArray(credentials) && + (credentials as JsonRecord)._preserveCacheControl === true; + const result: { model: string; messages: JsonRecord[]; @@ -120,14 +130,35 @@ export function claudeToOpenAIRequest(model, body, stream, credentials: unknown // System message if (body.system) { - const systemContent = Array.isArray(body.system) - ? body.system - .map((s) => stripAnthropicBillingHeader(s.text || "")) - .filter(Boolean) - .join("\n") - : stripAnthropicBillingHeader(body.system); + // When preserving cache_control for a caching-capable provider, and the client + // tagged any system block, keep the array-of-blocks shape so the breakpoint + // survives (DashScope reads cache_control off content blocks). Otherwise fall + // back to the joined-string form expected by generic OpenAI providers (#2069). + const systemHasCacheControl = + preserveCacheControl && + Array.isArray(body.system) && + body.system.some((s) => s && typeof s === "object" && s.cache_control !== undefined); - if (systemContent) { + const systemContent = systemHasCacheControl + ? body.system.map((s) => { + // body.system may be a mixed array — handle string elements (and + // null/non-object) defensively so we never drop text or throw. + if (typeof s === "string") return { type: "text", text: stripAnthropicBillingHeader(s) }; + const rawText = s && typeof s === "object" ? (s as JsonRecord).text || "" : ""; + const block: JsonRecord = { type: "text", text: stripAnthropicBillingHeader(rawText) }; + if (s && typeof s === "object" && (s as JsonRecord).cache_control !== undefined) { + block.cache_control = (s as JsonRecord).cache_control; + } + return block; + }) + : Array.isArray(body.system) + ? body.system + .map((s) => stripAnthropicBillingHeader(s.text || "")) + .filter(Boolean) + .join("\n") + : stripAnthropicBillingHeader(body.system); + + if (systemHasCacheControl || systemContent) { result.messages.push({ role: "system", content: systemContent, @@ -139,7 +170,7 @@ export function claudeToOpenAIRequest(model, body, stream, credentials: unknown if (body.messages && Array.isArray(body.messages)) { for (let i = 0; i < body.messages.length; i++) { const msg = body.messages[i]; - const converted = convertClaudeMessage(msg); + const converted = convertClaudeMessage(msg, preserveCacheControl); if (converted) { // Handle array of messages (multiple tool results) if (Array.isArray(converted)) { @@ -317,7 +348,7 @@ function fixMissingToolResponses(messages) { } // Convert single Claude message - returns single message or array of messages -function convertClaudeMessage(msg) { +function convertClaudeMessage(msg, preserveCacheControl = false) { const role = msg.role === "user" || msg.role === "tool" ? "user" : "assistant"; // Simple string content @@ -334,9 +365,16 @@ function convertClaudeMessage(msg) { for (const block of msg.content) { switch (block.type) { - case "text": - parts.push({ type: "text", text: block.text }); + case "text": { + const textPart: JsonRecord = { type: "text", text: block.text }; + // #2069 — carry the client's cache_control breakpoint through to + // caching-capable OpenAI-format providers (DashScope/alibaba, etc.). + if (preserveCacheControl && block.cache_control !== undefined) { + textPart.cache_control = block.cache_control; + } + parts.push(textPart); break; + } case "image": if (block.source?.type === "base64") { diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index 3603ce7a8f..e886bd3e63 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -92,8 +92,49 @@ const CACHING_PROVIDERS = new Set([ "openai", "codex", "azure", + // #2069 — Alibaba DashScope's OpenAI-compatible endpoints (alibaba / + // alibaba-cn, upstream "alicode"/"alicode-intl") natively honor + // `cache_control: {type:"ephemeral"}` breakpoints. Without these entries + // shouldPreserveCacheControl() returns false for Claude Code clients and the + // OpenAI-format translator strips cache_control, so DashScope never sees the + // hints and every request is a cache miss. + "alibaba", + "alibaba-cn", ]); +/** + * Providers that honor EXPLICIT `cache_control` breakpoints carried inside an + * OpenAI-format request body (i.e. the markers must be passed THROUGH the + * Claude→OpenAI translation instead of stripped). + * + * This is a strict subset of CACHING_PROVIDERS and deliberately excludes + * `openai` / `codex` / `azure`: those use AUTOMATIC prefix caching (#3955) and + * do NOT accept explicit `cache_control` fields in the request — forwarding the + * markers there is meaningless at best and a 400 "unknown field" at worst, and + * it broke the chatCore "strips cache markers for non-Claude providers" test. + * Claude-format providers re-inject markers via prepareClaudeRequest, so they + * are not listed here either. + */ +const OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS = new Set([ + // #2069 — DashScope OpenAI-compatible endpoints accept ephemeral breakpoints. + "alibaba", + "alibaba-cn", + // #3088 — Xiaomi MiMo honors OpenAI-format cache_control breakpoints. + "xiaomi-mimo", +]); + +/** + * Whether `cache_control` markers should be PASSED THROUGH the OpenAI-format + * translation for this provider (vs. stripped). Used to gate the request-side + * passthrough so generic / implicit-cache OpenAI providers keep getting cleaned. + */ +export function providerHonorsOpenAIFormatCacheControl( + provider: string | null | undefined +): boolean { + if (!provider) return false; + return OPENAI_FORMAT_CACHE_CONTROL_PROVIDERS.has(provider.toLowerCase()); +} + /** * Detect if the client is Claude Code or another caching-aware client */ diff --git a/tests/unit/dashscope-cache-control-openai-2069.test.ts b/tests/unit/dashscope-cache-control-openai-2069.test.ts new file mode 100644 index 0000000000..24b1212726 --- /dev/null +++ b/tests/unit/dashscope-cache-control-openai-2069.test.ts @@ -0,0 +1,194 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { translateRequest } from "../../open-sse/translator/index.ts"; +import "../../open-sse/translator/bootstrap.ts"; +import { + providerSupportsCaching, + providerHonorsOpenAIFormatCacheControl, + shouldPreserveCacheControl, +} from "../../open-sse/utils/cacheControlPolicy.ts"; + +// Regression for upstream decolua/9router#2069: cache_control markers are stripped +// when routing a Claude-format request to an OpenAI-compatible DashScope provider +// (alibaba / alibaba-cn = upstream "alicode"/"alicode-intl"). DashScope's +// OpenAI-compatible API natively honors `cache_control: {type:"ephemeral"}`, so the +// markers must survive translation when preservation is requested for that provider. + +function buildClaudeBody() { + return { + system: [ + { + type: "text", + text: "You are a coding assistant. ".repeat(60), + cache_control: { type: "ephemeral" }, + }, + ], + messages: [ + { + role: "user", + content: [ + { type: "text", text: "First, some uncached context." }, + { type: "text", text: "Cache me please.", cache_control: { type: "ephemeral" } }, + ], + }, + ], + }; +} + +function hasCacheControl(node: unknown): boolean { + return JSON.stringify(node).includes("cache_control"); +} + +describe("DashScope OpenAI-compat cache_control preservation (#2069)", () => { + test("alibaba is recognized as a prompt-caching provider", () => { + assert.equal(providerSupportsCaching("alibaba"), true); + assert.equal(providerSupportsCaching("alibaba-cn"), true); + }); + + test("shouldPreserveCacheControl is true for Claude Code → alibaba single model", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-cli/2.1.0 (external, sdk-cli)", + isCombo: false, + targetProvider: "alibaba", + targetFormat: "openai", + }), + true + ); + }); + + test("preserveCacheControl=true keeps cache_control on system + message text blocks", () => { + const out = translateRequest( + "claude", + "openai", + "alibaba/qwen3-coder-plus", + buildClaudeBody(), + false, + null, + "alibaba", + null, + { preserveCacheControl: true } + ) as { messages: Array> }; + + const system = out.messages.find((m) => m.role === "system"); + const user = out.messages.find((m) => m.role === "user"); + + assert.ok(system, "system message present"); + assert.ok(user, "user message present"); + assert.equal(hasCacheControl(system), true, "system cache_control preserved"); + assert.equal(hasCacheControl(user), true, "user cache_control preserved"); + + // The cache_control marker must land on the exact block the client tagged, + // not be smeared across uncached blocks. + const userContent = user!.content as Array>; + assert.ok(Array.isArray(userContent), "user content stays an array of blocks"); + const tagged = userContent.find((b) => b.text === "Cache me please."); + const untagged = userContent.find((b) => b.text === "First, some uncached context."); + assert.deepEqual(tagged?.cache_control, { type: "ephemeral" }); + assert.equal(untagged?.cache_control, undefined); + }); + + test("preserveCacheControl=false strips cache_control (OmniRoute manages caching)", () => { + const out = translateRequest( + "claude", + "openai", + "alibaba/qwen3-coder-plus", + buildClaudeBody(), + false, + null, + "alibaba", + null, + { preserveCacheControl: false } + ) as { messages: Array> }; + + assert.equal(hasCacheControl(out.messages), false, "cache_control stripped when not preserved"); + }); + + test("non-caching OpenAI provider still strips cache_control even if asked", () => { + // A generic OpenAI-compatible provider not on the caching allowlist must NOT + // receive cache_control passthrough — guards against blast radius. Here the + // policy gate (shouldPreserveCacheControl) would already be false, but even a + // direct preserve=true call only matters for caching-capable providers; this + // asserts we did not turn filterToOpenAIFormat into an unconditional passthrough. + assert.equal(providerSupportsCaching("groq"), false); + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-cli/2.1.0", + isCombo: false, + targetProvider: "groq", + targetFormat: "openai", + }), + false + ); + }); + + test("explicit-breakpoint predicate is narrow: DashScope/Xiaomi yes, implicit-cache OpenAI no", () => { + // alibaba / alibaba-cn / xiaomi-mimo accept explicit OpenAI-format markers. + assert.equal(providerHonorsOpenAIFormatCacheControl("alibaba"), true); + assert.equal(providerHonorsOpenAIFormatCacheControl("alibaba-cn"), true); + assert.equal(providerHonorsOpenAIFormatCacheControl("xiaomi-mimo"), true); + // openai / codex / azure are caching providers via AUTOMATIC prefix caching + // (#3955) — they do NOT take explicit cache_control in the request and must + // NOT receive passthrough (regression guard for chatcore-translation-paths). + assert.equal(providerHonorsOpenAIFormatCacheControl("openai"), false); + assert.equal(providerHonorsOpenAIFormatCacheControl("codex"), false); + assert.equal(providerHonorsOpenAIFormatCacheControl("azure"), false); + assert.equal(providerHonorsOpenAIFormatCacheControl(null), false); + }); + + test("mixed string + object system blocks: string text preserved, no crash", () => { + const body = { + system: [ + "plain string directive", + { type: "text", text: "tagged", cache_control: { type: "ephemeral" } }, + ], + messages: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + }; + const out = translateRequest( + "claude", + "openai", + "alibaba/qwen3-coder-plus", + body, + false, + null, + "alibaba", + null, + { preserveCacheControl: true } + ) as { messages: Array> }; + + const system = out.messages.find((m) => m.role === "system"); + const content = system!.content as Array>; + // The plain string element must not be silently dropped. + assert.equal( + content.some((b) => b.text === "plain string directive"), + true, + "string system element text is preserved" + ); + // The tagged object keeps its cache_control breakpoint. + const tagged = content.find((b) => b.text === "tagged"); + assert.deepEqual(tagged?.cache_control, { type: "ephemeral" }); + }); + + test("openai provider strips cache_control even with preserveCacheControl=true (implicit cache)", () => { + // Mirrors tests/unit/chatcore-translation-paths.test.ts: a Claude Code request + // routed to the `openai` provider must arrive WITHOUT cache_control even though + // shouldPreserveCacheControl() returns true (openai ∈ CACHING_PROVIDERS). + assert.equal(providerSupportsCaching("openai"), true); + const out = translateRequest( + "claude", + "openai", + "openai/gpt-4o-mini", + buildClaudeBody(), + false, + null, + "openai", + null, + { preserveCacheControl: true } + ) as { messages: Array> }; + assert.equal( + hasCacheControl(out.messages), + false, + "openai (implicit prefix cache) must not receive explicit cache_control" + ); + }); +});