From b84c915b230181c80960fe0ef722be055e415041 Mon Sep 17 00:00:00 2001 From: tombii Date: Sat, 28 Mar 2026 16:23:40 +0100 Subject: [PATCH 1/4] fix(sse): preserve cache_control in Claude passthrough mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Claude Code routes through OmniRoute (Claude → OmniRoute → Claude), OmniRoute was stripping all cache_control markers and replacing them with its own generic caching strategy. This broke Claude Code's carefully placed cache breakpoints for plans and other features. Changes: - Add preserveCacheControl parameter to prepareClaudeRequest() - Detect Claude passthrough mode (sourceFormat === targetFormat === CLAUDE) - Skip cache_control normalization when preserveCacheControl=true - Preserve client's cache_control markers in system, messages, and tools This ensures Claude Code's prompt caching optimization works correctly while maintaining OmniRoute's caching strategy for translation scenarios. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 4 + open-sse/translator/helpers/claudeHelper.ts | 37 ++-- open-sse/translator/index.ts | 4 +- .../claude-cache-control-passthrough.test.mjs | 175 ++++++++++++++++++ 4 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 tests/unit/claude-cache-control-passthrough.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 56eaa57200..c9d723e61a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### 🐛 Bug Fixes + +- **Claude Prompt Caching Passthrough** — Fixed cache_control markers being stripped in Claude passthrough mode (Claude → OmniRoute → Claude), which caused Claude Code users to deplete their Anthropic API quota 5-10x faster than direct connections. OmniRoute now preserves client's cache_control markers when sourceFormat and targetFormat are both Claude, ensuring prompt caching works correctly and dramatically reducing token consumption. + ## [3.1.8] - 2026-03-27 ### 🐛 Bug Fixes & Features diff --git a/open-sse/translator/helpers/claudeHelper.ts b/open-sse/translator/helpers/claudeHelper.ts index e03d21e584..0b1a614ecb 100644 --- a/open-sse/translator/helpers/claudeHelper.ts +++ b/open-sse/translator/helpers/claudeHelper.ts @@ -105,13 +105,14 @@ function markMessageCacheControl(msg, ttl) { } // Prepare request for Claude format endpoints -// - Cleanup cache_control +// - Cleanup cache_control (unless preserveCacheControl=true for passthrough) // - Filter empty messages // - Add thinking block for Anthropic endpoint (provider === "claude") // - Fix tool_use/tool_result ordering -export function prepareClaudeRequest(body, provider = null) { +export function prepareClaudeRequest(body, provider = null, preserveCacheControl = false) { // 1. System: remove all cache_control, add only to last block with ttl 1h - if (body.system && Array.isArray(body.system)) { + // In passthrough mode, preserve existing cache_control markers + if (body.system && Array.isArray(body.system) && !preserveCacheControl) { body.system = body.system.map((block, i) => { const { cache_control, ...rest } = block; if (i === body.system.length - 1) { @@ -127,11 +128,12 @@ export function prepareClaudeRequest(body, provider = null) { let filtered = []; // Pass 1: remove cache_control + filter empty messages + // In passthrough mode, preserve existing cache_control markers for (let i = 0; i < len; i++) { const msg = body.messages[i]; - // Remove cache_control from content blocks - if (Array.isArray(msg.content)) { + // Remove cache_control from content blocks (skip in passthrough mode) + if (Array.isArray(msg.content) && !preserveCacheControl) { for (const block of msg.content) { delete block.cache_control; } @@ -177,14 +179,17 @@ export function prepareClaudeRequest(body, provider = null) { // Claude Code-style prompt caching: // - cache the second-to-last user turn for conversation reuse // - cache the last assistant turn so the next user turn can reuse it - const userMessageIndexes = filtered.reduce((indexes, msg, index) => { - if (msg?.role === "user") indexes.push(index); - return indexes; - }, []); - const secondToLastUserIndex = - userMessageIndexes.length >= 2 ? userMessageIndexes[userMessageIndexes.length - 2] : -1; - if (secondToLastUserIndex >= 0) { - markMessageCacheControl(filtered[secondToLastUserIndex]); + // Skip in passthrough mode to preserve client's cache_control markers + if (!preserveCacheControl) { + const userMessageIndexes = filtered.reduce((indexes, msg, index) => { + if (msg?.role === "user") indexes.push(index); + return indexes; + }, []); + const secondToLastUserIndex = + userMessageIndexes.length >= 2 ? userMessageIndexes[userMessageIndexes.length - 2] : -1; + if (secondToLastUserIndex >= 0) { + markMessageCacheControl(filtered[secondToLastUserIndex]); + } } // Pass 2 (reverse): add cache_control to last assistant + handle thinking for Anthropic @@ -194,7 +199,8 @@ export function prepareClaudeRequest(body, provider = null) { if (msg.role === "assistant" && Array.isArray(ensureMessageContentArray(msg))) { // Add cache_control to last block of first (from end) assistant with content - if (!lastAssistantProcessed && markMessageCacheControl(msg)) { + // Skip in passthrough mode to preserve client's cache_control markers + if (!preserveCacheControl && !lastAssistantProcessed && markMessageCacheControl(msg)) { lastAssistantProcessed = true; } @@ -227,7 +233,8 @@ export function prepareClaudeRequest(body, provider = null) { // 3. Tools: remove all cache_control, add only to last non-deferred tool with ttl 1h // Tools with defer_loading=true cannot have cache_control (API rejects it) - if (body.tools && Array.isArray(body.tools)) { + // In passthrough mode, preserve existing cache_control markers + if (body.tools && Array.isArray(body.tools) && !preserveCacheControl) { body.tools = body.tools.map((tool) => { const { cache_control, ...rest } = tool; return rest; diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 25f70900fe..9dfb4b31ce 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -144,8 +144,10 @@ export function translateRequest( } // Final step: prepare request for Claude format endpoints + // In Claude passthrough mode (Claude → Claude), preserve cache_control markers if (targetFormat === FORMATS.CLAUDE) { - result = prepareClaudeRequest(result, provider); + const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE; + result = prepareClaudeRequest(result, provider, isClaudePassthrough); } // Normalize openai-responses input shape for providers that require list input. diff --git a/tests/unit/claude-cache-control-passthrough.test.mjs b/tests/unit/claude-cache-control-passthrough.test.mjs new file mode 100644 index 0000000000..5014d04992 --- /dev/null +++ b/tests/unit/claude-cache-control-passthrough.test.mjs @@ -0,0 +1,175 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { prepareClaudeRequest } from "../../open-sse/translator/helpers/claudeHelper.ts"; + +describe("Claude cache_control passthrough", () => { + test("preserveCacheControl=true preserves cache_control in system blocks", () => { + const body = { + system: [ + { type: "text", text: "System prompt 1" }, + { type: "text", text: "System prompt 2", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.system.length, 2); + assert.equal(result.system[0].cache_control, undefined); + assert.deepEqual(result.system[1].cache_control, { type: "ephemeral", ttl: "5m" }); + }); + + test("preserveCacheControl=false replaces cache_control in system blocks", () => { + const body = { + system: [ + { type: "text", text: "System prompt 1" }, + { type: "text", text: "System prompt 2", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + assert.equal(result.system.length, 2); + assert.equal(result.system[0].cache_control, undefined); + assert.deepEqual(result.system[1].cache_control, { type: "ephemeral", ttl: "1h" }); + }); + + test("preserveCacheControl=true preserves cache_control in message content blocks", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "User message 1" }, + { type: "text", text: "User message 2", cache_control: { type: "ephemeral" } }, + ], + }, + { + role: "assistant", + content: [ + { type: "text", text: "Assistant response", cache_control: { type: "ephemeral", ttl: "10m" } }, + ], + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.messages.length, 2); + assert.equal(result.messages[0].content[0].cache_control, undefined); + assert.deepEqual(result.messages[0].content[1].cache_control, { type: "ephemeral" }); + assert.deepEqual(result.messages[1].content[0].cache_control, { type: "ephemeral", ttl: "10m" }); + }); + + test("preserveCacheControl=false strips and re-adds cache_control in messages", () => { + const body = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "User message 1" }, + { type: "text", text: "User message 2", cache_control: { type: "ephemeral" } }, + ], + }, + { + role: "assistant", + content: [ + { type: "text", text: "Assistant response", cache_control: { type: "ephemeral", ttl: "10m" } }, + ], + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + // Original cache_control should be stripped and OmniRoute's strategy applied + assert.equal(result.messages.length, 2); + // User message should not have cache_control (only second-to-last user gets it) + assert.equal(result.messages[0].content[0].cache_control, undefined); + assert.equal(result.messages[0].content[1].cache_control, undefined); + // Last assistant should have cache_control added by OmniRoute + assert.deepEqual(result.messages[1].content[0].cache_control, { type: "ephemeral" }); + }); + + test("preserveCacheControl=true preserves cache_control in tools", () => { + const body = { + messages: [], + tools: [ + { name: "tool1", description: "Tool 1", input_schema: { type: "object" } }, + { + name: "tool2", + description: "Tool 2", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "30m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + assert.equal(result.tools.length, 2); + assert.equal(result.tools[0].cache_control, undefined); + assert.deepEqual(result.tools[1].cache_control, { type: "ephemeral", ttl: "30m" }); + }); + + test("preserveCacheControl=false replaces cache_control in tools", () => { + const body = { + messages: [], + tools: [ + { name: "tool1", description: "Tool 1", input_schema: { type: "object" } }, + { + name: "tool2", + description: "Tool 2", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "30m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", false); + + assert.equal(result.tools.length, 2); + assert.equal(result.tools[0].cache_control, undefined); + assert.deepEqual(result.tools[1].cache_control, { type: "ephemeral", ttl: "1h" }); + }); + + test("preserveCacheControl=true with Claude Code-style caching", () => { + const body = { + system: [ + { type: "text", text: "System", cache_control: { type: "ephemeral", ttl: "5m" } }, + ], + messages: [ + { + role: "user", + content: [{ type: "text", text: "Turn 1", cache_control: { type: "ephemeral" } }], + }, + { + role: "assistant", + content: [{ type: "text", text: "Response 1" }], + }, + { + role: "user", + content: [{ type: "text", text: "Turn 2" }], + }, + ], + tools: [ + { + name: "bash", + description: "Execute bash", + input_schema: { type: "object" }, + cache_control: { type: "ephemeral", ttl: "5m" }, + }, + ], + }; + + const result = prepareClaudeRequest(body, "claude", true); + + // All original cache_control should be preserved + assert.deepEqual(result.system[0].cache_control, { type: "ephemeral", ttl: "5m" }); + assert.deepEqual(result.messages[0].content[0].cache_control, { type: "ephemeral" }); + assert.equal(result.messages[1].content[0].cache_control, undefined); + assert.equal(result.messages[2].content[0].cache_control, undefined); + assert.deepEqual(result.tools[0].cache_control, { type: "ephemeral", ttl: "5m" }); + }); +}); From 94a00cb6d618119f28766c424dfa2912dc1fd483 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 28 Mar 2026 21:53:07 -0300 Subject: [PATCH 2/4] feat: improve dashboard layout for smaller screens (#659) --- src/app/globals.css | 13 +++++++++++++ src/shared/components/layouts/DashboardLayout.tsx | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/app/globals.css b/src/app/globals.css index c5f9362164..ac78afa1ab 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -337,3 +337,16 @@ button .material-symbols-outlined, .traffic-light.green { background: var(--color-traffic-green); } + +/* ── Mobile Layout Fixes (Issue #659) ── */ +@media (max-width: 768px) { + .ant-table-wrapper { + overflow-x: auto; + -webkit-overflow-scrolling: touch; + max-width: 100vw; + } + + .ant-table { + min-width: 600px; /* Prevent columns from crushing together */ + } +} diff --git a/src/shared/components/layouts/DashboardLayout.tsx b/src/shared/components/layouts/DashboardLayout.tsx index c9e068d056..da33e08dcc 100644 --- a/src/shared/components/layouts/DashboardLayout.tsx +++ b/src/shared/components/layouts/DashboardLayout.tsx @@ -57,8 +57,8 @@ export default function DashboardLayout({ children }) { >
setSidebarOpen(true)} /> -
-
+
+
{children}
From b9c7fd879fc63cd73415a1a2b193d729447a36cd Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 28 Mar 2026 23:09:31 -0300 Subject: [PATCH 3/4] fix(core): resolve routing schemas, CLI streaming leaks, and thinking tag extraction --- .npmignore | 9 ++++++++ open-sse/handlers/chatCore.ts | 9 ++++---- open-sse/handlers/responseSanitizer.ts | 9 ++++---- open-sse/utils/stream.ts | 30 ++++++++++++++++++++------ src/shared/validation/schemas.ts | 7 +++++- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/.npmignore b/.npmignore index c5ae66e718..c11b218d07 100644 --- a/.npmignore +++ b/.npmignore @@ -47,3 +47,12 @@ AGENTS.md # Build artifacts (pre-built goes inside app/) .next/ node_modules/ + +# Ignore large binary files and other build directories +*.tgz +*.AppImage +*.deb +*.rpm +electron/ +app/electron/ +app/vscode-extension/ diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 2a3ee9c03e..4cdbf8d139 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -531,10 +531,10 @@ export async function handleChatCore({ connectionId, duration: Date.now() - startTime, tokens: tokens || {}, - requestBody: attachLogMeta(body, { + requestBody: attachLogMeta((body as Record) ?? undefined, { claudePromptCache: claudeCacheMeta, }), - responseBody: attachLogMeta(responseBody ?? undefined, { + responseBody: attachLogMeta((responseBody as Record) ?? undefined, { claudePromptCache: claudeCacheMeta ? { applied: claudeCacheMeta.applied, @@ -1464,8 +1464,9 @@ export async function handleChatCore({ // Sanitize response for OpenAI SDK compatibility // Strips non-standard fields (x_groq, usage_breakdown, service_tier, etc.) - // Extracts tags into reasoning_content - if (sourceFormat === FORMATS.OPENAI) { + // Extracts and tags into reasoning_content + // Target format determines output shape. If we are outputting OpenAI shape or pseudo-OpenAI shape, sanitize. + if (targetFormat === FORMATS.OPENAI || targetFormat === FORMATS.OPENAI_RESPONSES) { translatedResponse = sanitizeOpenAIResponse(translatedResponse); } diff --git a/open-sse/handlers/responseSanitizer.ts b/open-sse/handlers/responseSanitizer.ts index cb56c5efda..24e3614831 100644 --- a/open-sse/handlers/responseSanitizer.ts +++ b/open-sse/handlers/responseSanitizer.ts @@ -32,13 +32,12 @@ function toNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) ? value : undefined; } -// ── Think tag regex ──────────────────────────────────────────────────────── -// Matches ... blocks (greedy, dotAll) -const THINK_TAG_REGEX = /([\s\S]*?)<\/think>/gi; +// Matches ... blocks and ... (greedy, dotAll) +const THINK_TAG_REGEX = /<(?:think|thinking)>([\s\S]*?)<\/(?:think|thinking)>/gi; -// #638: Collapse runs of 3+ consecutive newlines into \n\n +// #638, #727: Collapse runs of 2+ consecutive newlines into \n\n // Tool call responses from thinking models often accumulate excessive newlines -const EXCESSIVE_NEWLINES = /\n{3,}/g; +const EXCESSIVE_NEWLINES = /\n{2,}/g; function collapseExcessiveNewlines(text: string): string { return text.replace(EXCESSIVE_NEWLINES, "\n\n"); } diff --git a/open-sse/utils/stream.ts b/open-sse/utils/stream.ts index 8a2fe511f5..b8c788036e 100644 --- a/open-sse/utils/stream.ts +++ b/open-sse/utils/stream.ts @@ -528,31 +528,47 @@ export function createSSEStream(options: StreamOptions = {}) { // Content for call log is accumulated only from parsed (above) to avoid double-counting; // do not add again from item here. + // #723, #727: Sanitize intermediate stream chunks if target is OpenAI format loop + let itemSanitized: Record = item; + if (targetFormat === FORMATS.OPENAI || targetFormat === FORMATS.OPENAI_RESPONSES) { + itemSanitized = sanitizeStreamingChunk(itemSanitized) as Record; + + // Extract reasoning tags from content if translation generated them + const delta = itemSanitized?.choices?.[0]?.delta; + if (delta?.content && typeof delta.content === "string") { + const { content, thinking } = extractThinkingFromContent(delta.content); + delta.content = content; + if (thinking && !delta.reasoning_content) { + delta.reasoning_content = thinking; + } + } + } + // Filter empty chunks - if (!hasValuableContent(item, sourceFormat)) { + if (!hasValuableContent(itemSanitized, sourceFormat)) { continue; // Skip this empty chunk } // Inject estimated usage if finish chunk has no valid usage const isFinishChunk = - item.type === "message_delta" || item.choices?.[0]?.finish_reason; + itemSanitized.type === "message_delta" || itemSanitized.choices?.[0]?.finish_reason; if ( state.finishReason && isFinishChunk && - !hasValidUsage(item.usage) && + !hasValidUsage(itemSanitized.usage) && totalContentLength > 0 ) { const estimated = estimateUsage(body, totalContentLength, sourceFormat); - item.usage = filterUsageForFormat(estimated, sourceFormat); // Filter + already has buffer + itemSanitized.usage = filterUsageForFormat(estimated, sourceFormat); // Filter + already has buffer state.usage = estimated; } else if (state.finishReason && isFinishChunk && state.usage) { // Add buffer and filter usage for client (but keep original in state.usage for logging) const buffered = addBufferToUsage(state.usage); - item.usage = filterUsageForFormat(buffered, sourceFormat); + itemSanitized.usage = filterUsageForFormat(buffered, sourceFormat); } - const output = formatSSE(item, sourceFormat); - clientPayloadCollector.push(item); + const output = formatSSE(itemSanitized, sourceFormat); + clientPayloadCollector.push(itemSanitized); reqLogger?.appendConvertedChunk?.(output); controller.enqueue(encoder.encode(output)); } diff --git a/src/shared/validation/schemas.ts b/src/shared/validation/schemas.ts index f4d58c66df..0c32f398c6 100644 --- a/src/shared/validation/schemas.ts +++ b/src/shared/validation/schemas.ts @@ -78,6 +78,9 @@ const comboStrategySchema = z.enum([ "cost-optimized", "strict-random", "auto", + "fill-first", + // #729 schema fixes for combo edit/save + "p2c", ]); const comboRuntimeConfigSchema = z @@ -884,6 +887,7 @@ export const updateComboSchema = z system_message: z.string().max(50000).optional(), tool_filter_regex: z.string().max(1000).optional(), context_cache_protection: z.boolean().optional(), + context_length: z.number().int().min(1000).max(2000000).optional(), }) .superRefine((value, ctx) => { if ( @@ -895,7 +899,8 @@ export const updateComboSchema = z value.allowedProviders === undefined && value.system_message === undefined && value.tool_filter_regex === undefined && - value.context_cache_protection === undefined + value.context_cache_protection === undefined && + value.context_length === undefined ) { ctx.addIssue({ code: z.ZodIssueCode.custom, From c0cbf001998f24132a763906b36cabdaa9da21ae Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Sat, 28 Mar 2026 23:19:01 -0300 Subject: [PATCH 4/4] =?UTF-8?q?chore(release):=20v3.2.3=20=E2=80=94=20Enha?= =?UTF-8?q?ncements=20and=20Bugfixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 14 ++++++++++++++ docs/openapi.yaml | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11296be2ac..ec6b7fc26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ --- +## [3.2.3] — 2026-03-29 + +### ✨ Enhancements & Refactoring + +- **Provider Limits Quota UI (#728)** — Normalized quota limit logic and data labeling inside the Limits interface. + +### 🐛 Bug Fixes + +- **Core Routing Schemas & Leaks** — Expanded `comboStrategySchema` to natively support `fill-first` and `p2c` strategies to unblock complex combo editing natively. +- **Thinking Tags Extraction (CLI)** — Restructured CLI token responses sanitizer RegEx capturing model reasoning structures inside streams avoiding broken `` extractions breaking response text output format. +- **Strict Format Enforcements** — Hardened pipeline sanitization execution making it universally apply to translation mode targets. + +--- + ## [3.2.2] — 2026-03-29 ### ✨ New Features diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e9ffab3bbb..e5e46d9cb1 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OmniRoute API - version: 3.2.2 + version: 3.2.3 description: | OmniRoute is a local-first AI API proxy router. It provides an OpenAI-compatible endpoint that routes requests to multiple AI providers with load balancing, diff --git a/package-lock.json b/package-lock.json index 2f79d534ba..e32d44a088 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "omniroute", - "version": "3.2.2", + "version": "3.2.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "omniroute", - "version": "3.2.2", + "version": "3.2.3", "hasInstallScript": true, "license": "MIT", "workspaces": [ diff --git a/package.json b/package.json index 9e9779b72a..171ca763d3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "omniroute", - "version": "3.2.2", + "version": "3.2.3", "description": "Smart AI Router with auto fallback — route to FREE & cheap models, zero downtime. Works with Cursor, Cline, Claude Desktop, Codex, and any OpenAI-compatible tool.", "type": "module", "bin": {