diff --git a/CHANGELOG.md b/CHANGELOG.md index aaba4567ee..fd8b0973da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ _In development — bullets added per PR; finalized at release._ ### 🐛 Fixed +- **fix(compression): preserve the cacheable prefix for automatic-cache providers** — OpenAI / Codex (and Azure-OpenAI) use _automatic_ prefix caching: the upstream caches the longest matching prefix of a request (system prompt + earliest messages) **without** any explicit `cache_control` markers in the body. The cache-aware compression guard only protected that prefix when the request carried explicit `cache_control`, so for automatic-cache providers the guard was skipped — and with compression enabled and `preserveSystemPrompt: false` (or a prefix-compressing mode like `aggressive`/`ultra`) it rewrote the system prompt / earliest messages, guaranteeing a cache miss and **higher** token spend through OmniRoute than going direct. The guard now treats a caching provider as sufficient on its own (`isCachingProvider` alone, independent of `cache_control`) to skip the system prompt and downgrade prefix-compressing modes, and OpenAI/Codex/Azure are now recognized as caching providers. Compression is still off by default — this only affects operators who enabled it with prefix preservation turned off. ([#3955](https://github.com/diegosouzapw/OmniRoute/issues/3955)) - **fix(executors): DuckDuckGo AI Chat uses duckduckgo.com (fixes 400)** — the DuckDuckGo AI Chat executor fetched status/chat and set `Origin`/`Referer` against `https://duck.ai` while still sending `Sec-Fetch-Site: same-origin`, so the request's same-origin triplet (host + Origin + Referer) was inconsistent and the backend rejected it with HTTP 400. All current DDG reverse-engineering references — and the provider registry's own `baseUrl` — use `https://duckduckgo.com`; the executor now uses it consistently for the status URL, chat URL, `Origin`, and `Referer` (the same-origin header is now coherent). The `x-fe-version` scrape regex also required a 40-hex tail but the real served token has a 20-hex tail (e.g. `serp_20250401_100419_ET-19d438eb199b2bf7c300`), so it silently fell back to a hardcoded default; the pattern is relaxed to a bounded `{20,40}` tail (still ReDoS-safe). This addresses the DuckDuckGo half of the report; the separate Chipotle/`chipotle` upstream breakage is tracked independently. ([#4037](https://github.com/diegosouzapw/OmniRoute/issues/4037) — thanks @daniij) - **fix(security): bound the prompt-injection scan to the first 16 KB (hot-path perf)** — the prompt-injection guard joined every message/system string into one buffer and ran several regexes over the **whole** thing on every chat request, with no size cap — so a 300 KB body (pasted code, RAG context) meant O(body) CPU scanning on the hot path, a self-inflicted latency/GC source under concurrency. Both detection call sites (`detectInjection` in `inputSanitizer.ts` and the custom-pattern scan in `promptInjection.ts`) now slice the joined text to the first **16 KB** (`MAX_INJECTION_SCAN_BYTES`) before the regex loop. Injection directives sit near the top of a prompt, so the generous cap preserves real detection while scanning only a bounded prefix; the existing 10 MB body-size cap (which protects ingestion) is unchanged. ([#3932](https://github.com/diegosouzapw/OmniRoute/issues/3932) — thanks @KooshaPari) - **fix(sse): retry direct-connection socket failures on a fresh socket (fewer `502` bursts)** — the default direct-connection undici dispatcher pools keep-alive sockets for up to 4 s, but some edges (e.g. `nvidia`, `opencode-zen`) silently close idle keep-alive sockets within that window, so the next request reusing a pooled socket fails with `UND_ERR_SOCKET` ("other side closed") — in bursts. `proxyFetch` already retried once on such transient errors, but the retry reused the **same** pooled dispatcher and could grab another stale socket, then fell through to native fetch (which also pools) → the job sat in the rate-limit queue until the 30 s timeout → `502` + circuit-breaker open. The retry now uses a dedicated **no-keep-alive / no-pipelining** dispatcher so it opens a brand-new socket that can't be a dead pooled one; the first attempt still uses the pooled dispatcher (healthy keep-alive reuse is preserved). Complements the v3.8.29 diagnostics (`describeFetchCause`, #4281). ([#4252](https://github.com/diegosouzapw/OmniRoute/issues/4252) — thanks @klimadev) diff --git a/open-sse/services/compression/cachingAware.ts b/open-sse/services/compression/cachingAware.ts index 299127be53..54a72ed638 100644 --- a/open-sse/services/compression/cachingAware.ts +++ b/open-sse/services/compression/cachingAware.ts @@ -106,8 +106,14 @@ export function detectCachingContext( * @returns A CacheAwareStrategy object */ export function getCacheAwareStrategy(strategy: string, ctx: CachingContext): CacheAwareStrategy { - if (ctx.isCachingProvider && ctx.hasCacheControl) { - // Adjust strategy for caching providers with cache control + // #3955: a caching provider is enough on its own to protect the cacheable prefix. + // OpenAI / Codex (and other automatic-prefix-cache providers) carry NO explicit + // `cache_control` markers, yet the upstream still caches the longest matching prefix + // (system prompt / earliest messages). Gating on `hasCacheControl` skipped the guard + // for those providers, so a prefix-compressing mode rewrote the prefix → guaranteed + // cache miss. Treat `isCachingProvider` alone as sufficient; the explicit + // `cache_control` path is now a subset of this. + if (ctx.isCachingProvider) { return { strategy: ["aggressive", "ultra"].includes(strategy) ? "standard" : strategy, skipSystemPrompt: true, diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index 7fcae00510..3603ce7a8f 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -83,6 +83,15 @@ const CACHING_PROVIDERS = new Set([ // clients and filterToOpenAIFormat() strips cache_control, so Xiaomi never // sees the cache hints and every request is a cache miss. "xiaomi-mimo", + // #3955 — OpenAI / Codex / Azure-OpenAI use AUTOMATIC prefix caching: the longest + // matching prefix of a request is cached upstream WITHOUT any explicit cache_control + // markers. They must count as caching providers so the cache-aware compression guard + // preserves the cacheable prefix (system prompt / earliest messages) instead of + // rewriting it and forcing a cache miss. This also activates the intended + // `prompt_cache_key` cache-routing hint for OpenAI in chatCore. + "openai", + "codex", + "azure", ]); /** diff --git a/tests/unit/cache-control-claude-providers.test.ts b/tests/unit/cache-control-claude-providers.test.ts index 01e758a175..d868c796b3 100644 --- a/tests/unit/cache-control-claude-providers.test.ts +++ b/tests/unit/cache-control-claude-providers.test.ts @@ -22,8 +22,11 @@ describe("Cache Control Policy - Claude Protocol Providers", () => { assert.equal(providerSupportsCaching("minimax-cn", "claude"), true); assert.equal(providerSupportsCaching("kimi-coding", "claude"), true); - // Non-Claude providers without caching support - assert.equal(providerSupportsCaching("openai", "openai"), false); + // #3955 — OpenAI / Codex use automatic prefix caching (no cache_control needed). + assert.equal(providerSupportsCaching("openai", "openai"), true); + assert.equal(providerSupportsCaching("codex", "openai"), true); + + // Non-caching providers assert.equal(providerSupportsCaching("gemini", "gemini"), false); }); @@ -126,12 +129,13 @@ describe("Cache Control Policy - Claude Protocol Providers", () => { test("shouldPreserveCacheControl does not preserve for non-Claude format providers", () => { const claudeCodeUA = "Claude-Code/1.0.0"; + // gemini is non-Claude-format and has no prompt caching (openai/codex now do, #3955). assert.equal( shouldPreserveCacheControl({ userAgent: claudeCodeUA, isCombo: false, - targetProvider: "openai", - targetFormat: "openai", + targetProvider: "gemini", + targetFormat: "gemini", settings: { alwaysPreserveClientCache: "auto" }, }), false diff --git a/tests/unit/cache-control-policy.test.ts b/tests/unit/cache-control-policy.test.ts index 9708d294e6..0f67de43b5 100644 --- a/tests/unit/cache-control-policy.test.ts +++ b/tests/unit/cache-control-policy.test.ts @@ -42,10 +42,13 @@ describe("Cache Control Policy", () => { // #3088 — Xiaomi MiMo supports prompt caching; cache_control breakpoints // sent by Claude Code (via cc-switch) must be preserved, not stripped. assert.equal(providerSupportsCaching("xiaomi-mimo"), true); + // #3955 — OpenAI / Codex / Azure-OpenAI use automatic prefix caching. + assert.equal(providerSupportsCaching("openai"), true); + assert.equal(providerSupportsCaching("codex"), true); + assert.equal(providerSupportsCaching("azure"), true); }); test("rejects non-caching providers", () => { - assert.equal(providerSupportsCaching("openai"), false); assert.equal(providerSupportsCaching("gemini"), false); assert.equal(providerSupportsCaching("unknown"), false); assert.equal(providerSupportsCaching(null), false); @@ -142,11 +145,12 @@ describe("Cache Control Policy", () => { }); test("rejects non-caching providers", () => { + // gemini has no prompt caching (openai/codex now do, per #3955). assert.equal( shouldPreserveCacheControl({ userAgent: "claude-code/0.1.0", isCombo: false, - targetProvider: "openai", + targetProvider: "gemini", }), false ); diff --git a/tests/unit/compression-cache-guard-3955.test.ts b/tests/unit/compression-cache-guard-3955.test.ts new file mode 100644 index 0000000000..6794d4ff8e --- /dev/null +++ b/tests/unit/compression-cache-guard-3955.test.ts @@ -0,0 +1,127 @@ +/** + * #3955 — Compression must not wreck automatic prefix caching. + * + * OpenAI / Codex (and other OpenAI-format providers with automatic prompt caching) + * cache the longest matching prefix of a request WITHOUT any explicit `cache_control` + * markers in the body. The old cache-aware guard only protected the cacheable prefix + * when BOTH `isCachingProvider` AND `hasCacheControl` were true, so for automatic-cache + * providers (no `cache_control` markers) the guard was skipped. With compression active + * and `preserveSystemPrompt: false` (or a prefix-compressing mode) this rewrote the + * system prompt / earliest messages and guaranteed a cache miss — higher token spend + * through OmniRoute than going direct. + * + * Fix: `isCachingProvider` ALONE is sufficient to protect the prefix (skipSystemPrompt), + * independent of explicit `cache_control`. A non-caching provider is unaffected. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + detectCachingContext, + getCacheAwareStrategy, +} from "../../open-sse/services/compression/cachingAware.ts"; +import { resolveCacheAwareConfig } from "../../open-sse/services/compression/strategySelector.ts"; +import type { CompressionConfig } from "../../open-sse/services/compression/types.ts"; + +const LONG_SYSTEM_PROMPT = + "You are a meticulous coding assistant. ".repeat(64) + + "Follow every instruction precisely and never omit details."; + +function autoCacheBody(model: string) { + // NOTE: deliberately NO cache_control markers anywhere — this mirrors how + // OpenAI / Codex automatic prefix caching works (the prefix is cached implicitly). + return { + model, + messages: [ + { role: "system", content: LONG_SYSTEM_PROMPT }, + { role: "user", content: "Refactor this function for clarity." }, + ], + }; +} + +function cfg(overrides: Partial = {}): CompressionConfig { + return { + enabled: true, + defaultMode: "aggressive", + autoTriggerTokens: 0, + cacheMinutes: 5, + preserveSystemPrompt: true, + comboOverrides: {}, + ...overrides, + } as CompressionConfig; +} + +describe("#3955 automatic-cache prefix protection (no explicit cache_control)", () => { + it("treats openai as a caching provider for automatic prefix caching", () => { + const ctx = detectCachingContext(autoCacheBody("openai/gpt-4o"), { provider: "openai" }); + assert.equal(ctx.hasCacheControl, false, "no explicit cache_control markers present"); + assert.equal(ctx.isCachingProvider, true, "openai has automatic prefix caching"); + }); + + it("treats codex as a caching provider for automatic prefix caching", () => { + const ctx = detectCachingContext(autoCacheBody("codex/gpt-5-codex"), { provider: "codex" }); + assert.equal(ctx.hasCacheControl, false); + assert.equal(ctx.isCachingProvider, true, "codex has automatic prefix caching"); + }); + + it("skips/protects the system prompt for an auto-cache provider WITHOUT cache_control", () => { + const ctx = detectCachingContext(autoCacheBody("openai/gpt-4o"), { provider: "openai" }); + const result = getCacheAwareStrategy("aggressive", ctx); + // The cacheable prefix must be preserved even though no cache_control markers exist. + assert.equal(result.skipSystemPrompt, true); + // Prefix-compressing modes are downgraded so the cacheable prefix is not rewritten. + assert.equal(result.strategy, "standard"); + assert.equal(result.deterministicOnly, true); + }); + + it("protects codex the same way (ultra downgraded, system prompt skipped)", () => { + const ctx = detectCachingContext(autoCacheBody("codex/gpt-5-codex"), { provider: "codex" }); + const result = getCacheAwareStrategy("ultra", ctx); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.strategy, "standard"); + }); + + it("forces preserveSystemPrompt on for an auto-cache request that disabled it", () => { + // This is the end-to-end cache-miss scenario from #3955: compression active, + // preserveSystemPrompt explicitly off, automatic-cache provider, no cache_control. + const out = resolveCacheAwareConfig( + cfg({ preserveSystemPrompt: false }), + autoCacheBody("openai/gpt-4o"), + { provider: "openai" } + ); + assert.equal(out.preserveSystemPrompt, true, "cacheable prefix must stay uncompressed"); + }); + + it("leaves a NON-caching provider unaffected (no prefix protection without cache_control)", () => { + const ctx = detectCachingContext(autoCacheBody("google/gemini-2.5-pro"), { provider: "google" }); + assert.equal(ctx.isCachingProvider, false); + const result = getCacheAwareStrategy("aggressive", ctx); + assert.equal(result.skipSystemPrompt, false); + assert.equal(result.strategy, "aggressive"); + assert.equal(result.deterministicOnly, false); + + // And the config is left untouched (preserveSystemPrompt stays false). + const out = resolveCacheAwareConfig( + cfg({ preserveSystemPrompt: false }), + autoCacheBody("google/gemini-2.5-pro"), + { provider: "google" } + ); + assert.equal(out.preserveSystemPrompt, false); + }); + + it("still protects explicit cache_control providers (existing #3890 behavior intact)", () => { + const ctx = detectCachingContext( + { + messages: [ + { role: "system", content: "x", cache_control: { type: "ephemeral" } }, + { role: "user", content: "hi" }, + ], + }, + { provider: "anthropic", targetFormat: "claude" } + ); + assert.equal(ctx.hasCacheControl, true); + const result = getCacheAwareStrategy("aggressive", ctx); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.strategy, "standard"); + }); +}); diff --git a/tests/unit/compression/cachingAware.test.ts b/tests/unit/compression/cachingAware.test.ts index ce33905705..cf854f3522 100644 --- a/tests/unit/compression/cachingAware.test.ts +++ b/tests/unit/compression/cachingAware.test.ts @@ -26,10 +26,11 @@ describe("detectCachingContext", () => { assert.equal(ctx.isCachingProvider, true); }); - it("extracts openai provider from model string", () => { + it("extracts openai provider from model string (automatic prefix caching, #3955)", () => { const ctx = detectCachingContext({ model: "openai/gpt-4o" }); assert.equal(ctx.provider, "openai"); - assert.equal(ctx.isCachingProvider, false); + // #3955 — OpenAI uses automatic prefix caching; it counts as a caching provider. + assert.equal(ctx.isCachingProvider, true); }); it("extracts google provider from model string", () => { @@ -133,12 +134,15 @@ describe("getCacheAwareStrategy", () => { assert.equal(result.deterministicOnly, false); }); - it("keeps strategy unchanged when no cache_control even for caching provider", () => { + it("protects the prefix for a caching provider even WITHOUT cache_control (#3955)", () => { + // #3955 — automatic prefix caching (OpenAI/Codex/Anthropic) sets no cache_control + // markers, but the cacheable prefix must still be preserved. isCachingProvider alone + // is sufficient to skip the system prompt and downgrade prefix-compressing modes. const ctx = { hasCacheControl: false, provider: "anthropic", isCachingProvider: true }; const result = getCacheAwareStrategy("aggressive", ctx); - assert.equal(result.strategy, "aggressive"); - assert.equal(result.skipSystemPrompt, false); - assert.equal(result.deterministicOnly, false); + assert.equal(result.strategy, "standard"); + assert.equal(result.skipSystemPrompt, true); + assert.equal(result.deterministicOnly, true); }); it("returns none strategy unchanged", () => { diff --git a/tests/unit/compression/strategySelector-cache-aware.test.ts b/tests/unit/compression/strategySelector-cache-aware.test.ts index c7d1963282..880108b6e2 100644 --- a/tests/unit/compression/strategySelector-cache-aware.test.ts +++ b/tests/unit/compression/strategySelector-cache-aware.test.ts @@ -34,10 +34,12 @@ describe("resolveCacheAwareConfig (#3890)", () => { }); it("leaves a non-caching request untouched (preserveSystemPrompt stays false)", () => { + // google has no prompt caching, so the prefix-protection guard does not apply. + // (openai/codex now count as automatic-cache providers per #3955.) const out = resolveCacheAwareConfig( cfg({ preserveSystemPrompt: false }), { messages: [{ role: "system", content: "x" }] }, - { provider: "openai" } + { provider: "google" } ); assert.equal(out.preserveSystemPrompt, false); });