From 61c7aecd3bb70001d00b4bebd91be4d29d514105 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Wed, 1 Jul 2026 00:18:04 -0300 Subject: [PATCH] =?UTF-8?q?feat(compression):=20T08/H5=20(2.4)=20=E2=80=94?= =?UTF-8?q?=20usage-observed=20prefix=20freeze=20(opt-in)=20(#5744)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evolves the cache-aware guard to also learn which system prompts recur: observed >= threshold → treated as a stable cacheable prefix and preserved even for providers the static check misses. Content-addressed by a hash of the system prompt (OpenAI/Claude/Gemini), in-memory, freeze=preserve (never mutates). Opt-in/default-off (COMPRESSION_PREFIX_FREEZE_ENABLED); respects the never preserve-mode. New prefixFreeze.ts wired into resolveCacheAwareConfig. Regression guard: prefix-freeze.test.ts (10); 44 cache-aware tests green. gaps v3.8.42 — T08/H5 (2.4). --- .env.example | 6 + CHANGELOG.md | 2 + docs/reference/ENVIRONMENT.md | 2 + .../services/compression/cacheAwareConfig.ts | 25 +++- open-sse/services/compression/prefixFreeze.ts | 124 ++++++++++++++++++ tests/unit/compression/prefix-freeze.test.ts | 121 +++++++++++++++++ 6 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 open-sse/services/compression/prefixFreeze.ts create mode 100644 tests/unit/compression/prefix-freeze.test.ts diff --git a/.env.example b/.env.example index 69717d0036..e3bc58391d 100644 --- a/.env.example +++ b/.env.example @@ -694,6 +694,12 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # (>= 3 retrievals = never compressed). 1 disables the ramp (binary skip at the threshold only). # Used by: open-sse/services/compression/engines/ccr/index.ts. Default: 2. #COMPRESSION_CCR_RETRIEVAL_RAMP_FACTOR=2 +# T08/H5 — usage-observed prefix freeze (OPT-IN, default off). When enabled, a system prompt seen +# >= THRESHOLD times is treated as a stable cacheable prefix and preserved from compression even +# for providers the static cache-aware heuristic does not recognize (freeze = preserve, never +# mutates). Used by: open-sse/services/compression/prefixFreeze.ts. +#COMPRESSION_PREFIX_FREEZE_ENABLED=false # master switch (default false) +#COMPRESSION_PREFIX_FREEZE_THRESHOLD=3 # observations before a prefix is frozen # Skip the postinstall native-runtime warm-up (useful in CI / headless installs). Default: 0. # Used by: scripts/postinstall.mjs. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ba7917baa..6f2a114903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ - **compression (CCR):** the CCR retrieval-feedback (H8) is now **graduated** instead of a binary cliff. Previously a block retrieved `>= 3` times was flagged do-not-compress and everything below that stayed fully compressible. Now each prior retrieval raises a block's **effective `minChars` linearly** (`effectiveMinChars`), so frequently-retrieved content is compressed progressively less; the `>= 3` exclusion is preserved (as `Infinity`). The ramp is controlled by a `retrievalRampFactor` (default `2`, per-combo config or `COMPRESSION_CCR_RETRIEVAL_RAMP_FACTOR`); `1` reproduces the exact legacy binary behavior. Per-`(principal, hash)` isolation is unchanged. Regression guard: `tests/unit/compression/ccr-retrieval-ramp.test.ts` (12); existing CCR suites (51) stay green. gaps v3.8.42 — T08/H8. +- **compression (cache-aware):** add an **opt-in, default-off usage-observed prefix freeze** (H5). The cache-aware guard previously preserved the system prompt only for providers a *static* heuristic recognized as caching. It now also **learns** which system prompts actually recur: once a system prompt has been observed `>=` a threshold across requests, it is treated as a stable cacheable prefix and preserved from compression **even for providers the static check misses** — recovering prompt-cache hits that a prefix-compressing mode would otherwise bust. Content-addressed by a hash of the system prompt (OpenAI / Claude / Gemini shapes), in-memory + bounded, zero DB/IO; a "freeze" only *preserves* the prefix, so it never mutates a payload. Default OFF (`COMPRESSION_PREFIX_FREEZE_ENABLED`, threshold `_THRESHOLD`); respects the `never` preserve-mode (never freezes). New `open-sse/services/compression/prefixFreeze.ts`, wired into `resolveCacheAwareConfig`. Regression guard: `tests/unit/compression/prefix-freeze.test.ts` (10); 44 existing cache-aware / preserve-mode tests stay green. gaps v3.8.42 — T08/H5. + ### 🔧 Bug Fixes - **thinking / runtime-config (module-graph fix):** operator-configured proxy settings that are hydrated at **boot** but read **per-request** were silently ignored in production. Next.js compiles `instrumentation.ts` (boot hydration via `applyRuntimeSettings` / restore hooks) as a **separate webpack module graph** from the app-route / open-sse executors, so a module-local `let _config` singleton is **duplicated** — the boot copy is hydrated but the request path reads a different, un-hydrated copy. Live VPS validation proved the Thinking-Budget hydration ran to completion at boot yet `base.ts` still saw the `passthrough` default (this is why #5312 fix A stayed broken even after the boot-wiring fix). Fixed by backing the singletons with `globalThis` (the pattern `systemPrompt.ts` already uses for the Global System Prompt, #2470), so all module-graph copies share one instance: **`thinkingBudget.ts`** (the dashboard Thinking-Budget mode now reaches the executor), **`backgroundTaskDetector.ts`** (the opt-in background-model degradation now actually fires on requests), and **`systemTransforms.ts`** (operator pipeline overrides now reach the request path). `payloadRules.ts` was already safe (it lazily self-loads from the DB per request, #2986). Regression guards: `tests/unit/thinking-budget-globalthis-5312.test.ts` + `tests/unit/runtime-config-globalthis-5312.test.ts` (assert globalThis-backed sharing; a module-local `let` fails them). ([#5312](https://github.com/diegosouzapw/OmniRoute/issues/5312)) diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index f0cbe19bcb..7c39548eb2 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -410,6 +410,8 @@ detection above). | `COMPRESSION_PIPELINE_BREAKER_THRESHOLD` | `3` | `open-sse/services/compression/pipelineEngineBreaker.ts` | Consecutive cross-request failures before an engine's breaker opens. | | `COMPRESSION_PIPELINE_BREAKER_COOLDOWN_MS` | `30000` | `open-sse/services/compression/pipelineEngineBreaker.ts` | Milliseconds an opened engine stays skipped before a half-open probe. | | `COMPRESSION_CCR_RETRIEVAL_RAMP_FACTOR` | `2` | `open-sse/services/compression/engines/ccr/index.ts` | T08/H8 CCR retrieval-feedback ramp: each prior retrieval of a stored block raises its effective `minChars` linearly (frequently-retrieved content compresses less; `>=3` retrievals = never compressed). `1` disables the ramp (binary skip at the threshold only). | +| `COMPRESSION_PREFIX_FREEZE_ENABLED` | `false` | `open-sse/services/compression/prefixFreeze.ts` | T08/H5 usage-observed prefix freeze master switch. **Opt-in (default off)** — when on, a system prompt observed `>=` the threshold is treated as a stable cacheable prefix and preserved from compression even for providers the static cache heuristic misses (freeze only *preserves*, never mutates). | +| `COMPRESSION_PREFIX_FREEZE_THRESHOLD` | `3` | `open-sse/services/compression/prefixFreeze.ts` | Observations of a system prompt before it is treated as a frozen stable prefix. | | `OMNIROUTE_BOOTSTRAPPED` | `false` | `src/app/(dashboard)/dashboard/page.tsx` | Set `true` by bootstrap script after initial setup. Controls setup wizard visibility. | | `OMNIROUTE_ALLOW_BODY_PROJECT_OVERRIDE` | `0` | `open-sse/executors/antigravity.ts` | Escape hatch: allow request body to override the Antigravity project field. | | `ANTIGRAVITY_CREDITS` | _(unset)_ | `open-sse/services/antigravityCredits.ts` | Override Antigravity's advertised remaining credits (testing / forced values). | diff --git a/open-sse/services/compression/cacheAwareConfig.ts b/open-sse/services/compression/cacheAwareConfig.ts index e6d1f2d447..cf7b6127cb 100644 --- a/open-sse/services/compression/cacheAwareConfig.ts +++ b/open-sse/services/compression/cacheAwareConfig.ts @@ -5,6 +5,27 @@ import { normalizePreserveSystemPromptMode, resolvePreserveSystemPrompt, } from "./preserveSystemPromptMode.ts"; +import { + resolvePrefixFreezeConfig, + extractStablePrefixHash, + observePrefix, + isPrefixFrozen, +} from "./prefixFreeze.ts"; + +/** + * T08/H5 — augment the static cache signal with usage-observed prefix freeze. Opt-in + * (default off): when a system prompt has recurred `>= threshold` times, treat it as a stable + * cacheable prefix to preserve even for a provider the static check does not flag as caching. + * "Freeze" only *preserves* the prefix, so it never corrupts a payload. + */ +function observeAndCheckPrefixFreeze(body: Record): boolean { + const cfg = resolvePrefixFreezeConfig(); + if (!cfg.enabled) return false; + const hash = extractStablePrefixHash(body); + if (!hash) return false; + observePrefix(hash); + return isPrefixFrozen(hash, cfg.threshold); +} /** * #3890/#3955 + T05/C5: materialize the engine-facing `preserveSystemPrompt` @@ -27,9 +48,11 @@ export function resolveCacheAwareConfig( ): CompressionConfig { const mode = normalizePreserveSystemPromptMode(config); // No request body → no cacheable prefix to detect; honor the mode at its no-cache baseline. - const hasCache = body + // The signal is the static caching heuristic OR (H5, opt-in) a usage-observed stable prefix. + const staticCache = body ? getCacheAwareStrategy(config.defaultMode, detectCachingContext(body, context)).skipSystemPrompt : false; + const hasCache = staticCache || (body ? observeAndCheckPrefixFreeze(body) : false); const effective = resolvePreserveSystemPrompt(mode, { hasCache }); if (effective === config.preserveSystemPrompt) return config; return { ...config, preserveSystemPrompt: effective }; diff --git a/open-sse/services/compression/prefixFreeze.ts b/open-sse/services/compression/prefixFreeze.ts new file mode 100644 index 0000000000..a46595ea4d --- /dev/null +++ b/open-sse/services/compression/prefixFreeze.ts @@ -0,0 +1,124 @@ +/** + * T08/H5 — usage-observed prefix freeze (gaps v3.8.42; opt-in, default OFF). + * + * Augments the static cache-aware heuristic (`getCacheAwareStrategy`). Instead of only freezing the + * system prompt for providers the static check knows to cache, it OBSERVES which system prompts + * actually recur across requests and, once one has been seen `>= threshold` times, treats it as a + * stable cacheable prefix to preserve — even for a provider the static check does not recognize. + * + * Content-addressed by a hash of the system prompt (no principal needed): a "freeze" only + * *preserves* the prefix from compression, which never corrupts a payload and is safe to share + * across tenants. In-memory + bounded, zero DB/IO. Default OFF + * (`COMPRESSION_PREFIX_FREEZE_ENABLED`); when off the observer is never consulted (zero cost). + */ + +import crypto from "node:crypto"; + +export interface PrefixFreezeConfig { + /** Master switch. Default false — the resolver never observes/consults when off. */ + enabled: boolean; + /** Times a system prompt must be observed before it is treated as a frozen stable prefix. */ + threshold: number; +} + +export const DEFAULT_PREFIX_FREEZE: PrefixFreezeConfig = { enabled: false, threshold: 3 }; + +/** Upper bound on tracked prefixes (oldest evicted first), mirroring the CCR store cap. */ +export const MAX_PREFIX_ENTRIES = 5_000; + +const observations = new Map(); + +function boundedInc(hash: string): void { + if (!observations.has(hash) && observations.size >= MAX_PREFIX_ENTRIES) { + const oldest = observations.keys().next().value; + if (oldest !== undefined) observations.delete(oldest); + } + observations.set(hash, (observations.get(hash) ?? 0) + 1); +} + +function toPositiveInt(raw: string | undefined, fallback: number): number { + if (raw === undefined) return fallback; + const n = Number(raw); + return Number.isFinite(n) && n >= 1 ? Math.floor(n) : fallback; +} + +/** Resolve config from env (`COMPRESSION_PREFIX_FREEZE_ENABLED` / `_THRESHOLD`). Opt-in. */ +export function resolvePrefixFreezeConfig( + env: NodeJS.ProcessEnv = process.env +): PrefixFreezeConfig { + return { + enabled: env.COMPRESSION_PREFIX_FREEZE_ENABLED === "true", + threshold: toPositiveInt( + env.COMPRESSION_PREFIX_FREEZE_THRESHOLD, + DEFAULT_PREFIX_FREEZE.threshold + ), + }; +} + +// ─── prefix extraction ────────────────────────────────────────────────────────── + +function isRecord(v: unknown): v is Record { + return v !== null && typeof v === "object" && !Array.isArray(v); +} + +/** + * Pull plain text out of the system-prompt shapes: `string`, `Array<…>`, `{text}` (OpenAI/Claude + * text parts), and `{parts: [...]}` (Gemini `systemInstruction`). + */ +function collectText(value: unknown, out: string[]): void { + if (typeof value === "string") { + if (value.trim()) out.push(value); + return; + } + if (Array.isArray(value)) { + for (const item of value) collectText(item, out); + return; + } + if (isRecord(value)) { + if (typeof value.text === "string" && value.text.trim()) out.push(value.text); + if (value.parts !== undefined) collectText(value.parts, out); // Gemini systemInstruction + } +} + +/** + * Compute a stable 24-hex hash of the request's system prompt across the common body shapes + * (OpenAI `messages[].role==="system"`, Claude `system`, Gemini `systemInstruction`). Returns + * `null` when there is no system prompt to freeze. + */ +export function extractStablePrefixHash(body: unknown): string | null { + if (!isRecord(body)) return null; + const parts: string[] = []; + collectText(body.system, parts); + collectText(body.systemInstruction, parts); + collectText(body.system_instruction, parts); + const messages = body.messages; + if (Array.isArray(messages)) { + for (const msg of messages) { + if (isRecord(msg) && msg.role === "system") collectText(msg.content, parts); + } + } + if (parts.length === 0) return null; + return crypto.createHash("sha256").update(parts.join("\n")).digest("hex").slice(0, 24); +} + +// ─── observation + query ────────────────────────────────────────────────────────── + +/** Record one observation of a system-prompt hash (call once per request when enabled). */ +export function observePrefix(hash: string): void { + boundedInc(hash); +} + +/** True once a prefix has been observed `>= threshold` times (a stable, freezable prefix). */ +export function isPrefixFrozen(hash: string, threshold: number): boolean { + return (observations.get(hash) ?? 0) >= threshold; +} + +/** Current observation count for a prefix hash (telemetry/tests). */ +export function getPrefixObservations(hash: string): number { + return observations.get(hash) ?? 0; +} + +/** Clear all observations (tests + operator reset). */ +export function resetPrefixFreeze(): void { + observations.clear(); +} diff --git a/tests/unit/compression/prefix-freeze.test.ts b/tests/unit/compression/prefix-freeze.test.ts new file mode 100644 index 0000000000..7848965845 --- /dev/null +++ b/tests/unit/compression/prefix-freeze.test.ts @@ -0,0 +1,121 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { + resolvePrefixFreezeConfig, + extractStablePrefixHash, + observePrefix, + isPrefixFrozen, + getPrefixObservations, + resetPrefixFreeze, + DEFAULT_PREFIX_FREEZE, +} from "../../../open-sse/services/compression/prefixFreeze.ts"; +import { resolveCacheAwareConfig } from "../../../open-sse/services/compression/cacheAwareConfig.ts"; +import type { CompressionConfig } from "../../../open-sse/services/compression/types.ts"; + +// T08/H5 — usage-observed prefix freeze. Pure observer + integration via resolveCacheAwareConfig. + +beforeEach(() => resetPrefixFreeze()); + +describe("prefixFreeze — config", () => { + it("defaults to disabled / threshold 3", () => { + const cfg = resolvePrefixFreezeConfig({} as NodeJS.ProcessEnv); + assert.equal(cfg.enabled, false); + assert.equal(cfg.threshold, DEFAULT_PREFIX_FREEZE.threshold); + }); + it("reads env", () => { + const cfg = resolvePrefixFreezeConfig({ + COMPRESSION_PREFIX_FREEZE_ENABLED: "true", + COMPRESSION_PREFIX_FREEZE_THRESHOLD: "5", + } as NodeJS.ProcessEnv); + assert.equal(cfg.enabled, true); + assert.equal(cfg.threshold, 5); + }); +}); + +describe("prefixFreeze — extractStablePrefixHash", () => { + it("hashes an OpenAI system message", () => { + const h = extractStablePrefixHash({ + messages: [ + { role: "system", content: "You are helpful" }, + { role: "user", content: "hi" }, + ], + }); + assert.equal(typeof h, "string"); + assert.equal(h!.length, 24); + }); + it("hashes a Claude `system` field and a Gemini `systemInstruction`", () => { + const claude = extractStablePrefixHash({ system: "SP", messages: [] }); + const gemini = extractStablePrefixHash({ systemInstruction: { parts: [{ text: "SP" }] } }); + assert.ok(claude && gemini); + // same underlying text → same hash regardless of shape + assert.equal(claude, gemini); + }); + it("returns null when there is no system prompt", () => { + assert.equal(extractStablePrefixHash({ messages: [{ role: "user", content: "hi" }] }), null); + assert.equal(extractStablePrefixHash("not an object"), null); + }); + it("different system prompts hash differently", () => { + const a = extractStablePrefixHash({ system: "A" }); + const b = extractStablePrefixHash({ system: "B" }); + assert.notEqual(a, b); + }); +}); + +describe("prefixFreeze — observe/isFrozen", () => { + it("freezes only at/above the threshold, isolated per hash", () => { + observePrefix("h"); + observePrefix("h"); + assert.equal(isPrefixFrozen("h", 3), false); + assert.equal(getPrefixObservations("h"), 2); + observePrefix("h"); + assert.equal(isPrefixFrozen("h", 3), true); + // a different prefix is unaffected + assert.equal(isPrefixFrozen("other", 3), false); + }); +}); + +describe("resolveCacheAwareConfig — H5 integration", () => { + const baseConfig = { + defaultMode: "standard", + preserveSystemPromptMode: "whenNoCache", + preserveSystemPrompt: false, + } as unknown as CompressionConfig; + const body = { + messages: [ + { role: "system", content: "You are a careful assistant." }, + { role: "user", content: "hi" }, + ], + }; + const ctx = { provider: "test-noncaching-provider" }; + + afterEach(() => { + delete process.env.COMPRESSION_PREFIX_FREEZE_ENABLED; + delete process.env.COMPRESSION_PREFIX_FREEZE_THRESHOLD; + }); + + it("does nothing when disabled (default): non-caching provider never freezes the prefix", () => { + for (let i = 0; i < 5; i++) { + const out = resolveCacheAwareConfig(baseConfig, body, ctx); + assert.equal(out.preserveSystemPrompt, false); + } + }); + + it("when enabled, an observed-stable prefix flips preserveSystemPrompt at the threshold", () => { + process.env.COMPRESSION_PREFIX_FREEZE_ENABLED = "true"; + process.env.COMPRESSION_PREFIX_FREEZE_THRESHOLD = "3"; + assert.equal(resolveCacheAwareConfig(baseConfig, body, ctx).preserveSystemPrompt, false); // 1 + assert.equal(resolveCacheAwareConfig(baseConfig, body, ctx).preserveSystemPrompt, false); // 2 + assert.equal(resolveCacheAwareConfig(baseConfig, body, ctx).preserveSystemPrompt, true); // 3 → frozen + }); + + it("respects mode `never`: a frozen prefix is still not preserved", () => { + process.env.COMPRESSION_PREFIX_FREEZE_ENABLED = "true"; + process.env.COMPRESSION_PREFIX_FREEZE_THRESHOLD = "1"; + const neverConfig = { + ...baseConfig, + preserveSystemPromptMode: "never", + } as unknown as CompressionConfig; + const out = resolveCacheAwareConfig(neverConfig, body, ctx); + assert.equal(out.preserveSystemPrompt, false); + }); +});