diff --git a/CHANGELOG.md b/CHANGELOG.md index b78eabea0e..da167576cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ _In development — bullets added per PR; finalized at release._ ### ✨ New Features +- **feat(compression): result memoization for deterministic engines (opt-in)** — caches `(input, config) → result` for provably pure, stateless modes (`lite`/`standard`/`rtk` and stacked pipelines of `{lite,caveman,rtk}`) to skip recompute on the hot path. Opt-in via `memoizeCompressionResults` (default off → zero behavior change). Conservative opt-in whitelist (stateful `ccr`/`session-dedup` — which write the cross-request CCR store — and model-backed `ultra`/`aggressive`/`llmlingua` are never cached), principal-scoped (skipped without a principal, so no cross-principal body leak), and clone-on-store + clone-on-read. Tier-3 item of the compression feature-extraction roadmap (#21). - **feat(compression): inline transparency annotation** — surfaces `tokens=847→312; rules: filler×8, dedup×2` derived from existing compression stats. The `X-OmniRoute-Compression` response header is extended **append-only** (the `mode; source=X` prefix stays byte-identical, so existing header parsers don't break) and the compression studio cockpit shows a matching badge. Zero new computation — it aggregates the `rulesApplied`/`techniquesUsed` already on the stats. Tier-3 item of the compression feature-extraction roadmap (#18). - **feat(compression): saliency heatmap in the compression studio** — the preview studio can now color each token by saliency: `ultra` per-token `scoreToken` (0–1, green→red gradient) or universal kept/removed from the existing diff. A dry-run visualization behind a toggle (no cost on a normal preview; backward-compatible when off). Completes the visualization half of roadmap item #13 (the A/B comparison shipped in [#5080](https://github.com/diegosouzapw/OmniRoute/pull/5080)). - **feat(compression): composite-command splitter for RTK detection** — `cd /x && git status` now detects as `git-status` (previously the whole string was treated as one command and matched no filter). A quote-aware top-level tokenizer splits on `&&`/`||`/`;` (never inside quotes or `$(…)`/backtick subshells) and feeds the **last** segment to RTK command detection, so every RTK filter/renderer fires on commands wrapped in `cd … &&`/`||`/`;` chains. O(n), no RegExp over the command (ReDoS-safe). Tier-3 item of the compression feature-extraction roadmap (#16). diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 4eeb1d88e4..6c92db6841 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -162,7 +162,8 @@ "open-sse/services/combo.ts": 3368, "_rebaseline_2026_06_26_fidelity_gate_extraction": "Milestone-B fidelity-gate wiring residual: bodyToText+gateAdvance extracted to fidelityGateStep.ts (889->854, -35), but the StackOptions.fidelityGate field, the `const fidelityGate` reads at the two stacked-loop dispatch chokepoints, and the import of FidelityGateConfig are irreducible wiring that cannot leave strategySelector without an architectural refactor of the pre-existing stacked pipeline. Net: 889->854 (+6 vs the pre-Milestone-B frozen 848). Covered by tests/unit/compression/*.test.ts (940 pass).", "_rebaseline_2026_06_28_5243_risk_gate_prepass": "PR #5243 (compression risk-gate pre-pass) own growth: open-sse/services/compression/strategySelector.ts 854->899 (+45). The three exported entry points (applyCompression/applyStackedCompression/applyStackedCompressionAsync) become thin wrappers over pure-extracted private bodies (runCompression/runStackedCompression/runStackedCompressionAsync) so the risk-gate mask->run->restore wrapper sits strictly OUTSIDE the per-step loop — a single universal integration point. The wrapper logic itself (resolveRiskGate/withRiskGate) lives in the new riskGate/strategyWrap.ts (960 (+61 = the opt-in result-memoization branches in applyCompression/applyCompressionAsync — principal+determinism gate, makeMemoKey lookup/store with model+supportsVision folded into the key, recompute-with-memo-off). Default off (memoizeCompressionResults), so zero behavior change. The memo helpers live in the leaf resultMemo.ts ((); + +// Opt-IN whitelist (NOT opt-out): cache only engines proven pure + STATELESS across +// requests. Excluded on purpose: `ccr` and `session-dedup` write to the cross-request +// CCR store (`ccr/index.ts` ccrStore; session-dedup imports storeBlock), so their output +// depends on prior state → not safe to memoize; `ultra`/`aggressive`/`llmlingua` are +// model-backed/non-deterministic. Any NEW engine is excluded until explicitly vetted. +const DETERMINISTIC_ENGINES = new Set(["lite", "caveman", "rtk"]); + +/** Top-level modes safe to cache (whitelist — any unknown/new mode defaults to false). */ +const DETERMINISTIC_MODES = new Set(["lite", "standard", "rtk"]); + +export function isDeterministicMode(mode: CompressionMode, config?: CompressionConfig): boolean { + if (mode === "stacked") { + const pipeline = config?.stackedPipeline; + if (!pipeline || pipeline.length === 0) return false; + return pipeline.every((step) => DETERMINISTIC_ENGINES.has(step.engine)); + } + return DETERMINISTIC_MODES.has(mode); +} + +function sha256hex(text: string): string { + return crypto.createHash("sha256").update(text).digest("hex"); +} + +export function makeMemoKey( + body: Record, + mode: CompressionMode, + config: CompressionConfig, + principalId?: string, + model?: string, + supportsVision?: boolean | null +): string { + const bodyHash = sha256hex(JSON.stringify(body)); + // model + supportsVision MUST be part of the key: the `lite` engine strips data:image + // URLs only when vision is unsupported (replaceImageUrls / modelSupportsVision), so the + // same (body, config) yields a DIFFERENT result per target — omitting them returns a + // wrong (image-stripped or image-kept) cached body across vision/non-vision targets. + return sha256hex( + JSON.stringify({ + bodyHash, + mode, + config, + principalId: principalId ?? null, + model: model ?? null, + supportsVision: supportsVision ?? null, + }) + ); +} + +function boundedSet(key: string, value: CompressionResult): void { + if (!memoMap.has(key) && memoMap.size >= MEMO_CAP) { + const firstKey = memoMap.keys().next().value; + if (firstKey !== undefined) { + memoMap.delete(firstKey); + } + } + memoMap.set(key, value); +} + +export function memoLookup(key: string): CompressionResult | null { + const hit = memoMap.get(key); + if (!hit) return null; + // Return a clone so downstream mutation cannot corrupt the cached value. + return JSON.parse(JSON.stringify(hit)) as CompressionResult; +} + +export function memoStore(key: string, result: CompressionResult): void { + // Clone on STORE too (memoLookup already clones on read). Storing the caller's live + // object would let a later mutation of it (e.g. an async engine holding a sub-ref) + // corrupt the cached entry. Both ends isolated ⇒ the cache is immutable once stored. + boundedSet(key, JSON.parse(JSON.stringify(result)) as CompressionResult); +} + +/** For tests only — clears the in-process memo store. */ +export function clearMemoStore(): void { + memoMap.clear(); +} diff --git a/open-sse/services/compression/strategySelector.ts b/open-sse/services/compression/strategySelector.ts index 7b60243a36..ced40af5e3 100644 --- a/open-sse/services/compression/strategySelector.ts +++ b/open-sse/services/compression/strategySelector.ts @@ -40,10 +40,16 @@ import { withCompressionEntrypointGuards, withCompressionEntrypointGuardsAsync, } from "./entrypointWrap.ts"; +import { makeMemoKey, memoLookup, memoStore, isDeterministicMode } from "./resultMemo.ts"; export { resolveCacheAwareConfig } from "./cacheAwareConfig.ts"; // Re-export so existing importers (resolver test + chatCore dynamic import) keep resolving. -export { planFromHeader, formatCompressionMeta, formatCompressionAnnotation, buildNamedComboLookup }; +export { + planFromHeader, + formatCompressionMeta, + formatCompressionAnnotation, + buildNamedComboLookup, +}; /** Named-combo map: combo id → its stacked pipeline (operator-defined profiles). */ type NamedCombos = Record; @@ -267,6 +273,32 @@ function runCompression( if (mode === "off") { return { body, compressed: false, stats: null }; } + if ( + options?.config?.memoizeCompressionResults === true && + // Only memoize for an explicit principal — a missing principalId would collapse + // authenticated callers into the shared anonymous (null) key space and let one + // principal receive another's cached body. No principal ⇒ skip the cache. + typeof options?.principalId === "string" && + options.principalId.length > 0 && + isDeterministicMode(mode, options.config) + ) { + const key = makeMemoKey( + body, + mode, + options.config, + options.principalId, + options.model, + options.supportsVision + ); + const hit = memoLookup(key); + if (hit) return hit; + const result = runCompression({ ...body }, mode, { + ...options, + config: { ...options.config, memoizeCompressionResults: false }, + }); + memoStore(key, result); + return memoLookup(key)!; + } if (mode === "rtk") { return applyRtkCompression(body, { // Selecting the "rtk" mode IS the enable signal — run it even if the per-engine @@ -416,6 +448,32 @@ async function runCompressionAsync( cachingContext?: CachingDetectionContext; } ): Promise { + if ( + options?.config?.memoizeCompressionResults === true && + // Only memoize for an explicit principal — a missing principalId would collapse + // authenticated callers into the shared anonymous (null) key space and let one + // principal receive another's cached body. No principal ⇒ skip the cache. + typeof options?.principalId === "string" && + options.principalId.length > 0 && + isDeterministicMode(mode, options.config) + ) { + const key = makeMemoKey( + body, + mode, + options.config, + options.principalId, + options.model, + options.supportsVision + ); + const hit = memoLookup(key); + if (hit) return hit; + const result = await runCompressionAsync({ ...body }, mode, { + ...options, + config: { ...options.config, memoizeCompressionResults: false }, + }); + memoStore(key, result); + return memoLookup(key)!; + } if (mode === "stacked") { const adapter = adaptBodyForCompression(body); const result = await applyStackedCompressionAsync( @@ -783,7 +841,10 @@ function runStackedCompression( continue; } mergeStackStep(acc, step.engine, result); - if (decideStep(result, bailout).advance && gateAdvance(result, currentBody, fidelityGate, acc, step.engine)) { + if ( + decideStep(result, bailout).advance && + gateAdvance(result, currentBody, fidelityGate, acc, step.engine) + ) { currentBody = result.body; compressed = true; } @@ -867,7 +928,10 @@ async function runStackedCompressionAsync( continue; } mergeStackStep(acc, step.engine, result); - if (decideStep(result, bailout).advance && gateAdvance(result, currentBody, fidelityGate, acc, step.engine)) { + if ( + decideStep(result, bailout).advance && + gateAdvance(result, currentBody, fidelityGate, acc, step.engine) + ) { currentBody = result.body; compressed = true; } diff --git a/open-sse/services/compression/types.ts b/open-sse/services/compression/types.ts index 01c33e35ae..fe8d325871 100644 --- a/open-sse/services/compression/types.ts +++ b/open-sse/services/compression/types.ts @@ -195,6 +195,8 @@ export interface CompressionConfig { * swallowed; the lazy first-call path still applies. Default false. */ ultraSlmPrewarm?: boolean; + /** Opt-in result memoization for deterministic engines only (default off). */ + memoizeCompressionResults?: boolean; } export interface CompressionStats { diff --git a/tests/unit/compression/result-memo.test.ts b/tests/unit/compression/result-memo.test.ts new file mode 100644 index 0000000000..d24acc0df9 --- /dev/null +++ b/tests/unit/compression/result-memo.test.ts @@ -0,0 +1,343 @@ +import { describe, it, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { + memoLookup, + memoStore, + makeMemoKey, + isDeterministicMode, + clearMemoStore, + MEMO_CAP, +} from "../../../open-sse/services/compression/resultMemo.ts"; +import type { CompressionResult } from "../../../open-sse/services/compression/types.ts"; +import { DEFAULT_COMPRESSION_CONFIG } from "../../../open-sse/services/compression/types.ts"; +import { applyCompression } from "../../../open-sse/services/compression/strategySelector.ts"; + +const baseBody = { + messages: [{ role: "user", content: "hello world compress me please" }], + model: "gpt-4", +}; + +const memoConfig = { + ...DEFAULT_COMPRESSION_CONFIG, + enabled: true, + defaultMode: "lite" as const, + memoizeCompressionResults: true, +}; + +const noMemoConfig = { + ...DEFAULT_COMPRESSION_CONFIG, + enabled: true, + defaultMode: "lite" as const, + memoizeCompressionResults: false, +}; + +describe("resultMemo unit", () => { + beforeEach(() => { + clearMemoStore(); + }); + + it("memoLookup returns null on miss", () => { + const key = makeMemoKey(baseBody, "lite", noMemoConfig, "p1"); + assert.equal(memoLookup(key), null); + }); + + it("memoStore then memoLookup returns the stored result", () => { + const key = makeMemoKey(baseBody, "lite", memoConfig, "p1"); + const result: CompressionResult = { + body: { ...baseBody, _compressed: true }, + compressed: true, + stats: { + originalTokens: 10, + compressedTokens: 8, + savingsPercent: 20, + techniquesUsed: ["lite"], + mode: "lite", + timestamp: Date.now(), + }, + }; + memoStore(key, result); + const hit = memoLookup(key); + assert.notEqual(hit, null); + assert.equal(hit!.compressed, true); + }); + + it("different principalId produces a different key (MISS)", () => { + const key1 = makeMemoKey(baseBody, "lite", memoConfig, "principal-A"); + const key2 = makeMemoKey(baseBody, "lite", memoConfig, "principal-B"); + assert.notEqual(key1, key2); + + const result: CompressionResult = { + body: { ...baseBody }, + compressed: false, + stats: null, + }; + memoStore(key1, result); + assert.equal(memoLookup(key2), null); + }); + + it("cached body is a COPY — mutating returned body does not corrupt next hit", () => { + const key = makeMemoKey(baseBody, "lite", memoConfig, "p1"); + const result: CompressionResult = { + body: { messages: [{ role: "user", content: "original" }] }, + compressed: true, + stats: null, + }; + memoStore(key, result); + + const hit1 = memoLookup(key); + assert.notEqual(hit1, null); + // Mutate the returned body + (hit1!.body as Record)["injected"] = "evil"; + + // Second lookup should not see the mutation + const hit2 = memoLookup(key); + assert.notEqual(hit2, null); + assert.equal((hit2!.body as Record)["injected"], undefined); + }); + + it("FIFO eviction: after MEMO_CAP entries, oldest is evicted", () => { + const firstKey = makeMemoKey({ x: 0 }, "lite", memoConfig, "evict-test"); + const stub: CompressionResult = { body: {}, compressed: false, stats: null }; + memoStore(firstKey, stub); + + // Fill up to and beyond cap + for (let i = 1; i <= MEMO_CAP; i++) { + const k = makeMemoKey({ x: i }, "lite", memoConfig, "evict-test"); + memoStore(k, stub); + } + + // The first entry should have been evicted + assert.equal(memoLookup(firstKey), null); + }); +}); + +describe("isDeterministicMode", () => { + it("lite is deterministic", () => { + assert.equal(isDeterministicMode("lite", DEFAULT_COMPRESSION_CONFIG), true); + }); + + it("standard is deterministic", () => { + assert.equal(isDeterministicMode("standard", DEFAULT_COMPRESSION_CONFIG), true); + }); + + it("rtk is deterministic", () => { + assert.equal(isDeterministicMode("rtk", DEFAULT_COMPRESSION_CONFIG), true); + }); + + it("off is NOT deterministic (nothing to cache)", () => { + assert.equal(isDeterministicMode("off", DEFAULT_COMPRESSION_CONFIG), false); + }); + + it("aggressive is NOT deterministic (pluggable summarizer)", () => { + assert.equal(isDeterministicMode("aggressive", DEFAULT_COMPRESSION_CONFIG), false); + }); + + it("ultra is NOT deterministic (SLM tier)", () => { + assert.equal(isDeterministicMode("ultra", DEFAULT_COMPRESSION_CONFIG), false); + }); + + it("stacked with only deterministic engines IS deterministic", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [ + { engine: "rtk" as const, intensity: "standard" as const }, + { engine: "caveman" as const, intensity: "full" as const }, + ], + }; + assert.equal(isDeterministicMode("stacked", cfg), true); + }); + + it("stacked with ultra engine is NOT deterministic", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "ultra" as const }, { engine: "caveman" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with aggressive engine is NOT deterministic", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "aggressive" as const }, { engine: "rtk" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with llmlingua engine is NOT deterministic", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "llmlingua" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + // Stateful engines write to the cross-request CCR store (storeBlock): caching their + // output would skip the side-effect on a HIT, leaving CCR markers pointing at blocks + // that were never stored → broken `retrieve`. These MUST stay excluded from the memo. + it("stacked with ccr engine is NOT deterministic (writes cross-request CCR store)", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "ccr" as const }, { engine: "caveman" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with session-dedup engine is NOT deterministic (storeBlock side-effect)", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "session-dedup" as const }, { engine: "rtk" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with ionizer engine is NOT deterministic (storeBlock side-effect)", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "ionizer" as const }, { engine: "lite" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with headroom engine is NOT deterministic (excluded until vetted)", () => { + const cfg = { + ...DEFAULT_COMPRESSION_CONFIG, + stackedPipeline: [{ engine: "headroom" as const }, { engine: "lite" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); + + it("stacked with empty/undefined pipeline is NOT deterministic (safe default)", () => { + const cfg = { ...DEFAULT_COMPRESSION_CONFIG, stackedPipeline: undefined }; + assert.equal(isDeterministicMode("stacked", cfg), false); + }); +}); + +describe("applyCompression with memoization", () => { + beforeEach(() => { + clearMemoStore(); + }); + + it("flag OFF: two identical calls both compute (no caching path)", () => { + let callCount = 0; + // We can't easily spy on internal engine, so we verify via deterministic output + // equality between independent calls (proving cache isn't interfering). + // Use a body that will be lightly compressed. + const body = { + messages: [ + { role: "user", content: "The quick brown fox jumps over the lazy dog. ".repeat(20) }, + ], + model: "gpt-4", + }; + + const r1 = applyCompression(body, "lite", { config: noMemoConfig, principalId: "u1" }); + const r2 = applyCompression(body, "lite", { config: noMemoConfig, principalId: "u1" }); + + // Both compute — results should be equal (deterministic) but not the same object reference + assert.deepEqual(r1.stats?.mode, r2.stats?.mode); + // The memo store should be empty (flag was OFF) + const key = makeMemoKey(body, "lite", noMemoConfig, "u1"); + assert.equal(memoLookup(key), null); + }); + + it("flag ON + deterministic mode: 2nd call hits memo (stats identical)", () => { + const body = { + messages: [{ role: "user", content: "Memoization test content. ".repeat(15) }], + model: "gpt-4", + }; + + const r1 = applyCompression(body, "lite", { config: memoConfig, principalId: "u1" }); + const r2 = applyCompression(body, "lite", { config: memoConfig, principalId: "u1" }); + + // Both should have same stats (2nd hit from cache) + assert.deepEqual(r1.stats?.savingsPercent, r2.stats?.savingsPercent); + assert.deepEqual(r1.stats?.originalTokens, r2.stats?.originalTokens); + + // Verify cache was populated + const key = makeMemoKey(body, "lite", memoConfig, "u1"); + assert.notEqual(memoLookup(key), null); + }); + + it("flag ON + deterministic mode: different principalId = MISS", () => { + const body = { + messages: [{ role: "user", content: "Cross-principal test content. ".repeat(10) }], + model: "gpt-4", + }; + + applyCompression(body, "lite", { config: memoConfig, principalId: "principal-A" }); + // principal-B should NOT hit the cache for principal-A's result + const keyB = makeMemoKey(body, "lite", memoConfig, "principal-B"); + assert.equal(memoLookup(keyB), null); + }); + + it("flag ON + ultra mode: NOT cached", () => { + const body = { + messages: [{ role: "user", content: "Ultra mode should not be cached. ".repeat(10) }], + model: "gpt-4", + }; + + applyCompression(body, "ultra", { config: memoConfig, principalId: "u1" }); + const key = makeMemoKey(body, "ultra", memoConfig, "u1"); + assert.equal(memoLookup(key), null); + }); +}); + +// ── Core-review hardening regressions ────────────────────────────────────── +describe("resultMemo — core review hardening", () => { + beforeEach(() => clearMemoStore()); + + it("stacked pipeline with a stateful engine (ccr/session-dedup) is NOT deterministic", () => { + // ccr + session-dedup write to the cross-request CCR store → output depends on prior + // state → must never be cached, even though they are not model-backed. + const cfgCcr = { + ...memoConfig, + stackedPipeline: [{ engine: "rtk" as const }, { engine: "ccr" as const }], + }; + const cfgDedup = { ...memoConfig, stackedPipeline: [{ engine: "session-dedup" as const }] }; + assert.equal(isDeterministicMode("stacked", cfgCcr), false); + assert.equal(isDeterministicMode("stacked", cfgDedup), false); + // the pure default pipeline [rtk, caveman] stays cacheable + const cfgPure = { + ...memoConfig, + stackedPipeline: [{ engine: "rtk" as const }, { engine: "caveman" as const }], + }; + assert.equal(isDeterministicMode("stacked", cfgPure), true); + // an unknown/new mode is NOT cached by default (opt-in whitelist) + assert.equal(isDeterministicMode("totally-new-mode" as never, memoConfig), false); + }); + + it("a missing principalId is never memoized (no anonymous↔authenticated key collision)", () => { + const body = { + messages: [{ role: "user", content: "no principal here please" }], + model: "gpt-4", + }; + applyCompression(body, "lite", { config: memoConfig }); // no principalId + const key = makeMemoKey(body, "lite", memoConfig, undefined); + assert.equal(memoLookup(key), null); + }); + + it("mutating the result after store does not corrupt the cache (clone-on-store)", () => { + const key = "mutate-after-store"; + const result: CompressionResult = { + body: { messages: [{ role: "user", content: "original" }] }, + compressed: true, + stats: null, + }; + memoStore(key, result); + // mutate the caller's object AFTER storing + (result.body.messages as Array<{ content: string }>)[0].content = "TAMPERED"; + const got = memoLookup(key); + assert.equal((got!.body.messages as Array<{ content: string }>)[0].content, "original"); + }); + + it("key folds in model + supportsVision (lite image-strip depends on vision capability)", () => { + // Regression: lite strips data:image URLs only when vision is unsupported, so the same + // (body, config, principal) yields a DIFFERENT result per target. The key MUST include + // model + supportsVision, else a non-vision target's image-stripped body is served to a + // vision-capable target (and vice-versa). + const k = (model?: string, vision?: boolean | null) => + makeMemoKey(baseBody, "lite", memoConfig, "p1", model, vision); + assert.notEqual(k("gpt-4", false), k("gpt-4", true), "supportsVision must change the key"); + assert.notEqual(k("gpt-4", true), k("gemini-2", true), "model must change the key"); + assert.equal(k("gpt-4", true), k("gpt-4", true), "same inputs => same key (deterministic)"); + }); +});