From 8b73b37d094cb641fac5acf6605ffaa01ad12ba2 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Thu, 18 Jun 2026 23:10:12 -0300 Subject: [PATCH] chore(compression): remove vestigial reconstructCcr/SessionDedup round-trip helpers (#4226) Integrated into release/v3.8.29 (remove vestigial reconstructCcr/reconstructSessionDedup round-trip helpers). --- .../services/compression/engines/ccr/index.ts | 56 --------- .../compression/engines/headroom/index.ts | 5 +- .../engines/session-dedup/index.ts | 111 ++---------------- .../compression/ccr-marker-retrieve.test.ts | 33 ------ tests/unit/compression/session-dedup.test.ts | 25 +--- 5 files changed, 18 insertions(+), 212 deletions(-) diff --git a/open-sse/services/compression/engines/ccr/index.ts b/open-sse/services/compression/engines/ccr/index.ts index 704cac9f03..f67e39e594 100644 --- a/open-sse/services/compression/engines/ccr/index.ts +++ b/open-sse/services/compression/engines/ccr/index.ts @@ -53,8 +53,6 @@ const ENGINE_ID = "ccr"; const DEFAULT_MIN_CHARS = 600; /** Number of retrievals before a block is flagged "do-not-compress" for that principal. */ const RETRIEVAL_THRESHOLD = 3; -/** Regex to match CCR markers for reconstruction. */ -const MARKER_RE = /\[CCR retrieve hash=([0-9a-f]{24}) chars=\d+\]/g; /** * Maximum number of entries in each bounded store. * When inserting beyond this cap, the oldest entry (Map insertion order) is evicted. @@ -306,60 +304,6 @@ function validateCcrConfig(config: Record): EngineValidationRes return { valid: errors.length === 0, errors }; } -// ─── reconstruction helper ──────────────────────────────────────────────────── - -/** - * Reconstruct a body by replacing all `[CCR retrieve hash=<24hex> chars=N]` - * markers with the stored verbatim blocks for the given principal. - * - * Returns a new body object with markers restored to their original text. - * Markers that do not belong to the given principal remain unchanged. - */ -export function reconstructCcr( - body: Record, - principalId?: string -): Record { - const messages = body["messages"]; - if (!Array.isArray(messages)) return body; - - type MsgLike = { - role?: string; - content?: string | Array>; - [key: string]: unknown; - }; - - const restored = (messages as MsgLike[]).map((msg) => { - const content = msg["content"]; - - if (typeof content === "string") { - const reconstructed = content.replace(MARKER_RE, (_m, hash: string) => { - return retrieveBlock(hash, principalId) ?? _m; - }); - return reconstructed !== content ? { ...msg, content: reconstructed } : { ...msg }; - } - - if (Array.isArray(content)) { - let changed = false; - const newContent = content.map((part) => { - if (part["type"] !== "text" || typeof part["text"] !== "string") return part; - const reconstructed = (part["text"] as string).replace(MARKER_RE, (_m, hash: string) => { - return retrieveBlock(hash, principalId) ?? _m; - }); - if (reconstructed !== part["text"]) { - changed = true; - return { ...part, text: reconstructed }; - } - return part; - }); - return changed ? { ...msg, content: newContent } : { ...msg }; - } - - return { ...msg }; - }); - - return { ...body, messages: restored }; -} - // ─── engine export ──────────────────────────────────────────────────────────── export const ccrEngine: CompressionEngine = { diff --git a/open-sse/services/compression/engines/headroom/index.ts b/open-sse/services/compression/engines/headroom/index.ts index 37c42699c9..43f6638f2e 100644 --- a/open-sse/services/compression/engines/headroom/index.ts +++ b/open-sse/services/compression/engines/headroom/index.ts @@ -40,7 +40,6 @@ import { GCF_FENCE_OPEN, GCF_FENCE_CLOSE, decodeTabular, - TABULAR_MARKER_RE, } from "./tabular.ts"; export { encodeTabular, decodeTabular } from "./tabular.ts"; @@ -183,6 +182,10 @@ type MessageLike = { * Reverse the headroom compaction: find every ```gcf-generic or ```omni-tabular * block in message contents and decode it back to the original JSON string. * + * No production caller by design — the compact form is sent to the provider as-is. This is + * exported as the round-trip ORACLE that proves the GCF/tabular encoder is lossless: the + * losslessness regression tests encode via apply() then decode here and assert deep-equal. + * * Returns a new body with all compacted blocks expanded. */ export function reconstructHeadroom(body: Record): Record { diff --git a/open-sse/services/compression/engines/session-dedup/index.ts b/open-sse/services/compression/engines/session-dedup/index.ts index 668fe9dfb5..2ba7fe9f2b 100644 --- a/open-sse/services/compression/engines/session-dedup/index.ts +++ b/open-sse/services/compression/engines/session-dedup/index.ts @@ -45,16 +45,12 @@ const ENGINE_ID = "session-dedup"; const DEFAULT_MIN_BLOCK_CHARS = 80; /** Minimum number of lines a block must span to be a dedup candidate. */ const MIN_BLOCK_LINES = 3; -/** Marker pattern for reconstruction. */ -const MARKER_RE = /\[dedup:ref sha=([0-9a-f]{24})\]/g; -/** Key used to store the reverse map in the body object. */ -const DEDUP_MAP_KEY = "__sessionDedupMap__"; // ─── hash helper (SHA-256 prefix, collision-resistant) ─────────────────────── function hashBlock(text: string): string { - // 24 hex / 96 bits — collision-resistant (a 32-bit djb2 could collide and make - // reconstruction restore the WRONG block). Pass 2 additionally verifies block + // 24 hex / 96 bits — collision-resistant (a 32-bit djb2 could collide and make a + // dedup marker reference the WRONG block). Pass 2 additionally verifies block // equality before substituting, so a collision can never cause corruption. return crypto.createHash("sha256").update(text).digest("hex").slice(0, 24); } @@ -100,7 +96,6 @@ function dedupMessageTexts( minBlockChars: number ): { deduped: Map; - reverseMap: Map; dedupCount: number; } { // Pass 1: for each message, extract suffix blocks and record first ownership. @@ -118,12 +113,6 @@ function dedupMessageTexts( } } - // Build reverse map (sha → block) for all first-seen blocks. - const reverseMap = new Map(); - for (const [sha, { block }] of firstSeen) { - reverseMap.set(sha, block); - } - // Pass 2: for each message, find blocks that were FIRST seen in an earlier message. const deduped = new Map(); let dedupCount = 0; @@ -138,7 +127,7 @@ function dedupMessageTexts( const sha = hashBlock(block); const owner = firstSeen.get(sha); // owner.block === block guards against a (now astronomically unlikely) hash - // collision substituting a marker that would reconstruct to the wrong text. + // collision substituting a marker that would reference the wrong block. if (owner && owner.ownerMsgIdx < msgIdx && owner.block === block) { dupBlocks.push({ block, sha }); } @@ -174,7 +163,7 @@ function dedupMessageTexts( } } - return { deduped, reverseMap, dedupCount }; + return { deduped, dedupCount }; } // ─── message array processing ───────────────────────────────────────────────── @@ -191,7 +180,7 @@ type MessageLike = { function processMessages( messages: MessageLike[], minBlockChars: number -): { messages: MessageLike[]; reverseMap: Map; dedupCount: number } { +): { messages: MessageLike[]; dedupCount: number } { // Collect (msgIdx, text) for non-system string-content messages. // For multipart, index each text part separately. const msgTexts: Array<{ msgIdx: number; text: string }> = []; @@ -213,13 +202,13 @@ function processMessages( } if (msgTexts.length < 2) { - return { messages, reverseMap: new Map(), dedupCount: 0 }; + return { messages, dedupCount: 0 }; } - const { deduped, reverseMap, dedupCount } = dedupMessageTexts(msgTexts, minBlockChars); + const { deduped, dedupCount } = dedupMessageTexts(msgTexts, minBlockChars); if (dedupCount === 0) { - return { messages, reverseMap, dedupCount: 0 }; + return { messages, dedupCount: 0 }; } const result = messages.map((msg, i) => { @@ -248,7 +237,7 @@ function processMessages( return { ...msg }; }); - return { messages: result, reverseMap, dedupCount }; + return { messages: result, dedupCount }; } // ─── schema & validation ────────────────────────────────────────────────────── @@ -328,11 +317,10 @@ export const sessionDedupEngine: CompressionEngine = { } const start = performance.now(); - const { - messages: dedupedMessages, - reverseMap, - dedupCount, - } = processMessages(messages as MessageLike[], minBlockChars); + const { messages: dedupedMessages, dedupCount } = processMessages( + messages as MessageLike[], + minBlockChars + ); if (dedupCount === 0) { return { body, compressed: false, stats: null }; @@ -343,16 +331,6 @@ export const sessionDedupEngine: CompressionEngine = { messages: dedupedMessages, }; - // Store the reverse map as a NON-ENUMERABLE property so JSON.stringify does not - // include it in the serialized output (which goes to the upstream provider). - // reconstructSessionDedup reads it back via getOwnPropertyDescriptor. - Object.defineProperty(newBody, DEDUP_MAP_KEY, { - value: Object.fromEntries(reverseMap), - enumerable: false, - configurable: true, - writable: false, - }); - const durationMs = Math.round(performance.now() - start); const stats = createCompressionStats( body, @@ -378,66 +356,3 @@ export const sessionDedupEngine: CompressionEngine = { return validateSessionDedupConfig(config); }, }; - -// ─── reconstruction helper ──────────────────────────────────────────────────── - -/** - * Reverse the dedup: replace every `[dedup:ref sha=XXXXXXXX]` marker with the - * original block text stored in the reverse map attached to the body by - * `sessionDedupEngine.apply`. - * - * Returns a new body object without the internal `__sessionDedupMap__` key. - */ -export function reconstructSessionDedup(body: Record): Record { - // The reverse map is stored as a non-enumerable property so it doesn't appear - // in JSON.stringify. Access it via getOwnPropertyDescriptor. - const mapDescriptor = Object.getOwnPropertyDescriptor(body, DEDUP_MAP_KEY); - const rawMap = mapDescriptor?.value ?? body[DEDUP_MAP_KEY]; - if (!rawMap || typeof rawMap !== "object" || Array.isArray(rawMap)) return body; - - const reverseMap = new Map(Object.entries(rawMap as Record)); - - const messages = body["messages"]; - if (!Array.isArray(messages)) return body; - - type MsgLike = { - role?: string; - content?: string | Array>; - [key: string]: unknown; - }; - - const restored = (messages as MsgLike[]).map((msg) => { - const content = msg["content"]; - - if (typeof content === "string") { - const reconstructed = content.replace( - MARKER_RE, - (_m, sha: string) => reverseMap.get(sha) ?? _m - ); - return reconstructed !== content ? { ...msg, content: reconstructed } : { ...msg }; - } - - if (Array.isArray(content)) { - let changed = false; - const newContent = content.map((part) => { - if (part["type"] !== "text" || typeof part["text"] !== "string") return part; - const reconstructed = (part["text"] as string).replace( - MARKER_RE, - (_m, sha: string) => reverseMap.get(sha) ?? _m - ); - if (reconstructed !== part["text"]) { - changed = true; - return { ...part, text: reconstructed }; - } - return part; - }); - return changed ? { ...msg, content: newContent } : { ...msg }; - } - - return { ...msg }; - }); - - const { [DEDUP_MAP_KEY]: _dropped, ...restBody } = body; - void _dropped; - return { ...restBody, messages: restored }; -} diff --git a/tests/unit/compression/ccr-marker-retrieve.test.ts b/tests/unit/compression/ccr-marker-retrieve.test.ts index a12f767226..d265d8d0c6 100644 --- a/tests/unit/compression/ccr-marker-retrieve.test.ts +++ b/tests/unit/compression/ccr-marker-retrieve.test.ts @@ -11,7 +11,6 @@ import assert from "node:assert/strict"; import { ccrEngine, - reconstructCcr, retrieveBlock, recordRetrieval, shouldSkipCompression, @@ -109,30 +108,6 @@ describe("ccr engine", () => { assert.equal(retrieved, LARGE_TEXT, "retrieved block must equal the original verbatim text"); }); - it("reconstructs the body to deep-equal the original (round-trip)", () => { - resetCcrStore(); - const originalMessages = [ - { role: "user", content: LARGE_TEXT }, - { role: "assistant", content: "I understand your point." }, - ]; - const body = makeBody(originalMessages); - const result = ccrEngine.apply(body as Record); - - assert.equal(result.compressed, true, "should be compressed"); - - const reconstructed = reconstructCcr(result.body); - const reconstructedMessages = reconstructed.messages as Array<{ - role: string; - content: string; - }>; - - assert.deepEqual( - reconstructedMessages, - originalMessages, - "reconstructed messages must deep-equal original messages" - ); - }); - it("does NOT compress small blocks (below minChars threshold)", () => { resetCcrStore(); const body = makeBody([{ role: "user", content: SMALL_TEXT }]); @@ -205,14 +180,6 @@ describe("ccr engine", () => { assert.equal(result, null, "unknown hash must return null"); }); - it("reconstructCcr is idempotent when no markers are present", () => { - const body = makeBody([{ role: "user", content: SMALL_TEXT }]); - const reconstructed = reconstructCcr(body as Record); - assert.deepEqual(reconstructed.messages as Array<{ role: string; content: string }>, [ - { role: "user", content: SMALL_TEXT }, - ]); - }); - it("handles multipart content (type:text parts)", () => { resetCcrStore(); const body = { diff --git a/tests/unit/compression/session-dedup.test.ts b/tests/unit/compression/session-dedup.test.ts index 7bd4aae7b1..31a472ecd3 100644 --- a/tests/unit/compression/session-dedup.test.ts +++ b/tests/unit/compression/session-dedup.test.ts @@ -6,10 +6,7 @@ import { describe, it, before } from "node:test"; import assert from "node:assert/strict"; -import { - sessionDedupEngine, - reconstructSessionDedup, -} from "../../../open-sse/services/compression/engines/session-dedup/index.ts"; +import { sessionDedupEngine } from "../../../open-sse/services/compression/engines/session-dedup/index.ts"; import { registerBuiltinCompressionEngines, getCompressionEngine, @@ -88,26 +85,6 @@ describe("session-dedup engine", () => { assert.ok(result.stats!.compressedTokens < result.stats!.originalTokens); }); - it("round-trip: reconstructSessionDedup restores original body exactly", () => { - const body = makeBody([ - { role: "user", content: `Here is the code:\n${REPEATED_BLOCK}` }, - { role: "assistant", content: "I understand the code." }, - { role: "user", content: `Please review again:\n${REPEATED_BLOCK}` }, - ]); - - const result = sessionDedupEngine.apply(body as Record); - assert.equal(result.compressed, true); - - const restored = reconstructSessionDedup(result.body); - - // Deep-equal to original - assert.deepEqual( - restored.messages, - body.messages, - "reconstructed body must deep-equal original" - ); - }); - it("does NOT dedup small/unique blocks (no false positives)", () => { const body = makeBody([ { role: "user", content: "hi" },