pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-01 04:42:10 +03:00
Code Issues Packages Projects Releases Wiki Activity

chore(compression): remove vestigial reconstructCcr/SessionDedup round-trip helpers (#4226)

Browse Source 
Integrated into release/v3.8.29 (remove vestigial reconstructCcr/reconstructSessionDedup round-trip helpers).
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-06-18 23:10:12 -03:00
committed by GitHub
parent 90c4224c9c
commit 8b73b37d09
5 changed files with 18 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
open-sse/services/compression/engines/ccr/index.ts
View File

@@ -53,8 +53,6 @@ const ENGINE_ID = "ccr";
const DEFAULT_MIN_CHARS = 600;
/** Number of retrievals before a block is flagged "do-not-compress" for that principal. */
const RETRIEVAL_THRESHOLD = 3;
/** Regex to match CCR markers for reconstruction. */
const MARKER_RE = /\[CCR retrieve hash=([0-9a-f]{24}) chars=\d+\]/g;
/**
* Maximum number of entries in each bounded store.
* When inserting beyond this cap, the oldest entry (Map insertion order) is evicted.
@@ -306,60 +304,6 @@ function validateCcrConfig(config: Record<string, unknown>): EngineValidationRes
return { valid: errors.length === 0, errors };
}
// ─── reconstruction helper ────────────────────────────────────────────────────
/**
* Reconstruct a body by replacing all `[CCR retrieve hash=<24hex> chars=N]`
* markers with the stored verbatim blocks for the given principal.
*
* Returns a new body object with markers restored to their original text.
* Markers that do not belong to the given principal remain unchanged.
*/
export function reconstructCcr(
body: Record<string, unknown>,
principalId?: string
): Record<string, unknown> {
const messages = body["messages"];
if (!Array.isArray(messages)) return body;
type MsgLike = {
role?: string;
content?: string | Array<Record<string, unknown>>;
[key: string]: unknown;
};
const restored = (messages as MsgLike[]).map((msg) => {
const content = msg["content"];
if (typeof content === "string") {
const reconstructed = content.replace(MARKER_RE, (_m, hash: string) => {
return retrieveBlock(hash, principalId) ?? _m;
});
return reconstructed !== content ? { ...msg, content: reconstructed } : { ...msg };
}
if (Array.isArray(content)) {
let changed = false;
const newContent = content.map((part) => {
if (part["type"] !== "text" || typeof part["text"] !== "string") return part;
const reconstructed = (part["text"] as string).replace(MARKER_RE, (_m, hash: string) => {
return retrieveBlock(hash, principalId) ?? _m;
});
if (reconstructed !== part["text"]) {
changed = true;
return { ...part, text: reconstructed };
}
return part;
});
return changed ? { ...msg, content: newContent } : { ...msg };
}
return { ...msg };
});
return { ...body, messages: restored };
}
// ─── engine export ────────────────────────────────────────────────────────────
export const ccrEngine: CompressionEngine = {

5
open-sse/services/compression/engines/headroom/index.ts
View File

@@ -40,7 +40,6 @@ import {
GCF_FENCE_OPEN,
GCF_FENCE_CLOSE,
decodeTabular,
TABULAR_MARKER_RE,
} from "./tabular.ts";
export { encodeTabular, decodeTabular } from "./tabular.ts";
@@ -183,6 +182,10 @@ type MessageLike = {
* Reverse the headroom compaction: find every ```gcf-generic or ```omni-tabular
* block in message contents and decode it back to the original JSON string.
*
* No production caller by design — the compact form is sent to the provider as-is. This is
* exported as the round-trip ORACLE that proves the GCF/tabular encoder is lossless: the
* losslessness regression tests encode via apply() then decode here and assert deep-equal.
*
* Returns a new body with all compacted blocks expanded.
*/
export function reconstructHeadroom(body: Record<string, unknown>): Record<string, unknown> {

111
open-sse/services/compression/engines/session-dedup/index.ts
View File

@@ -45,16 +45,12 @@ const ENGINE_ID = "session-dedup";
const DEFAULT_MIN_BLOCK_CHARS = 80;
/** Minimum number of lines a block must span to be a dedup candidate. */
const MIN_BLOCK_LINES = 3;
/** Marker pattern for reconstruction. */
const MARKER_RE = /\[dedup:ref sha=([0-9a-f]{24})\]/g;
/** Key used to store the reverse map in the body object. */
const DEDUP_MAP_KEY = "__sessionDedupMap__";
// ─── hash helper (SHA-256 prefix, collision-resistant) ───────────────────────
function hashBlock(text: string): string {
// 24 hex / 96 bits — collision-resistant (a 32-bit djb2 could collide and make
// reconstruction restore the WRONG block). Pass 2 additionally verifies block
// 24 hex / 96 bits — collision-resistant (a 32-bit djb2 could collide and make a
// dedup marker reference the WRONG block). Pass 2 additionally verifies block
// equality before substituting, so a collision can never cause corruption.
return crypto.createHash("sha256").update(text).digest("hex").slice(0, 24);
}
@@ -100,7 +96,6 @@ function dedupMessageTexts(
minBlockChars: number
): {
deduped: Map<number, string>;
reverseMap: Map<string, string>;
dedupCount: number;
} {
// Pass 1: for each message, extract suffix blocks and record first ownership.
@@ -118,12 +113,6 @@ function dedupMessageTexts(
}
}
// Build reverse map (sha → block) for all first-seen blocks.
const reverseMap = new Map<string, string>();
for (const [sha, { block }] of firstSeen) {
reverseMap.set(sha, block);
}
// Pass 2: for each message, find blocks that were FIRST seen in an earlier message.
const deduped = new Map<number, string>();
let dedupCount = 0;
@@ -138,7 +127,7 @@ function dedupMessageTexts(
const sha = hashBlock(block);
const owner = firstSeen.get(sha);
// owner.block === block guards against a (now astronomically unlikely) hash
// collision substituting a marker that would reconstruct to the wrong text.
// collision substituting a marker that would reference the wrong block.
if (owner && owner.ownerMsgIdx < msgIdx && owner.block === block) {
dupBlocks.push({ block, sha });
}
@@ -174,7 +163,7 @@ function dedupMessageTexts(
}
}
return { deduped, reverseMap, dedupCount };
return { deduped, dedupCount };
}
// ─── message array processing ─────────────────────────────────────────────────
@@ -191,7 +180,7 @@ type MessageLike = {
function processMessages(
messages: MessageLike[],
minBlockChars: number
): { messages: MessageLike[]; reverseMap: Map<string, string>; dedupCount: number } {
): { messages: MessageLike[]; dedupCount: number } {
// Collect (msgIdx, text) for non-system string-content messages.
// For multipart, index each text part separately.
const msgTexts: Array<{ msgIdx: number; text: string }> = [];
@@ -213,13 +202,13 @@ function processMessages(
}
if (msgTexts.length < 2) {
return { messages, reverseMap: new Map(), dedupCount: 0 };
return { messages, dedupCount: 0 };
}
const { deduped, reverseMap, dedupCount } = dedupMessageTexts(msgTexts, minBlockChars);
const { deduped, dedupCount } = dedupMessageTexts(msgTexts, minBlockChars);
if (dedupCount === 0) {
return { messages, reverseMap, dedupCount: 0 };
return { messages, dedupCount: 0 };
}
const result = messages.map((msg, i) => {
@@ -248,7 +237,7 @@ function processMessages(
return { ...msg };
});
return { messages: result, reverseMap, dedupCount };
return { messages: result, dedupCount };
}
// ─── schema & validation ──────────────────────────────────────────────────────
@@ -328,11 +317,10 @@ export const sessionDedupEngine: CompressionEngine = {
}
const start = performance.now();
const {
messages: dedupedMessages,
reverseMap,
dedupCount,
} = processMessages(messages as MessageLike[], minBlockChars);
const { messages: dedupedMessages, dedupCount } = processMessages(
messages as MessageLike[],
minBlockChars
);
if (dedupCount === 0) {
return { body, compressed: false, stats: null };
@@ -343,16 +331,6 @@ export const sessionDedupEngine: CompressionEngine = {
messages: dedupedMessages,
};
// Store the reverse map as a NON-ENUMERABLE property so JSON.stringify does not
// include it in the serialized output (which goes to the upstream provider).
// reconstructSessionDedup reads it back via getOwnPropertyDescriptor.
Object.defineProperty(newBody, DEDUP_MAP_KEY, {
value: Object.fromEntries(reverseMap),
enumerable: false,
configurable: true,
writable: false,
});
const durationMs = Math.round(performance.now() - start);
const stats = createCompressionStats(
body,
@@ -378,66 +356,3 @@ export const sessionDedupEngine: CompressionEngine = {
return validateSessionDedupConfig(config);
},
};
// ─── reconstruction helper ────────────────────────────────────────────────────
/**
* Reverse the dedup: replace every `[dedup:ref sha=XXXXXXXX]` marker with the
* original block text stored in the reverse map attached to the body by
* `sessionDedupEngine.apply`.
*
* Returns a new body object without the internal `__sessionDedupMap__` key.
*/
export function reconstructSessionDedup(body: Record<string, unknown>): Record<string, unknown> {
// The reverse map is stored as a non-enumerable property so it doesn't appear
// in JSON.stringify. Access it via getOwnPropertyDescriptor.
const mapDescriptor = Object.getOwnPropertyDescriptor(body, DEDUP_MAP_KEY);
const rawMap = mapDescriptor?.value ?? body[DEDUP_MAP_KEY];
if (!rawMap || typeof rawMap !== "object" || Array.isArray(rawMap)) return body;
const reverseMap = new Map<string, string>(Object.entries(rawMap as Record<string, string>));
const messages = body["messages"];
if (!Array.isArray(messages)) return body;
type MsgLike = {
role?: string;
content?: string | Array<Record<string, unknown>>;
[key: string]: unknown;
};
const restored = (messages as MsgLike[]).map((msg) => {
const content = msg["content"];
if (typeof content === "string") {
const reconstructed = content.replace(
MARKER_RE,
(_m, sha: string) => reverseMap.get(sha) ?? _m
);
return reconstructed !== content ? { ...msg, content: reconstructed } : { ...msg };
}
if (Array.isArray(content)) {
let changed = false;
const newContent = content.map((part) => {
if (part["type"] !== "text" || typeof part["text"] !== "string") return part;
const reconstructed = (part["text"] as string).replace(
MARKER_RE,
(_m, sha: string) => reverseMap.get(sha) ?? _m
);
if (reconstructed !== part["text"]) {
changed = true;
return { ...part, text: reconstructed };
}
return part;
});
return changed ? { ...msg, content: newContent } : { ...msg };
}
return { ...msg };
});
const { [DEDUP_MAP_KEY]: _dropped, ...restBody } = body;
void _dropped;
return { ...restBody, messages: restored };
}

33
tests/unit/compression/ccr-marker-retrieve.test.ts
View File

@@ -11,7 +11,6 @@ import assert from "node:assert/strict";
import {
ccrEngine,
reconstructCcr,
retrieveBlock,
recordRetrieval,
shouldSkipCompression,
@@ -109,30 +108,6 @@ describe("ccr engine", () => {
assert.equal(retrieved, LARGE_TEXT, "retrieved block must equal the original verbatim text");
});
it("reconstructs the body to deep-equal the original (round-trip)", () => {
resetCcrStore();
const originalMessages = [
{ role: "user", content: LARGE_TEXT },
{ role: "assistant", content: "I understand your point." },
];
const body = makeBody(originalMessages);
const result = ccrEngine.apply(body as Record<string, unknown>);
assert.equal(result.compressed, true, "should be compressed");
const reconstructed = reconstructCcr(result.body);
const reconstructedMessages = reconstructed.messages as Array<{
role: string;
content: string;
}>;
assert.deepEqual(
reconstructedMessages,
originalMessages,
"reconstructed messages must deep-equal original messages"
);
});
it("does NOT compress small blocks (below minChars threshold)", () => {
resetCcrStore();
const body = makeBody([{ role: "user", content: SMALL_TEXT }]);
@@ -205,14 +180,6 @@ describe("ccr engine", () => {
assert.equal(result, null, "unknown hash must return null");
});
it("reconstructCcr is idempotent when no markers are present", () => {
const body = makeBody([{ role: "user", content: SMALL_TEXT }]);
const reconstructed = reconstructCcr(body as Record<string, unknown>);
assert.deepEqual(reconstructed.messages as Array<{ role: string; content: string }>, [
{ role: "user", content: SMALL_TEXT },
]);
});
it("handles multipart content (type:text parts)", () => {
resetCcrStore();
const body = {

25
tests/unit/compression/session-dedup.test.ts
View File

@@ -6,10 +6,7 @@
import { describe, it, before } from "node:test";
import assert from "node:assert/strict";
import {
sessionDedupEngine,
reconstructSessionDedup,
} from "../../../open-sse/services/compression/engines/session-dedup/index.ts";
import { sessionDedupEngine } from "../../../open-sse/services/compression/engines/session-dedup/index.ts";
import {
registerBuiltinCompressionEngines,
getCompressionEngine,
@@ -88,26 +85,6 @@ describe("session-dedup engine", () => {
assert.ok(result.stats!.compressedTokens < result.stats!.originalTokens);
});
it("round-trip: reconstructSessionDedup restores original body exactly", () => {
const body = makeBody([
{ role: "user", content: `Here is the code:\n${REPEATED_BLOCK}` },
{ role: "assistant", content: "I understand the code." },
{ role: "user", content: `Please review again:\n${REPEATED_BLOCK}` },
]);
const result = sessionDedupEngine.apply(body as Record<string, unknown>);
assert.equal(result.compressed, true);
const restored = reconstructSessionDedup(result.body);
// Deep-equal to original
assert.deepEqual(
restored.messages,
body.messages,
"reconstructed body must deep-equal original"
);
});
it("does NOT dedup small/unique blocks (no false positives)", () => {
const body = makeBody([
{ role: "user", content: "hi" },