From 5d89fa84e72a28d07273c588938ec10f692e2be0 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 19 Jun 2026 21:51:12 -0300 Subject: [PATCH] =?UTF-8?q?fix(compression):=20end-to-end=20audit=20?= =?UTF-8?q?=E2=80=94=20fixes=20across=20the=20whole=20compression=20flow?= =?UTF-8?q?=20(#4323)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(compression): SLM worker resolves deps+worker file without import.meta.url (B-SLM) The Next.js standalone bundle (webpack) replaces createRequire(import.meta.url) with a stub that always throws MODULE_NOT_FOUND, and freezes import.meta.url to the build-machine path. So depsAvailable() was always false (the worker never spawned) and resolveWorkerFile() anchored on a path absent at runtime — the SLM silently fell back to the aggressive summarizer in production. Confirmed by inspecting dist/.build/next/server/chunks/26410.js (stub module 215743 + frozen file:// path). Replace both with filesystem probing from runtime anchors (process.cwd(), process.argv[1]) that survive the bundle. Necessary complement to #4286 (deps co-location) for the SLM to actually engage in prod; still fail-open without it. VPS live validation deferred (Rule #18); local resolver regression tests added. * fix(compression): ultra heuristic preserves code blocks / inline code / URLs (B-ULTRA-CODE) ultra.ts called pruneByScore on raw text with no tombstoning, so the token pruner dropped low-score code tokens (`b)`, `{`, `+`) inside fenced blocks while leaving the fence markers intact — output that looked like valid code but was syntactically destroyed. caveman + llmlingua both extract/restore preserved blocks first; ultra was the only pruning engine that didn't. Add pruneProseOnly(): extractPreservedBlocks tombstones fenced code, inline code, URLs, CONST_CASE, versions; only the prose between placeholders is pruned; preserved blocks are re-stitched verbatim. * fix(compression): GCF round-trips values containing the inline-array pattern [..]: (B-GCF-QUOTE) A value like `ERR[404]: Not Found` / `[Speaker 1]: Hello` nested one level deep was emitted bare and re-parsed by the decoder as an inline-array header → it threw `count_mismatch` (or silently decoded wrong), losing the whole block. headroomEngine .apply() ships such blobs in prod, so this was a reachable lossless violation. Two complementary fixes, both per SPEC §2.4: - encode: needsQuote() now quotes strings matching `[`…`]``:` (spec compliance / other decoders). - decode: the inline-array branch only fires when the bracket is in the KEY position (no `=` before it), so a quoted `note="ERR[404]: …"` value falls through to key=value. * fix(compression): aggressive fidelity — keep text blocks, compress Anthropic tool_result, don't corrupt JSON (B-AGG-*) Three fidelity fixes in the aggressive path (each TDD, aggressive-fidelity.test.ts): - B-AGG-TEXTDROP: replaceTextContent dropped 2nd+ text blocks unconditionally; now a trailing block is dropped only when its text is already subsumed by newText, else kept. - B-AGG-ANTHROPIC-TR: tool-result compression only fired for OpenAI role:tool messages; now Anthropic-shape tool_result content blocks (inside user messages) are compressed too, preserving tool_use_id + block structure. - B-AGG-JSONTAG: the [COMPRESSED:aging:*] prefix corrupted JSON/code payloads; pure JSON is now kept verbatim+untagged (stays parseable), fenced blocks get the tag on a preceding line. * fix(compression): accessibility collapse preserves [ref] anchors + fires on interleaved trees (B-MCPA11Y-*) - B-MCPA11Y-ANCHORS: collapseRepeated silently dropped the omitted middle siblings' [ref=eNN] anchors (the agent could no longer click them); now every omitted ref is kept alongside the collapse notice. Wires the previously-dead preserveRefPattern. Invariant: extractRefs(input) ⊆ extractRefs(output). - B-MCPA11Y-COLLAPSE: noise removal blanked lines (replace→""), and a blank line broke the sibling run so collapse never fired on realistic interleaved trees; noise lines are now deleted, and the sibling walk skips stray blanks. * fix(compression): rtk intensity scales the line budget (B-RTK-INTENSITY) The intensity knob only set smartTruncate's preserveHead/Tail (16↔24), which rarely fired because the matched filter capped lines first — so minimal/standard/aggressive produced byte-identical output on filter-matched tool output. effectiveMaxLines() now scales the effective line budget (minimal 1.5x, standard 1x, aggressive 0.5x) at both the per-filter and engine-level truncation sites. Both go through smartTruncate with priorityPatterns, so error/failure lines survive at every intensity (tested). * fix(compression): robust language detection + auto-detect honors the detected pack (B-LANG-*) - B-LANG-DETECTOR: detector was first-match-wins on a single keyword, and some hints are English-ambiguous ("configuration" in fr, "error" in es) → English text misclassified. Now score-based (count native-keyword hits, highest wins), and the two English-ambiguous words are removed from the hint lists, so a lone shared word never misclassifies while sparse-keyword languages (id) still detect on a single native word. - B-LANG-DORMANT: with autoDetectLanguage on but enabledPacks ["en"], detected non-English text fell back to the English pack, whose `articles` rule deletes foreign articles (pt-BR "a"/"o"). Auto-detect now uses the detected pack directly (it always has rules); enabledPacks still gates manual selection. * fix(compression): mode selection enables its engine + align stacked allowlist (B-MODE-ENGINE-DECOUPLE, B-PIPELINE-DIVERGENCE) - B-MODE-ENGINE-DECOUPLE: picking the standard/rtk MODE now runs caveman/rtk regardless of the per-engine enabled flag — the mode selection is the enable signal (the per-engine flag still gates stacked pipeline steps). Previously an operator who picked a mode but left the engine toggle off got silent 0% compression. - B-PIPELINE-DIVERGENCE: the global stackedPipeline normalizer stripped session-dedup/ccr/headroom/llmlingua (engines the combo path accepts via KNOWN_ENGINE_IDS). The allowlist now matches, so the global setting can use all registered engines. * docs(compression): correct SLM "stable" claim + document partial packs / stacked telemetry limits - The llmlingua `stable:true` comment claimed the bundle walk-up + deps-gate were "confirmed against the live install" — that was wrong (webpack froze import.meta.url and stubbed createRequire, so the worker never spawned in prod). Corrected to reflect B-SLM. - COMPRESSION_ENGINES.md: add a Known limitations section (SLM dep co-location requirement, partial de/fr/ja packs, no-op engines absent from engineBreakdown). * fix(compression): cast normalized engine id to CompressionPipelineStep['engine'] (typecheck) --- docs/compression/COMPRESSION_ENGINES.md | 19 ++ open-sse/services/compression/aggressive.ts | 40 +++- open-sse/services/compression/caveman.ts | 14 +- .../engines/headroom/gcf/decode_generic.ts | 7 +- .../engines/headroom/gcf/scalar.ts | 5 + .../compression/engines/llmlingua/index.ts | 10 +- .../compression/engines/llmlingua/worker.ts | 150 ++++++++----- .../mcpAccessibility/collapseRepeated.ts | 37 ++++ .../engines/mcpAccessibility/index.ts | 19 +- .../services/compression/engines/rtk/index.ts | 16 +- .../services/compression/languageDetector.ts | 30 ++- .../services/compression/messageContent.ts | 16 +- .../services/compression/progressiveAging.ts | 51 ++++- .../services/compression/strategySelector.ts | 7 +- .../compression/toolResultCompressor.ts | 63 ++++++ open-sse/services/compression/ultra.ts | 31 ++- src/lib/db/compression.ts | 27 ++- .../compression/aggressive-fidelity.test.ts | 209 ++++++++++++++++++ .../gcf-inline-array-quote.test.ts | 34 +++ .../language-detect-select.test.ts | 50 +++++ .../llmlingua-worker-resolution.test.ts | 76 +++++++ .../mcpAccessibility-anchors.test.ts | 57 +++++ .../compression/mode-and-pipeline.test.ts | 62 ++++++ tests/unit/compression/rtk-intensity.test.ts | 49 ++++ .../ultra-code-preservation.test.ts | 32 +++ 25 files changed, 1006 insertions(+), 105 deletions(-) create mode 100644 tests/unit/compression/aggressive-fidelity.test.ts create mode 100644 tests/unit/compression/gcf-inline-array-quote.test.ts create mode 100644 tests/unit/compression/language-detect-select.test.ts create mode 100644 tests/unit/compression/llmlingua-worker-resolution.test.ts create mode 100644 tests/unit/compression/mcpAccessibility-anchors.test.ts create mode 100644 tests/unit/compression/mode-and-pipeline.test.ts create mode 100644 tests/unit/compression/rtk-intensity.test.ts create mode 100644 tests/unit/compression/ultra-code-preservation.test.ts diff --git a/docs/compression/COMPRESSION_ENGINES.md b/docs/compression/COMPRESSION_ENGINES.md index a27b33ec65..616c8c68a8 100644 --- a/docs/compression/COMPRESSION_ENGINES.md +++ b/docs/compression/COMPRESSION_ENGINES.md @@ -284,6 +284,25 @@ Compression exposes five MCP tools: | `omniroute_list_compression_combos` | `read:compression` | List compression combos | | `omniroute_compression_combo_stats` | `read:compression` | Read combo/engine analytics | +## Known limitations + +- **LLMLingua-2 (SLM) requires co-located optional deps.** The worker only runs in a + production build when `@atjsh/llmlingua-2` + peers are co-located into + `dist/node_modules` (see `scripts/build/colocateOptionals.mjs`, #4286). Without them the + engine fail-opens (returns the original text). Worker resolution no longer depends on + `import.meta.url` (it dies in the standalone bundle) — it anchors on the runtime + cwd / `argv[1]`. +- **Caveman language packs `de` / `fr` / `ja` are partial.** They ship `context` + + `filler` + `structural` rules but no `dedup` / `ultra` packs, so `ultra` intensity is + no stronger than `full` for those languages (they use only their own rules — there is no + silent fall-back to the English `dedup`/`ultra` rules, which would mangle foreign text). + `en` / `es` / `id` / `pt-BR` are complete. Contributions of `dedup.json` + `ultra.json` + for the partial packs are welcome. +- **Stacked telemetry only lists engines that compressed.** A stacked-pipeline step whose + engine ran but produced 0 % savings returns `stats:null` and so does not appear in + `engineBreakdown` — indistinguishable from a step that was skipped. Distinguishing + "ran, 0 %" from "skipped" would require a breakdown-model change and is deferred. + ## Validation The focused gates for this area are: diff --git a/open-sse/services/compression/aggressive.ts b/open-sse/services/compression/aggressive.ts index 9f8e2ab3db..ce54ce04d7 100644 --- a/open-sse/services/compression/aggressive.ts +++ b/open-sse/services/compression/aggressive.ts @@ -1,6 +1,10 @@ import type { AggressiveConfig, CompressionStats, Summarizer } from "./types.ts"; import { DEFAULT_AGGRESSIVE_CONFIG } from "./types.ts"; -import { compressToolResult } from "./toolResultCompressor.ts"; +import { + compressToolResult, + compressAnthropicToolResultBlock, + isAnthropicToolResultBlock, +} from "./toolResultCompressor.ts"; import { applyAging } from "./progressiveAging.ts"; import { RuleBasedSummarizer } from "./summarizer.ts"; import { cavemanCompress } from "./caveman.ts"; @@ -65,15 +69,35 @@ export function compressAggressive( try { const afterToolResult = currentMessages.map((msg) => { if (cfg.preserveSystemPrompt !== false && msg.role === "system") return msg; - if (msg.role !== "tool" && msg.role !== "function") return msg; - const text = extractTextContent(msg.content); - if (!text || COMPRESSED_MARKER_RE.test(text)) return msg; - const result = compressToolResult(text, cfg.toolStrategies); - if (result.strategy === "none" || result.saved <= 0) return msg; + // OpenAI-shape: a dedicated tool/function message whose content is the result text. + if (msg.role === "tool" || msg.role === "function") { + const text = extractTextContent(msg.content); + if (!text || COMPRESSED_MARKER_RE.test(text)) return msg; - toolResultSavings += result.saved; - return setContent(msg, result.compressed); + const result = compressToolResult(text, cfg.toolStrategies); + if (result.strategy === "none" || result.saved <= 0) return msg; + + toolResultSavings += result.saved; + return setContent(msg, result.compressed); + } + + // Anthropic-shape: `tool_result` content blocks live inside a (typically user) + // message's content array. Compress the text inside each block while preserving + // the tool_use_id and block structure exactly (B-AGG-ANTHROPIC-TR). + if (!Array.isArray(msg.content)) return msg; + if (!msg.content.some(isAnthropicToolResultBlock)) return msg; + + let blockSavings = 0; + const nextContent = msg.content.map((part) => { + if (!isAnthropicToolResultBlock(part)) return part; + const { block, saved } = compressAnthropicToolResultBlock(part, cfg.toolStrategies); + blockSavings += saved; + return block; + }); + if (blockSavings <= 0) return msg; + toolResultSavings += blockSavings; + return { ...msg, content: nextContent }; }); currentMessages = afterToolResult; } catch (err) { diff --git a/open-sse/services/compression/caveman.ts b/open-sse/services/compression/caveman.ts index 6a0b092bef..60b06d0514 100644 --- a/open-sse/services/compression/caveman.ts +++ b/open-sse/services/compression/caveman.ts @@ -522,11 +522,17 @@ export function cavemanCompress( ? detectCompressionLanguage(textPart) : (config.language ?? "en"); const enabledPacks = config.enabledLanguagePacks ?? ["en", detectedLanguage]; - const language = enabledPacks.includes(detectedLanguage) + // When auto-detect is on, honor the detected language directly: the detector only + // returns languages that have a rule pack, and falling back to the English pack on + // non-English text mangles it (the EN `articles` rule deletes pt-BR "a"/"o"). + // enabledPacks still gates MANUAL pack selection (auto-detect off). (B-LANG-DORMANT) + const language = config.autoDetectLanguage ? detectedLanguage - : enabledPacks.includes("en") - ? "en" - : detectedLanguage; + : enabledPacks.includes(detectedLanguage) + ? detectedLanguage + : enabledPacks.includes("en") + ? "en" + : detectedLanguage; const rules = getRulesForContext(msg.role, config.intensity, language).filter( (rule) => !config.skipRules.includes(rule.name) ); diff --git a/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts b/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts index bfc9ea38b1..67c614d338 100644 --- a/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts +++ b/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts @@ -139,10 +139,13 @@ function parseObjectBody( continue; } - // Inline array. + // Inline array. The bracket must be in the KEY position: an inline-array header is + // `name[...]: …`, never `key=value` whose value happens to contain `[..]:` (e.g. a + // quoted `note="ERR[404]: Not Found"`). Guard on no `=` before the bracket so such + // values fall through to the key=value branch instead of throwing (B-GCF-QUOTE). if (!content.startsWith("@") && !content.startsWith("##")) { const bracketIdx = content.indexOf("["); - if (bracketIdx > 0) { + if (bracketIdx > 0 && !content.slice(0, bracketIdx).includes("=")) { const rest = content.slice(bracketIdx); const closeIdx = rest.indexOf("]"); if (closeIdx >= 0) { diff --git a/open-sse/services/compression/engines/headroom/gcf/scalar.ts b/open-sse/services/compression/engines/headroom/gcf/scalar.ts index c8887eb233..fdf28e6060 100644 --- a/open-sse/services/compression/engines/headroom/gcf/scalar.ts +++ b/open-sse/services/compression/engines/headroom/gcf/scalar.ts @@ -8,6 +8,10 @@ const JSON_NUMBER_RE = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/; const NUMERIC_LIKE_RE = /^[+-]\.?\d|^\.\d|^0\d/; +// SPEC §2.4: a bracket pair immediately followed by `:` (e.g. `ERR[404]: Not Found`, +// `[Speaker 1]: Hello`). Bare, on a line-level key=value RHS, the decoder re-parses +// this as an inline-array header → count_mismatch / wrong value (B-GCF-QUOTE). +const INLINE_ARRAY_RE = /\[[^\]]*\]:/; /** Check if a string value must be quoted per Section 2.4. */ export function needsQuote(s: string): boolean { @@ -17,6 +21,7 @@ export function needsQuote(s: string): boolean { if (NUMERIC_LIKE_RE.test(s)) return true; if (s[0] === " " || s[s.length - 1] === " ") return true; if (s[0] === "#" || s[0] === "@" || s[0] === ".") return true; + if (INLINE_ARRAY_RE.test(s)) return true; for (let i = 0; i < s.length; i++) { const c = s.charCodeAt(i); if ( diff --git a/open-sse/services/compression/engines/llmlingua/index.ts b/open-sse/services/compression/engines/llmlingua/index.ts index db33201488..83e12e7168 100644 --- a/open-sse/services/compression/engines/llmlingua/index.ts +++ b/open-sse/services/compression/engines/llmlingua/index.ts @@ -354,9 +354,13 @@ export const llmlinguaEngine: CompressionEngine = { inputScope: "messages", targetLatencyMs: 200, supportsPreview: false, - // Promoted to stable after VPS validation (2026-06-16): the deployed worker - // compressed real prose (209→107 ch, ok=true), and the bundle's walk-up - // resolution + optional-deps gate were confirmed against the live install. + // Stable. The worker model itself was VPS-validated (real prose 209→107 ch, ok=true), + // but the EARLIER "walk-up + optional-deps gate confirmed in the bundle" claim was + // wrong: the Next standalone bundle (webpack) froze `import.meta.url` to the build path + // and stubbed `createRequire`, so in production the gate was always false and the worker + // never spawned (it silently fell back to the aggressive summarizer). Fixed in B-SLM — + // worker.ts now resolves deps + worker file from runtime anchors (cwd / argv[1]). The + // optional deps must also be co-located into dist/node_modules (#4286) to actually run. stable: true, }, diff --git a/open-sse/services/compression/engines/llmlingua/worker.ts b/open-sse/services/compression/engines/llmlingua/worker.ts index 1d5d7e75fe..4371aaee7f 100644 --- a/open-sse/services/compression/engines/llmlingua/worker.ts +++ b/open-sse/services/compression/engines/llmlingua/worker.ts @@ -31,8 +31,6 @@ */ import { Worker } from "node:worker_threads"; -import { createRequire } from "node:module"; -import { fileURLToPath } from "node:url"; import path from "node:path"; import fs from "node:fs"; @@ -49,27 +47,83 @@ const FIRST_CALL_TIMEOUT_MS = 60000; /** * Gate probe: `@atjsh/llmlingua-2` is the entry package that declares the others * (`@huggingface/transformers`, `@tensorflow/tfjs`, `js-tiktoken`) as peers. We probe - * ONLY it because the peers are ESM-only — e.g. `@huggingface/transformers@3.5.2`'s - * `exports` has no `require`/`default` condition, so `require.resolve()` throws - * `MODULE_NOT_FOUND` for it even when it is installed and `import()`-able (verified on - * the VPS). Gating on all four would therefore always fail-open. The worker still - * fail-opens if a peer is genuinely missing at `import()` time. + * ONLY it (by manifest existence) because the peers are ESM-only and `require.resolve` + * throws for them even when installed; the worker still fail-opens if a peer is + * genuinely missing at `import()` time. + * + * ⚠️ We do NOT use `createRequire(import.meta.url).resolve()` nor any other + * `import.meta.url`-based resolution: the Next.js standalone bundle (webpack) replaces + * `createRequire(import.meta.url)` with a stub module that ALWAYS throws + * `MODULE_NOT_FOUND` and freezes `import.meta.url` to the build-machine path, so such a + * gate is always false / mis-anchored in production (B-SLM). We probe the filesystem + * from runtime anchors that survive the bundle instead. */ -const GATE_DEP = "@atjsh/llmlingua-2"; +const GATE_DEP_REL = path.join("node_modules", "@atjsh", "llmlingua-2", "package.json"); + +/** Relative path (from an install root) to the esbuild'd / source worker entry. */ +const WORKER_JS_REL = path.join( + "open-sse", + "services", + "compression", + "engines", + "llmlingua", + "onnxWorker.js" +); +const WORKER_TS_REL = path.join( + "open-sse", + "services", + "compression", + "engines", + "llmlingua", + "onnxWorker.ts" +); + +const MAX_WALK_UP = 8; + +/** + * Walk up from each anchor directory (≤ MAX_WALK_UP levels) and return the first + * ancestor that actually contains `relPath`, or null. Pure + exported for tests. + * + * This deliberately avoids `import.meta.url`/`__dirname` (both dead in the standalone + * bundle) — see the GATE_DEP_REL comment. + */ +export function firstAncestorWith(anchors: string[], relPath: string): string | null { + for (const anchor of anchors) { + if (!anchor) continue; + let dir = path.resolve(anchor); + for (let i = 0; i <= MAX_WALK_UP; i++) { + if (fs.existsSync(path.join(dir, relPath))) return dir; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + } + return null; +} + +/** + * Runtime install-root anchors that SURVIVE the standalone bundle: + * - `process.cwd()` — `dist/server.js` runs `process.chdir(__dirname)` → the dist root. + * - `dirname(process.argv[1])` — the entry script (server.js / bin), walked up. + */ +function runtimeAnchors(): string[] { + const anchors = [process.cwd()]; + const argv1 = process.argv[1]; + if (typeof argv1 === "string" && argv1) anchors.push(path.dirname(argv1)); + return anchors; +} // ─── optional-deps gate (memoized) ────────────────────────────────────────────── let _depsAvailable: boolean | null = null; -/** Lazily (and once) check whether the optional LLMLingua dependency stack is installed. */ -function depsAvailable(): boolean { +/** + * Lazily (and once) check whether the optional LLMLingua dependency stack is installed, + * by probing `node_modules/@atjsh/llmlingua-2/package.json` from the runtime anchors. + */ +export function depsAvailable(): boolean { if (_depsAvailable !== null) return _depsAvailable; - try { - createRequire(import.meta.url).resolve(GATE_DEP); - _depsAvailable = true; - } catch { - _depsAvailable = false; - } + _depsAvailable = firstAncestorWith(runtimeAnchors(), GATE_DEP_REL) !== null; return _depsAvailable; } @@ -106,55 +160,33 @@ const warmedModels = new Set(); let idleTimer: NodeJS.Timeout | null = null; /** - * Resolve the worker entry file across dev and prod. + * Resolve the worker entry file across dev and prod WITHOUT `import.meta.url`. * - * Dev: `onnxWorker.ts` sits next to this file and runs via the tsx loader. + * Prod: the worker is esbuild'd to `/open-sse/.../onnxWorker.js` + * (scripts/build/prepublish.ts) + kept by the pack-artifact allowlist. The install + * root is found by walking up the runtime anchors (cwd / argv[1] dir), since the + * bundled module location (`import.meta.url`) is frozen to the build machine. * - * Prod: this module is collapsed into a `.next` chunk and the worker is esbuild'd to - * `/open-sse/services/compression/engines/llmlingua/onnxWorker.js` - * (scripts/build/prepublish.ts) + kept by the pack-artifact allowlist. The process - * `cwd` is NOT the app root (pm2 starts it from `/root`), so cwd-relative resolution - * is unreliable — we instead WALK UP from this module's location (`import.meta.url`, - * which in the standalone bundle is a real `/.next/...` path) until we find - * an ancestor that actually contains the worker at its known relative path. cwd - * candidates remain as a last-resort fallback. First existing candidate wins; a `.ts` - * choice gets the tsx loader, a `.js` choice runs natively. + * Dev (tsx): the same relative path resolves to the `.ts` source under the project + * root (cwd) and runs via the tsx loader. + * + * First existing candidate wins; a `.js` choice runs natively, a `.ts` choice gets the + * tsx loader. Exported for tests. */ -function resolveWorkerFile(): { workerFile: string; execArgv: string[] } { - const moduleDir = path.dirname(fileURLToPath(import.meta.url)); - const rel = path.join( - "open-sse", - "services", - "compression", - "engines", - "llmlingua", - "onnxWorker.js" - ); +export function resolveWorkerFile(): { workerFile: string; execArgv: string[] } { + const anchors = runtimeAnchors(); - // 1. Dev: sibling source/compiled file next to this module. - const devTs = path.join(moduleDir, "onnxWorker.ts"); - if (fs.existsSync(devTs)) return { workerFile: devTs, execArgv: ["--import", "tsx/esm"] }; - const devJs = path.join(moduleDir, "onnxWorker.js"); - if (fs.existsSync(devJs)) return { workerFile: devJs, execArgv: [] }; + // Prod first: the esbuild'd .js under the install root. + const jsRoot = firstAncestorWith(anchors, WORKER_JS_REL); + if (jsRoot) return { workerFile: path.join(jsRoot, WORKER_JS_REL), execArgv: [] }; - // 2. Prod: walk up from the bundled module location, then cwd, looking for the - // esbuild'd worker at /open-sse/.../onnxWorker.js. - const roots: string[] = []; - let dir = moduleDir; - for (let i = 0; i < 12; i++) { - roots.push(dir); - const parent = path.dirname(dir); - if (parent === dir) break; - dir = parent; - } - roots.push(process.cwd(), path.join(process.cwd(), "app")); - for (const root of roots) { - const candidate = path.join(root, rel); - if (fs.existsSync(candidate)) return { workerFile: candidate, execArgv: [] }; - } + // Dev: the .ts source (tsx loader). + const tsRoot = firstAncestorWith(anchors, WORKER_TS_REL); + if (tsRoot) + return { workerFile: path.join(tsRoot, WORKER_TS_REL), execArgv: ["--import", "tsx/esm"] }; - // 3. Nothing found — return the sibling .js path; the spawn will fail-open. - return { workerFile: devJs, execArgv: [] }; + // Nothing found — return a cwd-relative .js path; the spawn will fail-open. + return { workerFile: path.join(process.cwd(), WORKER_JS_REL), execArgv: [] }; } /** Reset the idle eviction timer; terminates the worker after the idle window. */ diff --git a/open-sse/services/compression/engines/mcpAccessibility/collapseRepeated.ts b/open-sse/services/compression/engines/mcpAccessibility/collapseRepeated.ts index de05516453..8e48e49a35 100644 --- a/open-sse/services/compression/engines/mcpAccessibility/collapseRepeated.ts +++ b/open-sse/services/compression/engines/mcpAccessibility/collapseRepeated.ts @@ -1,5 +1,28 @@ +import { MCP_ACCESSIBILITY_DEFAULTS } from "./constants.ts"; + const SIBLING_PATTERN = /^(\s*)-\s*([a-zA-Z]+)\b/; +/** + * Extract every `[ref=eNN]` anchor from a blob, preserving order, de-duplicated. These refs are how + * an agent clicks elements, so they MUST survive collapse. Uses the shared `preserveRefPattern`. + */ +export function extractRefs(text: string): string[] { + const seen = new Set(); + const refs: string[] = []; + // Fresh regex per call: the shared pattern carries the global flag (stateful lastIndex). + const pattern = new RegExp( + MCP_ACCESSIBILITY_DEFAULTS.preserveRefPattern.source, + MCP_ACCESSIBILITY_DEFAULTS.preserveRefPattern.flags + ); + for (const m of text.matchAll(pattern)) { + if (!seen.has(m[0])) { + seen.add(m[0]); + refs.push(m[0]); + } + } + return refs; +} + export function findNthSiblingEnd( lines: string[], start: number, @@ -64,6 +87,11 @@ export function collapseRepeated( j++; continue; } + // A blank line (noise removal can leave these) must not break a sibling run. + if (ln.trim() === "") { + j++; + continue; + } break; } const groupLen = j - i; @@ -74,6 +102,15 @@ export function collapseRepeated( out.push( `${indent}... [${groupLen - keepHead - keepTail} similar "${role}" items omitted by OmniRoute MCP filter]` ); + // BUG A invariant: the omitted middle siblings carry [ref=eNN] anchors the agent needs to + // click. Extract every ref from the dropped lines and keep them alongside the notice so + // extractRefs(input) ⊆ extractRefs(output) always holds. + const omittedRefs = extractRefs(lines.slice(headEnd, tailStart).join("\n")); + if (omittedRefs.length > 0) { + out.push( + `${indent} [refs of omitted "${role}" items (clickable): ${omittedRefs.join(" ")}]` + ); + } for (let k = tailStart; k < j; k++) out.push(lines[k]); } else { for (let k = i; k < j; k++) out.push(lines[k]); diff --git a/open-sse/services/compression/engines/mcpAccessibility/index.ts b/open-sse/services/compression/engines/mcpAccessibility/index.ts index 47ede74949..9750683aca 100644 --- a/open-sse/services/compression/engines/mcpAccessibility/index.ts +++ b/open-sse/services/compression/engines/mcpAccessibility/index.ts @@ -1,16 +1,25 @@ import { collapseRepeated } from "./collapseRepeated.ts"; import { MCP_ACCESSIBILITY_TAIL_RESERVE, type McpAccessibilityConfig } from "./constants.ts"; -const NOISE_PATTERNS: RegExp[] = [/^\s*-\s*generic:?\s*$/gm, /^\s*-\s*text:\s*""\s*$/gm]; +// Per-line (non-global, anchored) noise matchers. Used to DELETE whole noise lines rather than +// blank them: `replace(pattern, "")` would leave empty strings behind, and a blank line between two +// sibling headers breaks collapseRepeated's sibling run (an empty line is neither a sibling header +// nor an indented child), so collapse would never fire on realistic interleaved trees. +const NOISE_LINE_PATTERNS: RegExp[] = [/^\s*-\s*generic:?\s*$/, /^\s*-\s*text:\s*""\s*$/]; + +function isNoiseLine(line: string): boolean { + return NOISE_LINE_PATTERNS.some((p) => p.test(line)); +} export function smartFilterText(text: string, config: McpAccessibilityConfig): string { if (typeof text !== "string" || text.length < config.minLengthToProcess) { return text; } - let out = text; - for (const pattern of NOISE_PATTERNS) { - out = out.replace(pattern, ""); - } + // Drop noise lines entirely (not blank them) so interleaved noise does not split sibling runs. + let out = text + .split("\n") + .filter((line) => !isNoiseLine(line)) + .join("\n"); out = collapseRepeated( out, config.collapseThreshold, diff --git a/open-sse/services/compression/engines/rtk/index.ts b/open-sse/services/compression/engines/rtk/index.ts index 917dab77aa..9ea257e056 100644 --- a/open-sse/services/compression/engines/rtk/index.ts +++ b/open-sse/services/compression/engines/rtk/index.ts @@ -289,7 +289,7 @@ export function processRtkText( if (config.enabledFilters.length === 0 || config.enabledFilters.includes(filter.id)) { const filtered = applyLineFilter(result, { ...filter, - maxLines: filter.maxLines || config.maxLinesPerResult, + maxLines: effectiveMaxLines(filter.maxLines || config.maxLinesPerResult, config.intensity), }); result = filtered.text; if (filtered.appliedRules.length > 0) { @@ -352,7 +352,7 @@ export function processRtkText( } }); const truncated = smartTruncate(result, { - maxLines: config.maxLinesPerResult, + maxLines: effectiveMaxLines(config.maxLinesPerResult, config.intensity), maxChars: config.maxCharsPerResult, preserveHead: config.intensity === "aggressive" ? 16 : 24, preserveTail: config.intensity === "aggressive" ? 16 : 24, @@ -458,6 +458,18 @@ function processRtkContent( }; } +/** + * Scale a line budget by intensity so minimal / standard / aggressive produce + * meaningfully different output on truncation-based filters (B-RTK-INTENSITY). + * Truncation always runs through smartTruncate with priorityPatterns, so error / + * failure lines survive at EVERY intensity. (Include/collapse filters like docker-logs + * compress by content, not line budget, so they are intensity-independent by nature.) + */ +export function effectiveMaxLines(base: number, intensity: string | undefined): number { + const factor = intensity === "aggressive" ? 0.5 : intensity === "minimal" ? 1.5 : 1; + return Math.max(1, Math.round(base * factor)); +} + export function applyRtkCompression( body: Record, options: { config?: Partial; stepConfig?: Record } = {} diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts index 3b515ab2bb..230d6086f8 100644 --- a/open-sse/services/compression/languageDetector.ts +++ b/open-sse/services/compression/languageDetector.ts @@ -1,17 +1,39 @@ const LANGUAGE_HINTS: Record = { "pt-BR": [/\b(?:voce|você|preciso|arquivo|codigo|código|erro|falha|obrigado)\b/i], - es: [/\b(?:necesito|archivo|codigo|código|error|fallo|gracias|puedes)\b/i], + // NOTE: English-ambiguous words are intentionally excluded — "error" (es) and + // "configuration" (fr) are identical in English and would misclassify English text. + // Spanish/French keep their distinctive native spellings (fallo / erreur, etc). + es: [/\b(?:necesito|archivo|codigo|código|fallo|gracias|puedes)\b/i], de: [/\b(?:ich|datei|fehler|bitte|kannst|konfiguration|danke)\b/i], - fr: [/\b(?:fichier|erreur|merci|peux|configuration|besoin)\b/i], + fr: [/\b(?:fichier|erreur|merci|peux|besoin)\b/i], ja: [/[\u3040-\u30ff]/], id: [/\b(?:saya|kamu|anda|dengan|untuk|yang|tidak|bisa|terima\s+kasih|dari)\b/i], }; +/** + * Score each language by the NUMBER of native-keyword hits and pick the highest + * (English-ambiguous words are excluded from the hint lists, so a lone shared word + * never misclassifies English). Highest score wins; ties keep the earlier language; + * zero hits → English. (B-LANG-DETECTOR) + */ export function detectCompressionLanguage(text: string): string { + let best = "en"; + let bestScore = 0; for (const [language, patterns] of Object.entries(LANGUAGE_HINTS)) { - if (patterns.some((pattern) => pattern.test(text))) return language; + let score = 0; + for (const pattern of patterns) { + const global = pattern.flags.includes("g") + ? pattern + : new RegExp(pattern.source, pattern.flags + "g"); + const matches = text.match(global); + if (matches) score += matches.length; + } + if (score > bestScore) { + bestScore = score; + best = language; + } } - return "en"; + return best; } export function listSupportedCompressionLanguages(): string[] { diff --git a/open-sse/services/compression/messageContent.ts b/open-sse/services/compression/messageContent.ts index 7bd414d924..5c3acb91e0 100644 --- a/open-sse/services/compression/messageContent.ts +++ b/open-sse/services/compression/messageContent.ts @@ -63,12 +63,22 @@ export function replaceTextContent(msg: ChatMessageLike, newText: string): ChatM return { ...msg, content: newText }; } + // The first text block receives `newText`. Trailing text blocks are normally + // already subsumed by `newText` (callers build it from the JOIN of all text + // blocks via extractTextContent), so we drop them to avoid duplicating content. + // But if a caller's `newText` does NOT contain a trailing block's text, dropping + // it would silently lose content the model can no longer see (B-AGG-TEXTDROP) — + // so in that case we keep the trailing block instead of returning []. let replaced = false; const content = msg.content.flatMap((part) => { if (!isTextBlock(part)) return [part]; - if (replaced) return []; - replaced = true; - return [{ ...part, text: newText }]; + if (!replaced) { + replaced = true; + return [{ ...part, text: newText }]; + } + const partText = part.text ?? ""; + if (partText && !newText.includes(partText)) return [part]; + return []; }); if (!replaced) { diff --git a/open-sse/services/compression/progressiveAging.ts b/open-sse/services/compression/progressiveAging.ts index 3dbbdd6d77..3edde55247 100644 --- a/open-sse/services/compression/progressiveAging.ts +++ b/open-sse/services/compression/progressiveAging.ts @@ -5,11 +5,54 @@ import { cavemanCompress } from "./caveman.ts"; import { extractTextContent, replaceTextContent, type ChatMessageLike } from "./messageContent.ts"; const COMPRESSED_MARKER_RE = /^\[COMPRESSED:/; +const JSON_PREFIX_RE = /^\s*[{[]/; +const FENCE_RE = /^\s*```/; function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } +/** + * Structured content that an inline `[COMPRESSED:...]` prefix would corrupt: + * a pure-JSON payload (parses as JSON) or a fenced code block (B-AGG-JSONTAG). + */ +type StructuredKind = "json" | "fenced" | null; + +function structuredKind(text: string): StructuredKind { + const trimmed = text.trim(); + if (FENCE_RE.test(trimmed) && trimmed.endsWith("```")) return "fenced"; + if (JSON_PREFIX_RE.test(trimmed)) { + try { + JSON.parse(trimmed); + return "json"; + } catch { + return null; + } + } + return null; +} + +/** + * Build the aged content for a message, keeping structured payloads intact: + * - pure JSON: leave verbatim and untagged (stays JSON.parse-able). Aging engines + * (lite/caveman) do not shrink JSON anyway, so nothing is lost; tracked in stats + * via the unchanged content. The recursion guard relies on structuredKind() on the + * next pass, so re-running aging is a no-op (idempotent). + * - fenced code block: place the tag on its own line BEFORE the fence so the block + * stays valid and the content still starts with `[COMPRESSED:` (recursion guard). + * - everything else: inline-prepend the tag as before. + */ +function tagAged(tier: string, originalText: string, compressed: string): string { + const kind = structuredKind(originalText); + if (kind === "json") { + return originalText; + } + if (kind === "fenced") { + return `[COMPRESSED:aging:${tier}]\n${originalText}`; + } + return `[COMPRESSED:aging:${tier}] ${compressed}`; +} + type ChatMessage = ChatMessageLike; type CompressedResult = { @@ -62,7 +105,7 @@ export function applyAging( typeof compressed.body.messages[0].content === "string" ? compressed.body.messages[0].content : extractTextContent(compressed.body.messages[0].content); - const tagged = `[COMPRESSED:aging:light] ${newContent}`; + const tagged = tagAged("light", text, newContent); saved += estimateTokens(text) - estimateTokens(tagged); result.push(setContent(msg, tagged)); } else { @@ -75,7 +118,7 @@ export function applyAging( typeof compressed.body.messages[0].content === "string" ? compressed.body.messages[0].content : extractTextContent(compressed.body.messages[0].content); - const tagged = `[COMPRESSED:aging:moderate] ${newContent}`; + const tagged = tagAged("moderate", text, newContent); saved += estimateTokens(text) - estimateTokens(tagged); result.push(setContent(msg, tagged)); } else { @@ -84,12 +127,12 @@ export function applyAging( } else { if (msg.role === "assistant") { const summary = sum.summarize([msg]); - const tagged = `[COMPRESSED:aging:fullSummary] ${summary}`; + const tagged = tagAged("fullSummary", text, summary); saved += estimateTokens(text) - estimateTokens(tagged); result.push(setContent(msg, tagged)); } else if (msg.role === "user") { const firstLine = text.split("\n")[0]?.slice(0, 120) ?? ""; - const tagged = `[COMPRESSED:aging:fullSummary] ${firstLine}`; + const tagged = tagAged("fullSummary", text, firstLine); saved += estimateTokens(text) - estimateTokens(tagged); result.push(setContent(msg, tagged)); } else { diff --git a/open-sse/services/compression/strategySelector.ts b/open-sse/services/compression/strategySelector.ts index 07ad484be2..a9810adf3d 100644 --- a/open-sse/services/compression/strategySelector.ts +++ b/open-sse/services/compression/strategySelector.ts @@ -110,7 +110,9 @@ export function applyCompression( } if (mode === "rtk") { return applyRtkCompression(body, { - config: options?.config?.rtkConfig, + // Selecting the "rtk" mode IS the enable signal — run it even if the per-engine + // rtkConfig.enabled flag is off (that flag gates stacked steps). (B-MODE-ENGINE-DECOUPLE) + config: { ...(options?.config?.rtkConfig ?? {}), enabled: true }, }); } const adapter = adaptBodyForCompression(body); @@ -147,6 +149,9 @@ export function applyCompression( ), } : {}), + // Selecting the "standard" mode runs caveman regardless of the per-engine + // cavemanConfig.enabled flag (that flag gates stacked steps). (B-MODE-ENGINE-DECOUPLE) + enabled: true, }; const result = cavemanCompress( compressionBody as Parameters[0], diff --git a/open-sse/services/compression/toolResultCompressor.ts b/open-sse/services/compression/toolResultCompressor.ts index 040b996d24..510246f839 100644 --- a/open-sse/services/compression/toolResultCompressor.ts +++ b/open-sse/services/compression/toolResultCompressor.ts @@ -9,6 +9,7 @@ export interface CompressionResult { const ANSI_RE = /\x1b\[[0-9;]*[a-zA-Z]/g; const SHELL_PROMPT_RE = /\$\s/; const JSON_PREFIX_RE = /^\s*[{[]/; +const COMPRESSED_MARKER_RE = /^\[COMPRESSED:/; function isCodeLikeLine(rawLine: string): boolean { const line = rawLine.trimStart(); @@ -160,6 +161,68 @@ function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } +/** Minimal shape of an Anthropic `tool_result` content block. */ +export interface AnthropicToolResultBlock { + type: "tool_result"; + tool_use_id?: string; + content?: string | Array<{ type?: string; text?: string; [key: string]: unknown }>; + [key: string]: unknown; +} + +export function isAnthropicToolResultBlock(value: unknown): value is AnthropicToolResultBlock { + return ( + !!value && + typeof value === "object" && + (value as { type?: unknown }).type === "tool_result" + ); +} + +/** + * Compress the text inside an Anthropic-shape `tool_result` content block, + * reusing the same per-type strategies as OpenAI-shape tool messages. The + * `tool_use_id` and block type are preserved exactly; only the inner text is + * compressed. Returns the (possibly unchanged) block plus tokens saved. + */ +export function compressAnthropicToolResultBlock( + block: AnthropicToolResultBlock, + opts: ToolStrategiesConfig +): { block: AnthropicToolResultBlock; saved: number } { + const content = block.content; + + if (typeof content === "string") { + if (!content || COMPRESSED_MARKER_RE.test(content)) return { block, saved: 0 }; + const result = compressToolResult(content, opts); + if (result.strategy === "none" || result.saved <= 0) return { block, saved: 0 }; + return { block: { ...block, content: result.compressed }, saved: result.saved }; + } + + if (Array.isArray(content)) { + let saved = 0; + let changed = false; + const nextContent = content.map((part) => { + if ( + !part || + typeof part !== "object" || + part.type !== "text" || + typeof part.text !== "string" + ) { + return part; + } + const text = part.text; + if (!text || COMPRESSED_MARKER_RE.test(text)) return part; + const result = compressToolResult(text, opts); + if (result.strategy === "none" || result.saved <= 0) return part; + saved += result.saved; + changed = true; + return { ...part, text: result.compressed }; + }); + if (!changed) return { block, saved: 0 }; + return { block: { ...block, content: nextContent }, saved }; + } + + return { block, saved: 0 }; +} + export function compressToolResult(content: string, opts: ToolStrategiesConfig): CompressionResult { if (opts.fileContent) { const result = compressFileContent(content); diff --git a/open-sse/services/compression/ultra.ts b/open-sse/services/compression/ultra.ts index f53b722213..af02057a2f 100644 --- a/open-sse/services/compression/ultra.ts +++ b/open-sse/services/compression/ultra.ts @@ -1,10 +1,39 @@ import { pruneByScore } from "./ultraHeuristic.ts"; +import { extractPreservedBlocks } from "./preservation.ts"; import { DEFAULT_ULTRA_CONFIG } from "./types.ts"; import type { UltraConfig, CompressionStats, CompressionMode } from "./types.ts"; import { extractTextContent, mapTextContent, type ChatMessageLike } from "./messageContent.ts"; const COMPRESSED_PREFIX = "[COMPRESSED:"; +/** + * Prune PROSE only. Fenced code, inline code, URLs, CONST_CASE, versions, etc. are + * tombstoned by `extractPreservedBlocks` and re-stitched verbatim, so the heuristic + * NEVER mangles structured content (mirrors caveman.ts / llmlingua/index.ts). + * + * Without this, `pruneByScore` tokenizes the whole text and drops low-score tokens + * (`b)`, `{`, `+`, …) inside code blocks, corrupting them while leaving the fence + * markers intact — output that looks like valid code but isn't (B-ULTRA-CODE). + */ +function pruneProseOnly(text: string, rate: number, minScore: number): string { + const { text: withPlaceholders, blocks } = extractPreservedBlocks(text); + if (blocks.length === 0) return pruneByScore(text, rate, minScore); + + const placeholderToContent = new Map(blocks.map((b) => [b.placeholder, b.content])); + const escaped = blocks.map((b) => b.placeholder.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + const splitRe = new RegExp(`(${escaped.join("|")})`, "g"); + + return withPlaceholders + .split(splitRe) + .map((part) => { + if (!part) return ""; + const preserved = placeholderToContent.get(part); + if (preserved !== undefined) return preserved; // verbatim — never pruned + return pruneByScore(part, rate, minScore); // prose only + }) + .join(""); +} + export interface UltraCompressResult { messages: Array<{ role: string; content?: string | unknown[]; [key: string]: unknown }>; stats: CompressionStats; @@ -40,7 +69,7 @@ export function ultraCompress( const next = mapTextContent(msg, (textPart) => { if (!textPart || textPart.startsWith(COMPRESSED_PREFIX)) return textPart; messageOriginalChars += textPart.length; - const pruned = pruneByScore(textPart, compressionRate, minScoreThreshold); + const pruned = pruneProseOnly(textPart, compressionRate, minScoreThreshold); messageCompressedChars += pruned.length; return pruned; }) as Message; diff --git a/src/lib/db/compression.ts b/src/lib/db/compression.ts index 5f21064fcc..bbebb3a655 100644 --- a/src/lib/db/compression.ts +++ b/src/lib/db/compression.ts @@ -228,23 +228,32 @@ function normalizeContextEditingConfig(value: unknown): ContextEditingConfig { }; } -function normalizeStackedPipeline(value: unknown): CompressionPipelineStep[] { +// Engines allowed in the global stackedPipeline setting. MUST stay in sync with the +// compression-combo KNOWN_ENGINE_IDS (src/lib/db/compressionCombos.ts) — otherwise the +// global setting silently strips engines the combo path accepts (B-PIPELINE-DIVERGENCE). +const STACKED_PIPELINE_ENGINE_IDS = new Set([ + "lite", + "caveman", + "aggressive", + "ultra", + "rtk", + "headroom", + "session-dedup", + "ccr", + "llmlingua", +]); + +export function normalizeStackedPipeline(value: unknown): CompressionPipelineStep[] { const source = Array.isArray(value) ? value : (DEFAULT_COMPRESSION_CONFIG.stackedPipeline ?? []); const pipeline: CompressionPipelineStep[] = []; for (const entry of source) { const record = toRecord(entry); const engine = record.engine; - if ( - engine !== "lite" && - engine !== "caveman" && - engine !== "aggressive" && - engine !== "ultra" && - engine !== "rtk" - ) { + if (typeof engine !== "string" || !STACKED_PIPELINE_ENGINE_IDS.has(engine)) { continue; } pipeline.push({ - engine, + engine: engine as CompressionPipelineStep["engine"], ...(typeof record.intensity === "string" ? { intensity: record.intensity as CompressionPipelineStep["intensity"] } : {}), diff --git a/tests/unit/compression/aggressive-fidelity.test.ts b/tests/unit/compression/aggressive-fidelity.test.ts new file mode 100644 index 0000000000..3cba46b43a --- /dev/null +++ b/tests/unit/compression/aggressive-fidelity.test.ts @@ -0,0 +1,209 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + extractTextContent, + replaceTextContent, + type ChatMessageLike, +} from "../../../open-sse/services/compression/messageContent.ts"; +import { applyAging } from "../../../open-sse/services/compression/progressiveAging.ts"; +import { compressAggressive } from "../../../open-sse/services/compression/aggressive.ts"; +import type { AgingThresholds } from "../../../open-sse/services/compression/types.ts"; + +// ─── ISSUE 1 — B-AGG-TEXTDROP ──────────────────────────────────────────────── +// `replaceTextContent` previously dropped every text block after the first via +// flatMap → []. With the standard call pattern (newText = compressed JOIN of all +// text blocks), the joined original content must remain recoverable: nothing the +// model can no longer see may be silently lost. +describe("replaceTextContent — multi-text-block fidelity (B-AGG-TEXTDROP)", () => { + it("does not silently drop trailing text-block content absent from newText", () => { + const msg: ChatMessageLike = { + role: "user", + content: [ + { type: "text", text: "FIRST block alpha" }, + { type: "image", source: { foo: 1 } }, + { type: "text", text: "SECOND block bravo" }, + { type: "text", text: "THIRD block charlie" }, + ], + }; + + // newText does NOT subsume the trailing blocks (worst case): a caller that + // only summarized the first block. The trailing blocks' content must not + // silently vanish. + const replaced = replaceTextContent(msg, "NEWTEXT-only-first"); + + const out = extractTextContent(replaced.content); + assert.ok(out.includes("NEWTEXT-only-first"), "replacement text missing"); + assert.ok(out.includes("SECOND block bravo"), "second block silently dropped"); + assert.ok(out.includes("THIRD block charlie"), "third block silently dropped"); + + // Non-text blocks (image) must survive unchanged. + const blocks = replaced.content as Array<{ type?: string }>; + assert.ok( + blocks.some((b) => b.type === "image"), + "non-text block must be preserved" + ); + }); + + it("collapses trailing blocks when newText already subsumes them (no duplication)", () => { + const msg: ChatMessageLike = { + role: "user", + content: [ + { type: "text", text: "alpha" }, + { type: "text", text: "bravo" }, + ], + }; + // Standard call pattern: newText = compressed JOIN of all text blocks. + const joined = extractTextContent(msg.content); // "alpha\nbravo" + const replaced = replaceTextContent(msg, joined); + const blocks = replaced.content as Array<{ type?: string; text?: string }>; + const textBlocks = blocks.filter((b) => b.type === "text" || b.text !== undefined); + // Should collapse to a single text block — no duplicated "alpha"/"bravo". + assert.equal(textBlocks.length, 1, "subsumed trailing blocks should be collapsed"); + assert.equal(extractTextContent(replaced.content), joined); + }); + + it("aging a multi-text-block message keeps all original text represented", () => { + // Build a long conversation so the first (multi-block) message ages out. + const first: ChatMessageLike = { + role: "user", + content: [ + { type: "text", text: "alpha-marker request: fix login" }, + { type: "text", text: "bravo-marker error: TypeError: boom" }, + ], + }; + const msgs: ChatMessageLike[] = [first]; + for (let i = 1; i < 8; i++) { + msgs.push({ role: i % 2 ? "assistant" : "user", content: `filler ${i} ${"z".repeat(60)}` }); + } + const result = applyAging(msgs, { fullSummary: 10, moderate: 10, light: 3, verbatim: 1 }); + const out = extractTextContent(result.messages[0].content as ChatMessageLike["content"]); + // Light tier keeps content; both blocks' text must still be present (joined). + assert.ok(out.includes("alpha-marker"), "first text block lost during aging"); + assert.ok(out.includes("bravo-marker"), "second text block silently dropped during aging"); + }); +}); + +// ─── ISSUE 2 — B-AGG-ANTHROPIC-TR ──────────────────────────────────────────── +// Anthropic-shape tool_result blocks (a {type:"tool_result"} content block inside +// a user message) must be compressed too, preserving tool_use_id + block type. +describe("aggressive — Anthropic tool_result compression (B-AGG-ANTHROPIC-TR)", () => { + it("compresses the text inside an Anthropic tool_result block", () => { + const bigJsonArray = JSON.stringify( + Array.from({ length: 200 }, (_, i) => ({ id: i, name: `item${i}`, data: "x".repeat(40) })) + ); + const userMsg: ChatMessageLike = { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_ABC123", + content: [{ type: "text", text: bigJsonArray }], + }, + ], + }; + + const result = compressAggressive([userMsg]); + const outMsg = result.messages[0]; + const blocks = outMsg.content as Array>; + const tr = blocks.find((b) => b.type === "tool_result"); + + assert.ok(tr, "tool_result block must survive"); + assert.equal(tr!.type, "tool_result", "block type must be unchanged"); + assert.equal(tr!.tool_use_id, "toolu_ABC123", "tool_use_id must be preserved"); + + // The inner text must be smaller than the original (it was compressed). + const innerText = + typeof tr!.content === "string" + ? (tr!.content as string) + : ((tr!.content as Array<{ type?: string; text?: string }>) ?? []) + .filter((c) => c.type === "text") + .map((c) => c.text ?? "") + .join("\n"); + assert.ok( + innerText.length < bigJsonArray.length, + `tool_result inner text was not compressed (orig ${bigJsonArray.length}, got ${innerText.length})` + ); + assert.ok(result.stats.aggressive!.toolResultSavings > 0, "toolResultSavings must be > 0"); + }); + + it("compresses a string-form Anthropic tool_result block", () => { + const errorOutput = + "TypeError: Cannot read property 'x' of undefined\n" + + Array.from({ length: 40 }, (_, i) => ` at fn${i} (file${i}.ts:${i + 1}:${i + 5})`).join( + "\n" + ); + const userMsg: ChatMessageLike = { + role: "user", + content: [{ type: "tool_result", tool_use_id: "toolu_ERR", content: errorOutput }], + }; + const result = compressAggressive([userMsg]); + const tr = (result.messages[0].content as Array>).find( + (b) => b.type === "tool_result" + ); + assert.ok(tr, "tool_result block must survive"); + assert.equal(tr!.tool_use_id, "toolu_ERR"); + assert.ok((tr!.content as string).length < errorOutput.length, "string tool_result not compressed"); + }); +}); + +// ─── ISSUE 3 — B-AGG-JSONTAG ───────────────────────────────────────────────── +// Aging must not corrupt structured content (JSON / fenced code block) with a +// literal [COMPRESSED:...] inline prefix. +describe("progressiveAging — structured-content tag safety (B-AGG-JSONTAG)", () => { + const thresholds: AgingThresholds = { fullSummary: 10, moderate: 10, light: 3, verbatim: 1 }; + + function agedFirstContent(content: string, t: AgingThresholds): string { + const msgs: ChatMessageLike[] = [ + { role: "user", content }, + { role: "assistant", content: "a " + "z".repeat(60) }, + { role: "user", content: "b " + "z".repeat(60) }, + { role: "assistant", content: "c" }, + ]; + const result = applyAging(msgs, t); + const c = result.messages[0].content; + return typeof c === "string" ? c : extractTextContent(c as ChatMessageLike["content"]); + } + + it("keeps pure-JSON content JSON.parse-able after aging", () => { + const json = JSON.stringify({ + status: "ok", + items: Array.from({ length: 5 }, (_, i) => ({ id: i, name: `n${i}` })), + meta: { a: 1, b: 2 }, + }); + const aged = agedFirstContent(json, thresholds); + // Must remain valid JSON (no inline tag corruption). + assert.doesNotThrow(() => JSON.parse(aged), `aged JSON not parseable: ${aged.slice(0, 80)}`); + }); + + it("keeps a fenced code block valid after aging (tag outside the fence)", () => { + const fenced = "```json\n{\n \"a\": 1,\n \"b\": [1, 2, 3]\n}\n```"; + const aged = agedFirstContent(fenced, thresholds); + // The fenced block must still be present and intact. + assert.ok(aged.includes("```json"), "opening fence lost"); + assert.ok(aged.trimEnd().endsWith("```"), "closing fence lost"); + assert.ok(aged.includes('"a": 1'), "fenced payload corrupted"); + }); + + it("does not re-compress structured content on a second aging pass (recursion guard)", () => { + const json = JSON.stringify({ status: "ok", items: [{ id: 0 }, { id: 1 }], meta: { a: 1 } }); + const msgs: ChatMessageLike[] = [ + { role: "user", content: json }, + { role: "assistant", content: "a " + "z".repeat(60) }, + { role: "user", content: "b " + "z".repeat(60) }, + { role: "assistant", content: "c" }, + ]; + const first = applyAging(msgs, thresholds); + const second = applyAging(first.messages as ChatMessageLike[], thresholds); + const firstContent = JSON.stringify( + first.messages.map((m) => (m as ChatMessageLike).content) + ); + const secondContent = JSON.stringify( + second.messages.map((m) => (m as ChatMessageLike).content) + ); + assert.equal(secondContent, firstContent, "second aging pass changed structured content"); + // And it must still be parseable. + const c0 = (second.messages[0] as ChatMessageLike).content; + const text0 = typeof c0 === "string" ? c0 : extractTextContent(c0 as ChatMessageLike["content"]); + assert.doesNotThrow(() => JSON.parse(text0), "JSON corrupted after two aging passes"); + }); +}); diff --git a/tests/unit/compression/gcf-inline-array-quote.test.ts b/tests/unit/compression/gcf-inline-array-quote.test.ts new file mode 100644 index 0000000000..0904b61f0a --- /dev/null +++ b/tests/unit/compression/gcf-inline-array-quote.test.ts @@ -0,0 +1,34 @@ +/** + * Regression guard for B-GCF-QUOTE (lossless violation). + * + * SPEC §2.4 requires quoting a string that contains the inline-array pattern `[`…`]``:` + * (e.g. `ERR[404]: Not Found`, `[Speaker 1]: Hello`). needsQuote() lacked that rule, so + * such a value emitted bare on a line-level `key=value` RHS is re-parsed by the decoder + * (decode_generic.ts:142-160) as an inline-array header → throws `count_mismatch` (or + * silently decodes wrong). Reachable in prod: headroomEngine.apply() ships the blob. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { needsQuote } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/scalar.ts"; +import { encodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/generic.ts"; +import { decodeGeneric } from "@omniroute/open-sse/services/compression/engines/headroom/gcf/decode_generic.ts"; + +test("needsQuote flags the inline-array pattern [..]: (SPEC §2.4)", () => { + assert.equal(needsQuote("ERR[404]: Not Found"), true); + assert.equal(needsQuote("[Speaker 1]: Hello"), true); + assert.equal(needsQuote("[1]: y"), true); + // Must not over-trigger on innocuous brackets without the colon. + assert.equal(needsQuote("arr[0] index"), false); + assert.equal(needsQuote("plain value here"), false); +}); + +test("GCF round-trips nested values containing [..]: losslessly (B-GCF-QUOTE)", () => { + const data = [ + { id: 1, status: "x", code: 1, meta: { note: "ERR[404]: Not Found" } }, + { id: 2, status: "y", code: 2, meta: { note: "[Speaker 1]: Hello world" } }, + { id: 3, status: "z", code: 3, meta: { note: "plain note" } }, + ]; + const encoded = encodeGeneric(data); + const decoded = decodeGeneric(encoded); + assert.deepEqual(decoded, data); +}); diff --git a/tests/unit/compression/language-detect-select.test.ts b/tests/unit/compression/language-detect-select.test.ts new file mode 100644 index 0000000000..4a877f152b --- /dev/null +++ b/tests/unit/compression/language-detect-select.test.ts @@ -0,0 +1,50 @@ +/** + * Guards for B-LANG-DETECTOR + B-LANG-DORMANT. + * + * B-LANG-DETECTOR: the detector was first-match-wins on a single keyword, and some hint + * words are English-ambiguous ("configuration" in fr, "error" in es) → English text + * misclassified as fr/es. Now it is score-based and needs ≥2 hits to leave English. + * + * B-LANG-DORMANT: with autoDetectLanguage on but enabledPacks ["en"], detected non-English + * text fell back to the English pack, whose `articles` rule deletes foreign articles + * (pt-BR "a"/"o"). Auto-detect must use the detected pack (it always has rules). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { detectCompressionLanguage } from "@omniroute/open-sse/services/compression/languageDetector.ts"; +import { cavemanCompress } from "@omniroute/open-sse/services/compression/caveman.ts"; + +test("detector ignores single English-ambiguous keywords (configuration/error)", () => { + assert.equal( + detectCompressionLanguage("Please update the configuration and fix the error in the file"), + "en" + ); +}); + +test("detector still recognizes genuine non-English text (>=2 native keywords)", () => { + assert.equal(detectCompressionLanguage("Por favor preciso do arquivo com erro"), "pt-BR"); + assert.equal(detectCompressionLanguage("これはテストですコードを確認"), "ja"); +}); + +test("auto-detect uses the detected pt-BR pack, not the mangling English pack (B-LANG-DORMANT)", () => { + // pt-BR prose with an article the English `articles` rule would delete ("a configuração"). + const text = + "Por favor, você poderia revisar a configuração do arquivo? Obrigado pela ajuda com isso."; + const res = cavemanCompress({ messages: [{ role: "user", content: text }] } as Record< + string, + unknown + >, { + enabled: true, + autoDetectLanguage: true, + enabledLanguagePacks: ["en"], // the production-default that used to force the English pack + intensity: "full", + compressRoles: ["user"], + minMessageLength: 0, + } as Record); + const rules = res.stats?.rulesApplied ?? []; + // A pt-BR rule must have run (proves the pt-BR pack was selected, not English). + assert.ok( + rules.some((r) => r.startsWith("pt_")), + `expected a pt_* rule to apply, got: ${JSON.stringify(rules)}` + ); +}); diff --git a/tests/unit/compression/llmlingua-worker-resolution.test.ts b/tests/unit/compression/llmlingua-worker-resolution.test.ts new file mode 100644 index 0000000000..4924c83ab3 --- /dev/null +++ b/tests/unit/compression/llmlingua-worker-resolution.test.ts @@ -0,0 +1,76 @@ +/** + * Regression guard for B-SLM: the LLMLingua worker must resolve its deps + worker + * file WITHOUT relying on `import.meta.url`. + * + * Root cause (confirmed via dist/.build/next/server/chunks/26410.js): webpack + * replaces `createRequire(import.meta.url)` with a stub that throws MODULE_NOT_FOUND + * and freezes `import.meta.url` to the build-machine path. Both make the worker + * never spawn in the standalone bundle. The resolution must use runtime anchors + * (process.cwd() / process.argv[1]) instead. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + firstAncestorWith, + resolveWorkerFile, + depsAvailable, +} from "@omniroute/open-sse/services/compression/engines/llmlingua/worker.ts"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const WORKER_SRC = path.resolve( + here, + "../../../open-sse/services/compression/engines/llmlingua/worker.ts" +); + +/** Strip // line and block comments so we scan CODE, not doc-comments that may mention the banned APIs. */ +function stripComments(src: string): string { + return src.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:])\/\/.*$/gm, "$1"); +} + +test("worker.ts CODE never uses import.meta.url / createRequire (both die in the standalone bundle)", () => { + const code = stripComments(fs.readFileSync(WORKER_SRC, "utf8")); + assert.ok(!code.includes("import.meta"), "worker.ts code must not reference import.meta"); + assert.ok( + !code.includes('from "node:module"'), + "worker.ts must not import from node:module (createRequire)" + ); + assert.ok( + !code.includes('from "node:url"'), + "worker.ts must not import from node:url (fileURLToPath)" + ); +}); + +test("firstAncestorWith walks up from anchors to find a marker", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "slm-root-")); + try { + // Build /dist/node_modules/@atjsh/llmlingua-2/package.json and an anchor deep inside. + const pkgDir = path.join(tmp, "dist", "node_modules", "@atjsh", "llmlingua-2"); + fs.mkdirSync(pkgDir, { recursive: true }); + fs.writeFileSync(path.join(pkgDir, "package.json"), "{}"); + const anchor = path.join(tmp, "dist", ".build", "next", "server", "chunks"); + fs.mkdirSync(anchor, { recursive: true }); + + const rel = path.join("node_modules", "@atjsh", "llmlingua-2", "package.json"); + const found = firstAncestorWith([anchor], rel); + assert.equal(found, path.join(tmp, "dist"), "must find the dist root by walking up"); + assert.equal(firstAncestorWith([anchor], path.join("node_modules", "nope")), null); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +}); + +test("resolveWorkerFile returns an existing onnxWorker file (no import.meta.url)", () => { + // In this test env cwd is the worktree root, which contains the real source tree. + const { workerFile } = resolveWorkerFile(); + assert.ok(fs.existsSync(workerFile), `resolved worker file must exist: ${workerFile}`); + assert.ok(/onnxWorker\.(t|j)s$/.test(workerFile), `must point at onnxWorker: ${workerFile}`); +}); + +test("depsAvailable is true when @atjsh/llmlingua-2 is installed (symlinked node_modules)", () => { + assert.equal(depsAvailable(), true); +}); diff --git a/tests/unit/compression/mcpAccessibility-anchors.test.ts b/tests/unit/compression/mcpAccessibility-anchors.test.ts new file mode 100644 index 0000000000..d64ce834be --- /dev/null +++ b/tests/unit/compression/mcpAccessibility-anchors.test.ts @@ -0,0 +1,57 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { smartFilterText } from "@omniroute/open-sse/services/compression/engines/mcpAccessibility/index.ts"; +import { DEFAULT_MCP_ACCESSIBILITY_CONFIG } from "@omniroute/open-sse/services/compression/engines/mcpAccessibility/constants.ts"; + +/** Regex-extract every [ref=eNN] anchor from a blob, as a sorted unique list. */ +function extractRefs(s: string): string[] { + const refs = new Set(); + for (const m of s.matchAll(/\[ref=e\d+\]/g)) { + refs.add(m[0]); + } + return [...refs].sort(); +} + +/** + * Build a realistic accessibility snapshot: a `list` with 40 `listitem` siblings, each carrying a + * clickable child `link "..." [ref=eNN]`, with interleaved `- generic:` / `- text: ""` noise lines + * between siblings (exactly the kind of tree real MCP accessibility dumps produce). + */ +function buildSnapshot(): string { + const lines: string[] = ['- list "Results":']; + for (let i = 0; i < 40; i++) { + lines.push(` - listitem:`); + lines.push(` - generic:`); + lines.push(` - link "Result ${i}" [ref=e${i}]`); + lines.push(` - text: ""`); + // interleaved noise BETWEEN siblings — this is what breaks the sibling run for collapse + lines.push(` - generic:`); + lines.push(` - text: ""`); + } + // pad past minLengthToProcess so smartFilterText actually runs + return lines.join("\n").padEnd(3000, " "); +} + +test("collapse fires on interleaved tree AND no [ref=eXX] anchor is lost", () => { + const input = buildSnapshot(); + const out = smartFilterText(input, DEFAULT_MCP_ACCESSIBILITY_CONFIG); + + // BUG B: collapse must actually fire despite interleaved noise lines. + assert.ok(out.length < input.length, "output shorter (compressed)"); + assert.ok( + out.includes('items omitted by OmniRoute MCP filter'), + "collapse notice present (collapse fired)" + ); + + // BUG A: every [ref=eNN] in the input must survive in the output (agent can still click them). + const inRefs = extractRefs(input); + const outRefs = extractRefs(out); + assert.equal(inRefs.length, 40, "sanity: 40 refs in input"); + for (const r of inRefs) { + assert.ok(outRefs.includes(r), `ref ${r} must survive collapse (extractRefs(input) ⊆ extractRefs(output))`); + } + + // It still compresses meaningfully. + const savings = input.length - out.length; + assert.ok(savings > 0, "savings > 0"); +}); diff --git a/tests/unit/compression/mode-and-pipeline.test.ts b/tests/unit/compression/mode-and-pipeline.test.ts new file mode 100644 index 0000000000..7361f5d320 --- /dev/null +++ b/tests/unit/compression/mode-and-pipeline.test.ts @@ -0,0 +1,62 @@ +/** + * Guards for B-MODE-ENGINE-DECOUPLE and B-PIPELINE-DIVERGENCE. + * + * B-MODE-ENGINE-DECOUPLE: selecting a single MODE must run its engine even if the + * per-engine `enabled` flag is off — the mode selection IS the enable signal (the + * per-engine flag still gates STACKED pipeline steps). Previously standard/rtk silently + * no-op'd when cavemanConfig.enabled / rtkConfig.enabled was false. + * + * B-PIPELINE-DIVERGENCE: the global stackedPipeline normalizer stripped + * session-dedup/ccr/headroom/llmlingua (engines the combo path allows), so those engines + * could never run via the global setting. The allowlists must agree. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { applyCompression } from "@omniroute/open-sse/services/compression/strategySelector.ts"; +import { normalizeStackedPipeline } from "../../../src/lib/db/compression.ts"; + +test("standard mode compresses even when cavemanConfig.enabled is false (B-MODE-ENGINE-DECOUPLE)", () => { + const body = { + messages: [ + { + role: "user", + content: + "Please could you kindly review the configuration of the file. ".repeat(8) + + "I would really appreciate it, thank you so much for your help here.", + }, + ], + }; + const res = applyCompression(body, "standard", { + config: { + cavemanConfig: { enabled: false, compressRoles: ["user"], intensity: "full", minMessageLength: 0 }, + }, + } as Record); + assert.ok(res.compressed, "standard mode must run caveman regardless of cavemanConfig.enabled"); +}); + +test("rtk mode compresses even when rtkConfig.enabled is false (B-MODE-ENGINE-DECOUPLE)", () => { + const content = + Array.from({ length: 60 }, (_, i) => `line ${String(i).padStart(3, "0")} routine output`).join( + "\n" + ) + "\nERROR: boom"; + const res = applyCompression({ messages: [{ role: "tool", content }] }, "rtk", { + config: { rtkConfig: { enabled: false, intensity: "standard", applyToToolResults: true } }, + } as Record); + assert.ok(res.compressed, "rtk mode must run regardless of rtkConfig.enabled"); +}); + +test("normalizeStackedPipeline keeps headroom/ccr/session-dedup/llmlingua (B-PIPELINE-DIVERGENCE)", () => { + const pipe = normalizeStackedPipeline([ + { engine: "session-dedup" }, + { engine: "ccr" }, + { engine: "headroom" }, + { engine: "llmlingua" }, + { engine: "rtk", intensity: "standard" }, + { engine: "bogus-engine" }, // unknown ids still dropped + ]); + const engines = pipe.map((s) => s.engine); + for (const e of ["session-dedup", "ccr", "headroom", "llmlingua", "rtk"]) { + assert.ok(engines.includes(e), `${e} must survive normalize`); + } + assert.ok(!engines.includes("bogus-engine"), "unknown engine ids are still dropped"); +}); diff --git a/tests/unit/compression/rtk-intensity.test.ts b/tests/unit/compression/rtk-intensity.test.ts new file mode 100644 index 0000000000..b761faf6ce --- /dev/null +++ b/tests/unit/compression/rtk-intensity.test.ts @@ -0,0 +1,49 @@ +/** + * Regression guard for B-RTK-INTENSITY: the intensity knob used to be nearly inert + * (it only set smartTruncate's preserveHead/Tail 16↔24). It must now scale the effective + * line budget so minimal / standard / aggressive differ on truncation-based filters, + * WITHOUT ever dropping error/failure lines (priorityPatterns protect them at every + * intensity). Include/collapse filters (e.g. docker-logs) compress by content and are + * intensity-independent by nature — so the deterministic proof is on effectiveMaxLines. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + applyRtkCompression, + effectiveMaxLines, +} from "@omniroute/open-sse/services/compression/engines/rtk/index.ts"; + +test("effectiveMaxLines scales the line budget by intensity (minimal > standard > aggressive)", () => { + const min = effectiveMaxLines(120, "minimal"); + const std = effectiveMaxLines(120, "standard"); + const agg = effectiveMaxLines(120, "aggressive"); + assert.equal(std, 120, "standard is the baseline"); + assert.ok(min > std, `minimal (${min}) keeps more than standard (${std})`); + assert.ok(agg < std, `aggressive (${agg}) keeps fewer than standard (${std})`); + assert.ok(min > agg, "minimal keeps strictly more than aggressive"); + assert.ok(effectiveMaxLines(1, "aggressive") >= 1, "never below 1 line"); + assert.equal(effectiveMaxLines(120, undefined), 120, "unknown intensity = baseline"); +}); + +test("rtk preserves error/failure lines at EVERY intensity", () => { + const lines: string[] = []; + for (let i = 0; i < 400; i++) { + if (i === 120) lines.push("ERROR: connection refused at step 120"); + else if (i === 300) lines.push("FAILED: assertion mismatch at step 300"); + else lines.push(`line ${String(i).padStart(4, "0")} routine output text here`); + } + const content = lines.join("\n"); + for (const intensity of ["minimal", "standard", "aggressive"] as const) { + const res = applyRtkCompression({ messages: [{ role: "tool", content }] }, { + enabled: true, + intensity, + applyToToolResults: true, + } as Record); + const out = + typeof res.body.messages?.[0]?.content === "string" + ? (res.body.messages[0].content as string) + : ""; + assert.ok(out.includes("ERROR: connection refused"), `${intensity}: ERROR line must survive`); + assert.ok(out.includes("FAILED: assertion mismatch"), `${intensity}: FAILED line must survive`); + } +}); diff --git a/tests/unit/compression/ultra-code-preservation.test.ts b/tests/unit/compression/ultra-code-preservation.test.ts new file mode 100644 index 0000000000..fb4ff71891 --- /dev/null +++ b/tests/unit/compression/ultra-code-preservation.test.ts @@ -0,0 +1,32 @@ +/** + * Regression guard for B-ULTRA-CODE: the ultra heuristic must NOT corrupt fenced code + * blocks / inline code / URLs. It used to call pruneByScore on raw text (no tombstoning), + * so code tokens like `b)` / `{` / `+` scored < minScore and were pruned, turning + * `add(a, b) { return a + b; }` into `add(a, return`. caveman + llmlingua both + * extract/restore preserved blocks first; ultra must too. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { ultraCompress } from "@omniroute/open-sse/services/compression/ultra.ts"; + +test("ultraCompress preserves fenced code, inline code, and URLs byte-identical", () => { + const code = "```ts\nexport function add(a, b) {\n return a + b;\n}\n```"; + const inline = "`add(x, y)`"; + const url = "https://example.com/api/v1/auth?id=42"; + const filler = + "Here is a fairly long explanatory paragraph that should be pruned heavily because " + + "it contains lots of low information filler words and redundant phrasing repeated " + + "many times over and over again to ensure the heuristic actually triggers pruning. "; + // Realistic layout: fenced code block sits on its own line (markdown convention). + const text = `${filler}\n\n${code}\n\nThen call ${inline} and see ${url} for details.\n\n${filler}`; + + const { messages } = ultraCompress([{ role: "user", content: text }], { + maxTokensPerMessage: 0, + }); + const out = typeof messages[0].content === "string" ? messages[0].content : ""; + + assert.ok(out.includes(code), `fenced code block must survive byte-identical:\n${out}`); + assert.ok(out.includes(inline), `inline code must survive byte-identical:\n${out}`); + assert.ok(out.includes(url), `URL must survive byte-identical:\n${out}`); + assert.ok(out.length < text.length, "prose must still be compressed"); +});