mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-07 15:52:52 +03:00
* fix(compression): SLM worker resolves deps+worker file without import.meta.url (B-SLM) The Next.js standalone bundle (webpack) replaces createRequire(import.meta.url) with a stub that always throws MODULE_NOT_FOUND, and freezes import.meta.url to the build-machine path. So depsAvailable() was always false (the worker never spawned) and resolveWorkerFile() anchored on a path absent at runtime — the SLM silently fell back to the aggressive summarizer in production. Confirmed by inspecting dist/.build/next/server/chunks/26410.js (stub module 215743 + frozen file:// path). Replace both with filesystem probing from runtime anchors (process.cwd(), process.argv[1]) that survive the bundle. Necessary complement to #4286 (deps co-location) for the SLM to actually engage in prod; still fail-open without it. VPS live validation deferred (Rule #18); local resolver regression tests added. * fix(compression): ultra heuristic preserves code blocks / inline code / URLs (B-ULTRA-CODE) ultra.ts called pruneByScore on raw text with no tombstoning, so the token pruner dropped low-score code tokens (`b)`, `{`, `+`) inside fenced blocks while leaving the fence markers intact — output that looked like valid code but was syntactically destroyed. caveman + llmlingua both extract/restore preserved blocks first; ultra was the only pruning engine that didn't. Add pruneProseOnly(): extractPreservedBlocks tombstones fenced code, inline code, URLs, CONST_CASE, versions; only the prose between placeholders is pruned; preserved blocks are re-stitched verbatim. * fix(compression): GCF round-trips values containing the inline-array pattern [..]: (B-GCF-QUOTE) A value like `ERR[404]: Not Found` / `[Speaker 1]: Hello` nested one level deep was emitted bare and re-parsed by the decoder as an inline-array header → it threw `count_mismatch` (or silently decoded wrong), losing the whole block. headroomEngine .apply() ships such blobs in prod, so this was a reachable lossless violation. Two complementary fixes, both per SPEC §2.4: - encode: needsQuote() now quotes strings matching `[`…`]``:` (spec compliance / other decoders). - decode: the inline-array branch only fires when the bracket is in the KEY position (no `=` before it), so a quoted `note="ERR[404]: …"` value falls through to key=value. * fix(compression): aggressive fidelity — keep text blocks, compress Anthropic tool_result, don't corrupt JSON (B-AGG-*) Three fidelity fixes in the aggressive path (each TDD, aggressive-fidelity.test.ts): - B-AGG-TEXTDROP: replaceTextContent dropped 2nd+ text blocks unconditionally; now a trailing block is dropped only when its text is already subsumed by newText, else kept. - B-AGG-ANTHROPIC-TR: tool-result compression only fired for OpenAI role:tool messages; now Anthropic-shape tool_result content blocks (inside user messages) are compressed too, preserving tool_use_id + block structure. - B-AGG-JSONTAG: the [COMPRESSED:aging:*] prefix corrupted JSON/code payloads; pure JSON is now kept verbatim+untagged (stays parseable), fenced blocks get the tag on a preceding line. * fix(compression): accessibility collapse preserves [ref] anchors + fires on interleaved trees (B-MCPA11Y-*) - B-MCPA11Y-ANCHORS: collapseRepeated silently dropped the omitted middle siblings' [ref=eNN] anchors (the agent could no longer click them); now every omitted ref is kept alongside the collapse notice. Wires the previously-dead preserveRefPattern. Invariant: extractRefs(input) ⊆ extractRefs(output). - B-MCPA11Y-COLLAPSE: noise removal blanked lines (replace→""), and a blank line broke the sibling run so collapse never fired on realistic interleaved trees; noise lines are now deleted, and the sibling walk skips stray blanks. * fix(compression): rtk intensity scales the line budget (B-RTK-INTENSITY) The intensity knob only set smartTruncate's preserveHead/Tail (16↔24), which rarely fired because the matched filter capped lines first — so minimal/standard/aggressive produced byte-identical output on filter-matched tool output. effectiveMaxLines() now scales the effective line budget (minimal 1.5x, standard 1x, aggressive 0.5x) at both the per-filter and engine-level truncation sites. Both go through smartTruncate with priorityPatterns, so error/failure lines survive at every intensity (tested). * fix(compression): robust language detection + auto-detect honors the detected pack (B-LANG-*) - B-LANG-DETECTOR: detector was first-match-wins on a single keyword, and some hints are English-ambiguous ("configuration" in fr, "error" in es) → English text misclassified. Now score-based (count native-keyword hits, highest wins), and the two English-ambiguous words are removed from the hint lists, so a lone shared word never misclassifies while sparse-keyword languages (id) still detect on a single native word. - B-LANG-DORMANT: with autoDetectLanguage on but enabledPacks ["en"], detected non-English text fell back to the English pack, whose `articles` rule deletes foreign articles (pt-BR "a"/"o"). Auto-detect now uses the detected pack directly (it always has rules); enabledPacks still gates manual selection. * fix(compression): mode selection enables its engine + align stacked allowlist (B-MODE-ENGINE-DECOUPLE, B-PIPELINE-DIVERGENCE) - B-MODE-ENGINE-DECOUPLE: picking the standard/rtk MODE now runs caveman/rtk regardless of the per-engine enabled flag — the mode selection is the enable signal (the per-engine flag still gates stacked pipeline steps). Previously an operator who picked a mode but left the engine toggle off got silent 0% compression. - B-PIPELINE-DIVERGENCE: the global stackedPipeline normalizer stripped session-dedup/ccr/headroom/llmlingua (engines the combo path accepts via KNOWN_ENGINE_IDS). The allowlist now matches, so the global setting can use all registered engines. * docs(compression): correct SLM "stable" claim + document partial packs / stacked telemetry limits - The llmlingua `stable:true` comment claimed the bundle walk-up + deps-gate were "confirmed against the live install" — that was wrong (webpack froze import.meta.url and stubbed createRequire, so the worker never spawned in prod). Corrected to reflect B-SLM. - COMPRESSION_ENGINES.md: add a Known limitations section (SLM dep co-location requirement, partial de/fr/ja packs, no-op engines absent from engineBreakdown). * fix(compression): cast normalized engine id to CompressionPipelineStep['engine'] (typecheck)
50 lines
2.4 KiB
TypeScript
50 lines
2.4 KiB
TypeScript
/**
|
|
* Regression guard for B-RTK-INTENSITY: the intensity knob used to be nearly inert
|
|
* (it only set smartTruncate's preserveHead/Tail 16↔24). It must now scale the effective
|
|
* line budget so minimal / standard / aggressive differ on truncation-based filters,
|
|
* WITHOUT ever dropping error/failure lines (priorityPatterns protect them at every
|
|
* intensity). Include/collapse filters (e.g. docker-logs) compress by content and are
|
|
* intensity-independent by nature — so the deterministic proof is on effectiveMaxLines.
|
|
*/
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
applyRtkCompression,
|
|
effectiveMaxLines,
|
|
} from "@omniroute/open-sse/services/compression/engines/rtk/index.ts";
|
|
|
|
test("effectiveMaxLines scales the line budget by intensity (minimal > standard > aggressive)", () => {
|
|
const min = effectiveMaxLines(120, "minimal");
|
|
const std = effectiveMaxLines(120, "standard");
|
|
const agg = effectiveMaxLines(120, "aggressive");
|
|
assert.equal(std, 120, "standard is the baseline");
|
|
assert.ok(min > std, `minimal (${min}) keeps more than standard (${std})`);
|
|
assert.ok(agg < std, `aggressive (${agg}) keeps fewer than standard (${std})`);
|
|
assert.ok(min > agg, "minimal keeps strictly more than aggressive");
|
|
assert.ok(effectiveMaxLines(1, "aggressive") >= 1, "never below 1 line");
|
|
assert.equal(effectiveMaxLines(120, undefined), 120, "unknown intensity = baseline");
|
|
});
|
|
|
|
test("rtk preserves error/failure lines at EVERY intensity", () => {
|
|
const lines: string[] = [];
|
|
for (let i = 0; i < 400; i++) {
|
|
if (i === 120) lines.push("ERROR: connection refused at step 120");
|
|
else if (i === 300) lines.push("FAILED: assertion mismatch at step 300");
|
|
else lines.push(`line ${String(i).padStart(4, "0")} routine output text here`);
|
|
}
|
|
const content = lines.join("\n");
|
|
for (const intensity of ["minimal", "standard", "aggressive"] as const) {
|
|
const res = applyRtkCompression({ messages: [{ role: "tool", content }] }, {
|
|
enabled: true,
|
|
intensity,
|
|
applyToToolResults: true,
|
|
} as Record<string, unknown>);
|
|
const out =
|
|
typeof res.body.messages?.[0]?.content === "string"
|
|
? (res.body.messages[0].content as string)
|
|
: "";
|
|
assert.ok(out.includes("ERROR: connection refused"), `${intensity}: ERROR line must survive`);
|
|
assert.ok(out.includes("FAILED: assertion mismatch"), `${intensity}: FAILED line must survive`);
|
|
}
|
|
});
|