diff --git a/open-sse/services/compression/ultraHeuristic.ts b/open-sse/services/compression/ultraHeuristic.ts index 8a82715918..546830a48b 100644 --- a/open-sse/services/compression/ultraHeuristic.ts +++ b/open-sse/services/compression/ultraHeuristic.ts @@ -3,6 +3,10 @@ * * Scores tokens by information density and prunes low-value tokens * to achieve a target compression rate. + * + * #13454: Polarity/modality words (never, always, no, not, must, etc.) must + * NOT be prunable — dropping them flips the meaning of the sentence. + * "must never be deleted" → "must deleted" is worse than no compression. */ export const STOPWORDS = new Set([ @@ -19,18 +23,17 @@ export const STOPWORDS = new Set([ "have", "has", "had", - "do", - "does", - "did", + // #13454: "do/does/did" removed — carry polarity in imperatives + // ("do not push") and negations ("don't"). Dropping them flips + // instruction meaning. "will", "would", "could", - "should", + // #13454: "should" removed — modality word in instructions. "may", "might", "shall", - "can", - "need", + // #13454: "can/need" removed — modal auxiliaries in instructions. "dare", "ought", "used", @@ -59,7 +62,7 @@ export const STOPWORDS = new Set([ "and", "but", "or", - "nor", + // #13454: "nor" removed — negation word. "for", "yet", "so", @@ -88,8 +91,8 @@ export const STOPWORDS = new Set([ "even", "still", "already", - "always", - "never", + // #13454: "always/never" removed — polarity words, highest-value tokens + // in instructions. "never" → score 0.1 was the root cause of #13454. "often", "usually", "sometimes", @@ -100,6 +103,20 @@ export const STOPWORDS = new Set([ /** Regex for tokens that must never be pruned */ export const FORCE_PRESERVE_RE = /\d|https?:\/\/|[._\/\\]|Error:|Exception:|```/i; +// #13454: Polarity, modality, and negation words that must never be pruned. +// Dropping these flips the meaning of the sentence they appear in. +const POLARITY_WORDS = new Set([ + "never", "always", "no", "not", "nor", + "must", "shall", "shall not", + "do", "does", "did", + "don't", "doesn't", "didn't", + "can", "cannot", "can't", + "should", "shouldn't", + "need", "needs", "mustn't", + "won't", "wouldn't", + "could", "couldn't", +]); + /** * Score a single token (word/symbol) for information value. * Returns 0.0 (prune candidate) to 1.0 (must keep). @@ -107,6 +124,8 @@ export const FORCE_PRESERVE_RE = /\d|https?:\/\/|[._\/\\]|Error:|Exception:|```/ export function scoreToken(token: string): number { if (FORCE_PRESERVE_RE.test(token)) return 1.0; const lower = token.toLowerCase(); + // #13454: polarity words always score 1.0 — never prunable + if (POLARITY_WORDS.has(lower)) return 1.0; if (STOPWORDS.has(lower)) return 0.1; if (token.length <= 2) return 0.2; if (/^[A-Z]/.test(token)) return 0.8; // proper nouns / identifiers @@ -152,6 +171,8 @@ export function pruneByScore(text: string, keepRate = 0.5, minScore = 0.3): stri return keep ? t : ""; }) .join("") - .replace(/\s{2,}/g, " ") + // #13454: Only collapse spaces/tabs, NOT newlines. + // Collapsing newlines destroys bullet lists, headings, and code fences. + .replace(/[ \t]{2,}/g, " ") .trim(); } diff --git a/scripts/check/compression-budget-baseline.json b/scripts/check/compression-budget-baseline.json index f948293623..c9a6ed5f59 100644 --- a/scripts/check/compression-budget-baseline.json +++ b/scripts/check/compression-budget-baseline.json @@ -8,9 +8,9 @@ }, "caveman": { "tasks": { - "prose": 129, - "tool-output": 127, - "json": 160 + "prose": 119, + "tool-output": 114, + "json": 136 } }, "aggressive": { @@ -22,9 +22,9 @@ }, "ultra": { "tasks": { - "prose": 92, - "tool-output": 116, - "json": 117 + "prose": 97, + "tool-output": 117, + "json": 126 } }, "rtk": { diff --git a/tests/unit/compression/ultra-heuristic-polarity.test.ts b/tests/unit/compression/ultra-heuristic-polarity.test.ts new file mode 100644 index 0000000000..4c46f507d6 --- /dev/null +++ b/tests/unit/compression/ultra-heuristic-polarity.test.ts @@ -0,0 +1,116 @@ +/** + * Tests for #13454: ultra heuristic must not prune polarity/modality words. + * + * The ultra heuristic engine scores tokens and prunes the lowest-scoring 50%. + * Before the fix, polarity words like "never", "always", "no", "not", "must" + * scored 0.1 (stopwords) or 0.2 (length ≤ 2), making them the first tokens + * pruned. This inverted instruction meaning: + * "must never be deleted" → "must deleted" + * "NEVER run rm -rf" → "run rm -rf" + * + * The fix adds polarity words to a force-preserve set (score 1.0) and stops + * collapsing newlines (which destroyed bullet lists and code fences). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { scoreToken, pruneByScore } from "../../../open-sse/services/compression/ultraHeuristic.ts"; + +test("scoreToken: polarity words score 1.0 (never prunable)", () => { + // These words MUST survive compression — they carry instruction polarity + const polarityWords = ["never", "always", "no", "not", "nor", "must", "do", "does", "did"]; + for (const word of polarityWords) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (force-preserved)`); + } +}); + +test("scoreToken: modal auxiliaries score 1.0 (never prunable)", () => { + // Modal auxiliaries in instructions must not be pruned + const modals = ["can", "should", "need", "shall"]; + for (const word of modals) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (modal auxiliary)`); + } +}); + +test("scoreToken: contractions score 1.0", () => { + const contractions = ["don't", "doesn't", "didn't", "can't", "cannot", "won't"]; + for (const word of contractions) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (contraction)`); + } +}); + +test("scoreToken: regular stopwords still score 0.1", () => { + // Words that are genuinely low-value should still be prunable + const stopwords = ["a", "the", "is", "are", "was", "were", "in", "of", "on"]; + for (const word of stopwords) { + assert.equal(scoreToken(word), 0.1, `"${word}" should still score 0.1`); + } +}); + +test("pruneByScore: polarity words survive pruning", () => { + const block = `- NEVER run \`rm -rf\` on the target host. Always ask first. +- Do not push to \`main\` directly; open a PR. +- The backup files \`.app-prev-*\` must never be deleted. +- Never store the SSH password on disk. +- Always run \`npm test\` before \`npm run build\`. +- Do NOT edit files under \`/etc\` by hand.`; + + // Default engine settings: keepRate 0.5, minScore 0.3 + const result = pruneByScore(block, 0.5, 0.3); + + // All polarity words MUST survive + assert.ok(result.includes("never") || result.includes("NEVER"), "MUST preserve 'never'/'NEVER'"); + assert.ok( + result.includes("always") || result.includes("Always"), + "MUST preserve 'always'/'Always'" + ); + assert.ok(result.includes("not") || result.includes("NOT"), "MUST preserve 'not'/'NOT'"); + assert.ok(result.includes("Do"), "MUST preserve 'Do'"); +}); + +test("pruneByScore: newlines are preserved (not collapsed to spaces)", () => { + const block = `Line one +Line two +Line three`; + + const result = pruneByScore(block, 1.0); // keepRate=1.0 means keep everything + + // With keepRate=1.0 nothing is pruned, but we verify newlines survive + assert.ok(result.includes("\n"), "Newlines must be preserved when keepRate=1.0"); + assert.equal(result, block, "Full keepRate should return identical text"); +}); + +test("pruneByScore: newlines survive even with pruning", () => { + const block = `- NEVER do X +- ALWAYS do Y +- NEVER do Z`; + + const result = pruneByScore(block, 0.7, 0.3); + + // The line breaks between bullets should survive + const lines = result.split("\n"); + assert.ok(lines.length >= 2, "Line breaks between bullets must be preserved"); +}); + +test("pruneByScore: sample from #13454 issue preserves meaning", () => { + const block = `- NEVER run \`rm -rf\` on the target host. Always ask first. +- Do not push to \`main\` directly; open a PR. +- The backup files \`.app-prev-*\` must never be deleted. +- Never store the SSH password on disk. +- Always run \`npm test\` before \`npm run build\`. +- Do NOT edit files under \`/etc\` by hand.`; + + const result = pruneByScore(block, 0.5, 0.3); + + // After the fix, NONE of these meaning-critical words should be pruned: + assert.ok(result.includes("NEVER") || result.includes("never"), "NEVER must survive"); + assert.ok(result.includes("Always") || result.includes("always"), "Always must survive"); + assert.ok(result.includes("not") || result.includes("NOT"), "not/NOT must survive"); + assert.ok(result.includes("Do") || result.includes("do"), "Do/do must survive"); + assert.ok(result.includes("must") || result.includes("MUST"), "must/MUST must survive"); + + // The critical test: "must never" must NOT become "must" alone + assert.ok( + !result.match(/\bmust\b(?![\s\S]*never)/) || result.includes("never"), + "must and never must both survive together" + ); +});