fix(sse): spare live user message across all aggressive compression sub-paths (#11386)

Merged via consolidated batch validation. Aggressive compression could collapse the live user's active prompt into a [COMPRESSED:summary] marker; now spares the last user message across all sub-paths (applyAging, fallback summarizer, caveman/lite). Own test passes.

open-sse/services/compression/aggressive.ts

@@ -64,6 +64,7 @@ export function compressAggressive(
   let summarizerSavings = 0;
   let toolResultSavings = 0;
   let agingSavings = 0;
+  const lastUserIdx = currentMessages.findLastIndex((m) => m.role === "user");
 
   // Step 1: Tool-result compression
   try {
@@ -110,7 +111,8 @@ export function compressAggressive(
       currentMessages,
       cfg.thresholds,
       summarizer,
-      cfg.preserveSystemPrompt !== false
+      cfg.preserveSystemPrompt !== false,
+      lastUserIdx
     );
     agingSavings = agingResult.saved;
     currentMessages = agingResult.messages as ChatMessage[];
@@ -121,8 +123,9 @@ export function compressAggressive(
   // Step 3: Fallback summarizer for remaining long messages
   if (cfg.summarizerEnabled) {
     try {
-      currentMessages = currentMessages.map((msg) => {
+      currentMessages = currentMessages.map((msg, idx) => {
         if (cfg.preserveSystemPrompt !== false && msg.role === "system") return msg;
+        if (idx === lastUserIdx) return msg;
         const text = extractTextContent(msg.content);
         if (!text || COMPRESSED_MARKER_RE.test(text)) return msg;
         if (text.length <= cfg.maxTokensPerMessage * 4) return msg;
@@ -133,7 +136,10 @@ export function compressAggressive(
         });
         if (summary && summary.length < text.length) {
           summarizerSavings += estimateTokens(text) - estimateTokens(summary);
-          return setContent(msg, `[COMPRESSED:summary] ${summary}`);
+          const finalSummary = COMPRESSED_MARKER_RE.test(summary)
+            ? summary
+            : `[COMPRESSED:summary] ${summary}`;
+          return setContent(msg, finalSummary);
         }
         return msg;
       });
@@ -153,13 +159,27 @@ export function compressAggressive(
 
   if (resultStats.savingsPercent < cfg.minSavingsThreshold * 100) {
     try {
-      const cavemanResult = cavemanCompress({ messages: currentMessages as unknown as Parameters<typeof cavemanCompress>[0]["messages"] });
-      if (cavemanResult?.compressed && cavemanResult.stats) {
-        const cavemanSavings = cavemanResult.stats.savingsPercent ?? 0;
-        if (cavemanSavings > resultStats.savingsPercent) {
-          currentMessages = (cavemanResult.body?.messages ?? currentMessages) as ChatMessage[];
-          resultStats.compressedTokens = cavemanResult.stats.compressedTokens ?? compressedTokens;
-          resultStats.savingsPercent = cavemanSavings;
+      const cavemanResult = cavemanCompress(
+        {
+          messages: currentMessages as unknown as Parameters<typeof cavemanCompress>[0]["messages"],
+        },
+        { enabled: true }
+      );
+      if (cavemanResult?.compressed && cavemanResult.body?.messages) {
+        const rawMsgs = cavemanResult.body.messages as ChatMessage[];
+        const candidateMsgs = rawMsgs.map((msg, idx) =>
+          idx === lastUserIdx ? currentMessages[idx] : msg
+        );
+        const candidateTokens = candidateMsgs.reduce(
+          (sum, m) => sum + estimateTokens(extractTextContent(m.content)),
+          0
+        );
+        const candidateSavings =
+          originalTokens > 0 ? ((originalTokens - candidateTokens) / originalTokens) * 100 : 0;
+        if (candidateSavings > resultStats.savingsPercent) {
+          currentMessages = candidateMsgs;
+          resultStats.compressedTokens = candidateTokens;
+          resultStats.savingsPercent = candidateSavings;
           resultStats.techniquesUsed.push("caveman-fallback");
         }
       }
@@ -172,12 +192,21 @@ export function compressAggressive(
         { messages: currentMessages },
         { preserveSystemPrompt: cfg.preserveSystemPrompt !== false }
       );
-      if (liteResult?.compressed && liteResult.stats) {
-        const liteSavings = liteResult.stats.savingsPercent ?? 0;
-        if (liteSavings > resultStats.savingsPercent) {
-          currentMessages = (liteResult.body?.messages ?? currentMessages) as ChatMessage[];
-          resultStats.compressedTokens = liteResult.stats.compressedTokens ?? compressedTokens;
-          resultStats.savingsPercent = liteSavings;
+      if (liteResult?.compressed && liteResult.body?.messages) {
+        const rawMsgs = liteResult.body.messages as ChatMessage[];
+        const candidateMsgs = rawMsgs.map((msg, idx) =>
+          idx === lastUserIdx ? currentMessages[idx] : msg
+        );
+        const candidateTokens = candidateMsgs.reduce(
+          (sum, m) => sum + estimateTokens(extractTextContent(m.content)),
+          0
+        );
+        const candidateSavings =
+          originalTokens > 0 ? ((originalTokens - candidateTokens) / originalTokens) * 100 : 0;
+        if (candidateSavings > resultStats.savingsPercent) {
+          currentMessages = candidateMsgs;
+          resultStats.compressedTokens = candidateTokens;
+          resultStats.savingsPercent = candidateSavings;
           resultStats.techniquesUsed.push("lite-fallback");
         }
       }

open-sse/services/compression/progressiveAging.ts

@@ -67,7 +67,8 @@ export function applyAging(
   messages: unknown[],
   thresholds?: AgingThresholds,
   summarizer?: Summarizer,
-  preserveSystemPrompt = true
+  preserveSystemPrompt = true,
+  spareUserIndex?: number
 ): { messages: unknown[]; saved: number } {
   const t = thresholds ?? DEFAULT_AGGRESSIVE_CONFIG.thresholds;
   const sum = summarizer ?? {
@@ -81,6 +82,9 @@ export function applyAging(
   const typed = messages as ChatMessage[];
   if (typed.length === 0) return { messages: [], saved: 0 };
 
+  const lastUserIdx =
+    spareUserIndex !== undefined ? spareUserIndex : typed.findLastIndex((m) => m.role === "user");
+
   const totalMessages = typed.length;
   const result: ChatMessage[] = [];
   let saved = 0;
@@ -89,7 +93,11 @@ export function applyAging(
     const msg = typed[i];
     const text = extractTextContent(msg.content);
 
-    if ((preserveSystemPrompt && msg.role === "system") || COMPRESSED_MARKER_RE.test(text)) {
+    if (
+      (preserveSystemPrompt && msg.role === "system") ||
+      COMPRESSED_MARKER_RE.test(text) ||
+      i === lastUserIdx
+    ) {
       result.push(msg);
       continue;
     }

tests/unit/compression-aggressive-spare-last-user.test.ts

@@ -0,0 +1,154 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { compressAggressive } from "../../open-sse/services/compression/aggressive.ts";
+import { extractTextContent } from "../../open-sse/services/compression/messageContent.ts";
+
+describe("Aggressive compression: spare live user instruction", () => {
+  it("spares live (last) user message and keeps tail marker intact", () => {
+    const tailMarker = "TAILMARKER-CRITICAL-PAYLOAD-9988";
+    // ~20KB content with tail marker at the end
+    const longContent =
+      "Let us review this codebase in detail.\n" +
+      "const x = 1;\n".repeat(1500) +
+      `\n${tailMarker}`;
+    assert.ok(longContent.length > 16000, `Expected content > 16KB, got ${longContent.length}`);
+
+    const messages = [{ role: "user", content: longContent }];
+
+    const result = compressAggressive(messages);
+    const lastMsg = result.messages[0];
+    const text = extractTextContent(lastMsg.content);
+
+    assert.ok(text.includes(tailMarker), "Tail marker must be preserved in live user message");
+    assert.equal(text, longContent, "Live user message must remain verbatim");
+  });
+
+  it("compresses historical long user messages while preserving live user message", () => {
+    const oldTailMarker = "OLD-TAIL-MARKER-HISTORICAL-1122";
+    const liveTailMarker = "LIVE-TAIL-MARKER-CURRENT-3344";
+    const oldLongContent =
+      "Historical prompt:\n" + "const oldCode = 2;\n".repeat(1200) + `\n${oldTailMarker}`;
+    const liveLongContent =
+      "Current live instruction:\n" + "const liveCode = 3;\n".repeat(1200) + `\n${liveTailMarker}`;
+
+    const messages = [
+      { role: "user", content: oldLongContent },
+      { role: "assistant", content: "Understood, I am ready for the next instruction." },
+      { role: "user", content: liveLongContent },
+    ];
+
+    const result = compressAggressive(messages);
+    assert.equal(result.messages.length, 3);
+
+    const oldMsgText = extractTextContent(result.messages[0].content);
+    const assistantMsgText = extractTextContent(result.messages[1].content);
+    const liveMsgText = extractTextContent(result.messages[2].content);
+
+    // Old message should be summarized
+    assert.ok(oldMsgText.startsWith("[COMPRESSED:"), "Old message should be compressed");
+    assert.ok(
+      oldMsgText.length < oldLongContent.length,
+      "Old message should be significantly shortened"
+    );
+
+    // Assistant message preserved
+    assert.equal(assistantMsgText, "Understood, I am ready for the next instruction.");
+
+    // Live message must remain intact
+    assert.ok(liveMsgText.includes(liveTailMarker), "Live message tail marker must survive");
+    assert.equal(liveMsgText, liveLongContent, "Live message must remain verbatim");
+  });
+
+  it("F1: Step 2 applyAging does not compress the last user message even when aging threshold triggers", () => {
+    const livePrompt = "Live user command: deploy to staging immediately and verify health.";
+    const messages = [
+      { role: "user", content: "Historical step 1: initial setup" },
+      { role: "assistant", content: "Step 1 completed successfully." },
+      { role: "user", content: "Historical step 2: database migrations" },
+      { role: "assistant", content: "Step 2 migrations applied." },
+      { role: "user", content: "Historical step 3: seed test data" },
+      { role: "assistant", content: "Step 3 seed finished." },
+      { role: "user", content: livePrompt },
+      { role: "assistant", content: "Acknowledged, preparing to deploy." },
+      { role: "assistant", content: "Checking cluster health." },
+      { role: "assistant", content: "Waiting for approval." },
+    ];
+
+    // With 10 messages, live user message is at index 6 (distanceFromEnd = 3).
+    // In standard aging, distanceFromEnd 3 triggers moderate tier (caveman).
+    // The last user message must be spared from aging.
+    const result = compressAggressive(messages, {
+      thresholds: { fullSummary: 5, moderate: 3, light: 2, verbatim: 1 },
+    });
+
+    const liveUserMsg = result.messages[6];
+    const text = extractTextContent(liveUserMsg.content);
+    assert.equal(text, livePrompt, "Last user message must not be touched by applyAging");
+    assert.ok(!text.startsWith("[COMPRESSED:aging:"), "Last user message must not have aging marker");
+  });
+
+  it("F2: Step 4 caveman fallback does not compress the last user message", () => {
+    const livePrompt =
+      "Please urgently check if the server is running on the default port 8080 and report back.";
+    const messages = [
+      { role: "user", content: "Earlier question about logs." },
+      { role: "assistant", content: "Earlier answer about logs." },
+      { role: "user", content: livePrompt },
+    ];
+
+    // Disable summarizer to trigger Step 4 fallback path with high minSavingsThreshold
+    const result = compressAggressive(messages, {
+      summarizerEnabled: false,
+      minSavingsThreshold: 0.99,
+    });
+
+    const liveUserMsg = result.messages[2];
+    const text = extractTextContent(liveUserMsg.content);
+    assert.equal(text, livePrompt, "Last user message must remain verbatim despite caveman fallback");
+  });
+
+  it("F2: Step 4 lite fallback does not compress the last user message", () => {
+    const livePrompt =
+      "Please    verify    the    whitespace    formatting    in    the    target    output.";
+    const messages = [
+      { role: "user", content: "Old setup prompt." },
+      { role: "assistant", content: "Old setup response." },
+      { role: "user", content: livePrompt },
+    ];
+
+    const result = compressAggressive(messages, {
+      summarizerEnabled: false,
+      minSavingsThreshold: 0.99,
+    });
+
+    const liveUserMsg = result.messages[2];
+    const text = extractTextContent(liveUserMsg.content);
+    assert.equal(text, livePrompt, "Last user message must keep verbatim whitespace in fallback");
+  });
+
+  it("F3: does not duplicate [COMPRESSED:summary] marker when mid-string markers or repeated summaries occur", () => {
+    const oldLongContent =
+      "Historical log analysis containing [COMPRESSED:summary] in text:\n" +
+      "function analyze() { return 42; }\n".repeat(1200);
+
+    const messages = [
+      { role: "user", content: oldLongContent },
+      { role: "assistant", content: "Done." },
+      { role: "user", content: "Short follow-up" },
+    ];
+
+    const result = compressAggressive(messages);
+    const oldMsgText = extractTextContent(result.messages[0].content);
+
+    assert.ok(oldMsgText.startsWith("[COMPRESSED:summary]"), "Should start with compressed marker");
+    assert.equal(
+      oldMsgText.startsWith("[COMPRESSED:summary] [COMPRESSED:summary]"),
+      false,
+      "Must not contain doubled marker prefix"
+    );
+
+    // F3: Ensure count of leading markers is exactly 1 (no mid-string duplication / corrupt prefix)
+    const markerMatch = oldMsgText.match(/^\[COMPRESSED:summary\]\s+/g);
+    assert.ok(markerMatch && markerMatch.length === 1, "Exactly one leading marker prefix expected");
+  });
+});