feat(compression): wire RTK comment-stripping config + honor preserveDocstrings (#4242)

Integrated into release/v3.8.29

open-sse/services/compression/engines/rtk/codeStripper.ts

@@ -59,7 +59,7 @@ export function detectCodeLanguage(text: string): CodeLanguage {
  * division without parser context). Bails out entirely when JSX is present so
  * JSX expression-container comments are never corrupted.
  */
-function stripJsTsComments(text: string): string {
+function stripJsTsComments(text: string, preserveDocstrings: boolean): string {
   const source = ts.createSourceFile(
     "snippet.tsx",
     text,
@@ -94,6 +94,9 @@ function stripJsTsComments(text: string): string {
   if (ranges.size === 0) return text;
   let result = text;
   for (const range of [...ranges.values()].sort((a, b) => b.pos - a.pos)) {
+    // Keep JSDoc/docstring block comments (`/** ... */`) when preserveDocstrings is on — they
+    // carry API documentation that is worth more than the bytes they cost.
+    if (preserveDocstrings && text.startsWith("/**", range.pos)) continue;
     result = result.slice(0, range.pos) + result.slice(range.end);
   }
   return result;
@@ -125,7 +128,7 @@ export function stripCode(
     opts.removeComments &&
     (resolvedLanguage === "javascript" || resolvedLanguage === "typescript")
   ) {
-    result = stripJsTsComments(result);
+    result = stripJsTsComments(result, opts.preserveDocstrings);
   }
 
   if (opts.removeEmptyLines) result = result.replace(/^\s*$(?:\r?\n)?/gm, "");

open-sse/services/compression/engines/rtk/index.ts

@@ -306,7 +306,12 @@ export function processRtkText(
     result = result.replace(
       /```([A-Za-z0-9_+.-]*)\r?\n([\s\S]*?)```/g,
       (match, languageHint: string, code: string) => {
-        const stripped = stripCode(code, normalizeCodeLanguage(languageHint));
+        const stripped = stripCode(code, normalizeCodeLanguage(languageHint), {
+          // Opt-in comment removal (default off = no silent production change). Docstrings/JSDoc
+          // are preserved unless explicitly disabled.
+          removeComments: config.stripCodeComments === true,
+          preserveDocstrings: config.preserveDocstrings !== false,
+        });
         if (stripped.strippedLines <= 0 && stripped.text === code.trim()) return match;
         strippedCodeBlocks++;
         const fenceLanguage = languageHint?.trim() || stripped.language;

open-sse/services/compression/types.ts

@@ -79,6 +79,10 @@ export interface RtkConfig {
   enableGrouping?: boolean;
   /** R5: minimum consecutive similar-line run to trigger grouping. Default: 3. */
   groupingThreshold?: number;
+  /** R1/N3: remove comments from fenced code blocks when stripping code. Default: false. */
+  stripCodeComments?: boolean;
+  /** R1/N3: keep JSDoc/docstring block comments when removing comments. Default: true. */
+  preserveDocstrings?: boolean;
 }
 
 export interface CompressionLanguageConfig {
@@ -232,6 +236,8 @@ export const DEFAULT_RTK_CONFIG: RtkConfig = {
   rawOutputMaxBytes: 1_048_576,
   enableGrouping: false,
   groupingThreshold: 3,
+  stripCodeComments: false,
+  preserveDocstrings: true,
 };
 
 export const DEFAULT_COMPRESSION_LANGUAGE_CONFIG: CompressionLanguageConfig = {

src/lib/db/compression.ts

@@ -182,6 +182,14 @@ function normalizeRtkConfig(value: unknown): RtkConfig {
       2,
       100
     ),
+    stripCodeComments:
+      typeof record.stripCodeComments === "boolean"
+        ? record.stripCodeComments
+        : (DEFAULT_RTK_CONFIG.stripCodeComments ?? false),
+    preserveDocstrings:
+      typeof record.preserveDocstrings === "boolean"
+        ? record.preserveDocstrings
+        : (DEFAULT_RTK_CONFIG.preserveDocstrings ?? true),
   };
 }
 

src/shared/validation/compressionConfigSchemas.ts

@@ -51,6 +51,8 @@ export const rtkConfigSchema = z
     rawOutputMaxBytes: z.number().int().min(1024).max(10_000_000).optional(),
     enableGrouping: z.boolean().optional(),
     groupingThreshold: z.number().int().min(2).max(100).optional(),
+    stripCodeComments: z.boolean().optional(),
+    preserveDocstrings: z.boolean().optional(),
   })
   .strict();
 

tests/unit/compression/rtk-strip-comments.test.ts

@@ -0,0 +1,136 @@
+import { describe, it, beforeEach, afterEach, after } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  applyRtkCompression,
+  stripCode,
+} from "../../../open-sse/services/compression/index.ts";
+import { rtkConfigSchema } from "../../../src/shared/validation/compressionConfigSchemas.ts";
+import { DEFAULT_RTK_CONFIG } from "../../../open-sse/services/compression/types.ts";
+
+// codeStripper has always supported removeComments + preserveDocstrings, but the feature was
+// unreachable: the RTK engine called stripCode with no options (so comments were never removed
+// through the runtime), and preserveDocstrings was folded into opts yet never honored by
+// stripJsTsComments. This proves the new RTK config fields wire it end to end and that
+// preserveDocstrings now keeps JSDoc.
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rtk-strip-"));
+const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../../src/lib/db/core.ts");
+const { getCompressionSettings, updateCompressionSettings } = await import(
+  "../../../src/lib/db/compression.ts"
+);
+
+describe("RTK strip-code-comments — stripCode behavior", () => {
+  it("removes line/block comments but keeps JSDoc when preserveDocstrings is on", () => {
+    const code = [
+      "/** Adds two numbers. */",
+      "function add(a, b) {",
+      "  // inline note",
+      "  return a + b; /* trailing */",
+      "}",
+    ].join("\n");
+    const out = stripCode(code, "typescript", {
+      removeComments: true,
+      preserveDocstrings: true,
+      removeEmptyLines: false,
+      collapseWhitespace: false,
+    });
+    assert.ok(out.text.includes("/** Adds two numbers. */"), "JSDoc preserved");
+    assert.ok(!out.text.includes("inline note"), "line comment removed");
+    assert.ok(!out.text.includes("trailing"), "trailing block comment removed");
+  });
+
+  it("removes JSDoc too when preserveDocstrings is off", () => {
+    const out = stripCode("/** doc */\nconst x = 1; // n", "typescript", {
+      removeComments: true,
+      preserveDocstrings: false,
+      removeEmptyLines: false,
+      collapseWhitespace: false,
+    });
+    assert.ok(!out.text.includes("doc"), "JSDoc removed when not preserving");
+    assert.ok(!out.text.includes("// n"), "line comment removed");
+  });
+});
+
+describe("RTK strip-code-comments — runtime reachability", () => {
+  it("strips fenced-block comments when applyToCodeBlocks + stripCodeComments are on", () => {
+    // RTK only processes tool/assistant messages (shouldCompressMessage); code-block stripping
+    // rides on applyToCodeBlocks for assistant content regardless of applyToAssistantMessages.
+    const body = {
+      messages: [
+        {
+          role: "assistant",
+          content: "```ts\n// secret note\nconst x = 1;\n/* block secret */\nconst y = 2;\n```",
+        },
+      ],
+    };
+    const result = applyRtkCompression(body, {
+      config: {
+        ...DEFAULT_RTK_CONFIG,
+        enabled: true,
+        applyToCodeBlocks: true,
+        stripCodeComments: true,
+      },
+    });
+    const serialized = JSON.stringify(result.body.messages);
+    assert.ok(!serialized.includes("secret note"), "line comment stripped via runtime");
+    assert.ok(!serialized.includes("block secret"), "block comment stripped via runtime");
+    assert.match(serialized, /const x = 1/);
+    assert.match(serialized, /const y = 2/);
+  });
+
+  it("leaves fenced-block comments intact when stripCodeComments is off (default)", () => {
+    const body = {
+      messages: [{ role: "assistant", content: "```ts\n// keep me\nconst x = 1;\n```" }],
+    };
+    const result = applyRtkCompression(body, {
+      config: { ...DEFAULT_RTK_CONFIG, enabled: true, applyToCodeBlocks: true },
+    });
+    assert.match(JSON.stringify(result.body.messages), /keep me/);
+  });
+});
+
+describe("RTK strip-code-comments — config persistence", () => {
+  beforeEach(() => {
+    core.resetDbInstance();
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+  });
+
+  afterEach(() => {
+    core.resetDbInstance();
+  });
+
+  after(() => {
+    core.resetDbInstance();
+    fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+    if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
+    else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
+  });
+
+  it("accepts stripCodeComments / preserveDocstrings on the write schema", () => {
+    assert.equal(
+      rtkConfigSchema.safeParse({ stripCodeComments: true, preserveDocstrings: false }).success,
+      true
+    );
+  });
+
+  it("preserves stripCodeComments / preserveDocstrings through a DB round-trip", async () => {
+    const settings = await updateCompressionSettings({
+      rtkConfig: { ...DEFAULT_RTK_CONFIG, stripCodeComments: true, preserveDocstrings: false },
+    });
+    assert.equal(settings.rtkConfig.stripCodeComments, true);
+    assert.equal(settings.rtkConfig.preserveDocstrings, false);
+
+    core.resetDbInstance();
+    const reread = await getCompressionSettings();
+    assert.equal(reread.rtkConfig.stripCodeComments, true);
+    assert.equal(reread.rtkConfig.preserveDocstrings, false);
+  });
+});