diff --git a/open-sse/services/compression/engines/rtk/codeStripper.ts b/open-sse/services/compression/engines/rtk/codeStripper.ts index 05b63c5e45..655c7d6ac3 100644 --- a/open-sse/services/compression/engines/rtk/codeStripper.ts +++ b/open-sse/services/compression/engines/rtk/codeStripper.ts @@ -59,7 +59,7 @@ export function detectCodeLanguage(text: string): CodeLanguage { * division without parser context). Bails out entirely when JSX is present so * JSX expression-container comments are never corrupted. */ -function stripJsTsComments(text: string): string { +function stripJsTsComments(text: string, preserveDocstrings: boolean): string { const source = ts.createSourceFile( "snippet.tsx", text, @@ -94,6 +94,9 @@ function stripJsTsComments(text: string): string { if (ranges.size === 0) return text; let result = text; for (const range of [...ranges.values()].sort((a, b) => b.pos - a.pos)) { + // Keep JSDoc/docstring block comments (`/** ... */`) when preserveDocstrings is on — they + // carry API documentation that is worth more than the bytes they cost. + if (preserveDocstrings && text.startsWith("/**", range.pos)) continue; result = result.slice(0, range.pos) + result.slice(range.end); } return result; @@ -125,7 +128,7 @@ export function stripCode( opts.removeComments && (resolvedLanguage === "javascript" || resolvedLanguage === "typescript") ) { - result = stripJsTsComments(result); + result = stripJsTsComments(result, opts.preserveDocstrings); } if (opts.removeEmptyLines) result = result.replace(/^\s*$(?:\r?\n)?/gm, ""); diff --git a/open-sse/services/compression/engines/rtk/index.ts b/open-sse/services/compression/engines/rtk/index.ts index 8114f021eb..917dab77aa 100644 --- a/open-sse/services/compression/engines/rtk/index.ts +++ b/open-sse/services/compression/engines/rtk/index.ts @@ -306,7 +306,12 @@ export function processRtkText( result = result.replace( /```([A-Za-z0-9_+.-]*)\r?\n([\s\S]*?)```/g, (match, languageHint: string, code: string) => { - const stripped = stripCode(code, normalizeCodeLanguage(languageHint)); + const stripped = stripCode(code, normalizeCodeLanguage(languageHint), { + // Opt-in comment removal (default off = no silent production change). Docstrings/JSDoc + // are preserved unless explicitly disabled. + removeComments: config.stripCodeComments === true, + preserveDocstrings: config.preserveDocstrings !== false, + }); if (stripped.strippedLines <= 0 && stripped.text === code.trim()) return match; strippedCodeBlocks++; const fenceLanguage = languageHint?.trim() || stripped.language; diff --git a/open-sse/services/compression/types.ts b/open-sse/services/compression/types.ts index 06af2d192a..94db8751d0 100644 --- a/open-sse/services/compression/types.ts +++ b/open-sse/services/compression/types.ts @@ -79,6 +79,10 @@ export interface RtkConfig { enableGrouping?: boolean; /** R5: minimum consecutive similar-line run to trigger grouping. Default: 3. */ groupingThreshold?: number; + /** R1/N3: remove comments from fenced code blocks when stripping code. Default: false. */ + stripCodeComments?: boolean; + /** R1/N3: keep JSDoc/docstring block comments when removing comments. Default: true. */ + preserveDocstrings?: boolean; } export interface CompressionLanguageConfig { @@ -232,6 +236,8 @@ export const DEFAULT_RTK_CONFIG: RtkConfig = { rawOutputMaxBytes: 1_048_576, enableGrouping: false, groupingThreshold: 3, + stripCodeComments: false, + preserveDocstrings: true, }; export const DEFAULT_COMPRESSION_LANGUAGE_CONFIG: CompressionLanguageConfig = { diff --git a/src/lib/db/compression.ts b/src/lib/db/compression.ts index c87e03b326..5f21064fcc 100644 --- a/src/lib/db/compression.ts +++ b/src/lib/db/compression.ts @@ -182,6 +182,14 @@ function normalizeRtkConfig(value: unknown): RtkConfig { 2, 100 ), + stripCodeComments: + typeof record.stripCodeComments === "boolean" + ? record.stripCodeComments + : (DEFAULT_RTK_CONFIG.stripCodeComments ?? false), + preserveDocstrings: + typeof record.preserveDocstrings === "boolean" + ? record.preserveDocstrings + : (DEFAULT_RTK_CONFIG.preserveDocstrings ?? true), }; } diff --git a/src/shared/validation/compressionConfigSchemas.ts b/src/shared/validation/compressionConfigSchemas.ts index 5d865ce3cc..57f5741d08 100644 --- a/src/shared/validation/compressionConfigSchemas.ts +++ b/src/shared/validation/compressionConfigSchemas.ts @@ -51,6 +51,8 @@ export const rtkConfigSchema = z rawOutputMaxBytes: z.number().int().min(1024).max(10_000_000).optional(), enableGrouping: z.boolean().optional(), groupingThreshold: z.number().int().min(2).max(100).optional(), + stripCodeComments: z.boolean().optional(), + preserveDocstrings: z.boolean().optional(), }) .strict(); diff --git a/tests/unit/compression/rtk-strip-comments.test.ts b/tests/unit/compression/rtk-strip-comments.test.ts new file mode 100644 index 0000000000..f9b70963ab --- /dev/null +++ b/tests/unit/compression/rtk-strip-comments.test.ts @@ -0,0 +1,136 @@ +import { describe, it, beforeEach, afterEach, after } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + applyRtkCompression, + stripCode, +} from "../../../open-sse/services/compression/index.ts"; +import { rtkConfigSchema } from "../../../src/shared/validation/compressionConfigSchemas.ts"; +import { DEFAULT_RTK_CONFIG } from "../../../open-sse/services/compression/types.ts"; + +// codeStripper has always supported removeComments + preserveDocstrings, but the feature was +// unreachable: the RTK engine called stripCode with no options (so comments were never removed +// through the runtime), and preserveDocstrings was folded into opts yet never honored by +// stripJsTsComments. This proves the new RTK config fields wire it end to end and that +// preserveDocstrings now keeps JSDoc. + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rtk-strip-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../../src/lib/db/core.ts"); +const { getCompressionSettings, updateCompressionSettings } = await import( + "../../../src/lib/db/compression.ts" +); + +describe("RTK strip-code-comments — stripCode behavior", () => { + it("removes line/block comments but keeps JSDoc when preserveDocstrings is on", () => { + const code = [ + "/** Adds two numbers. */", + "function add(a, b) {", + " // inline note", + " return a + b; /* trailing */", + "}", + ].join("\n"); + const out = stripCode(code, "typescript", { + removeComments: true, + preserveDocstrings: true, + removeEmptyLines: false, + collapseWhitespace: false, + }); + assert.ok(out.text.includes("/** Adds two numbers. */"), "JSDoc preserved"); + assert.ok(!out.text.includes("inline note"), "line comment removed"); + assert.ok(!out.text.includes("trailing"), "trailing block comment removed"); + }); + + it("removes JSDoc too when preserveDocstrings is off", () => { + const out = stripCode("/** doc */\nconst x = 1; // n", "typescript", { + removeComments: true, + preserveDocstrings: false, + removeEmptyLines: false, + collapseWhitespace: false, + }); + assert.ok(!out.text.includes("doc"), "JSDoc removed when not preserving"); + assert.ok(!out.text.includes("// n"), "line comment removed"); + }); +}); + +describe("RTK strip-code-comments — runtime reachability", () => { + it("strips fenced-block comments when applyToCodeBlocks + stripCodeComments are on", () => { + // RTK only processes tool/assistant messages (shouldCompressMessage); code-block stripping + // rides on applyToCodeBlocks for assistant content regardless of applyToAssistantMessages. + const body = { + messages: [ + { + role: "assistant", + content: "```ts\n// secret note\nconst x = 1;\n/* block secret */\nconst y = 2;\n```", + }, + ], + }; + const result = applyRtkCompression(body, { + config: { + ...DEFAULT_RTK_CONFIG, + enabled: true, + applyToCodeBlocks: true, + stripCodeComments: true, + }, + }); + const serialized = JSON.stringify(result.body.messages); + assert.ok(!serialized.includes("secret note"), "line comment stripped via runtime"); + assert.ok(!serialized.includes("block secret"), "block comment stripped via runtime"); + assert.match(serialized, /const x = 1/); + assert.match(serialized, /const y = 2/); + }); + + it("leaves fenced-block comments intact when stripCodeComments is off (default)", () => { + const body = { + messages: [{ role: "assistant", content: "```ts\n// keep me\nconst x = 1;\n```" }], + }; + const result = applyRtkCompression(body, { + config: { ...DEFAULT_RTK_CONFIG, enabled: true, applyToCodeBlocks: true }, + }); + assert.match(JSON.stringify(result.body.messages), /keep me/); + }); +}); + +describe("RTK strip-code-comments — config persistence", () => { + beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + }); + + afterEach(() => { + core.resetDbInstance(); + }); + + after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + }); + + it("accepts stripCodeComments / preserveDocstrings on the write schema", () => { + assert.equal( + rtkConfigSchema.safeParse({ stripCodeComments: true, preserveDocstrings: false }).success, + true + ); + }); + + it("preserves stripCodeComments / preserveDocstrings through a DB round-trip", async () => { + const settings = await updateCompressionSettings({ + rtkConfig: { ...DEFAULT_RTK_CONFIG, stripCodeComments: true, preserveDocstrings: false }, + }); + assert.equal(settings.rtkConfig.stripCodeComments, true); + assert.equal(settings.rtkConfig.preserveDocstrings, false); + + core.resetDbInstance(); + const reread = await getCompressionSettings(); + assert.equal(reread.rtkConfig.stripCodeComments, true); + assert.equal(reread.rtkConfig.preserveDocstrings, false); + }); +});