From 831829dbc345ce5abd900e695a42da779bcb0f7b Mon Sep 17 00:00:00 2001 From: Raxxoor Date: Fri, 8 May 2026 21:35:43 +0100 Subject: [PATCH] fix(compression): support Responses input and expand Spanish rules (#2028) Integrated into release/v3.8.0 --- open-sse/handlers/chatCore.ts | 15 +- open-sse/services/compression/bodyAdapter.ts | 160 +++++++++++++++ .../compression/engines/cavemanAdapter.ts | 30 ++- .../compression/engines/rtk/codeStripper.ts | 38 +--- .../services/compression/engines/rtk/index.ts | 86 +++++++- open-sse/services/compression/outputMode.ts | 6 +- .../compression/rules/es/context.json | 49 +++++ .../services/compression/rules/es/dedup.json | 38 ++++ .../services/compression/rules/es/filler.json | 40 ++++ .../compression/rules/es/structural.json | 75 +++++++ .../services/compression/rules/es/ultra.json | 107 ++++++++++ .../services/compression/strategySelector.ts | 41 ++-- src/lib/db/compressionAnalytics.ts | 19 +- tests/unit/compression/body-adapter.test.ts | 189 ++++++++++++++++++ .../compression/compressionAnalytics.test.ts | 31 +++ tests/unit/compression/language-packs.test.ts | 22 ++ tests/unit/compression/outputMode.test.ts | 11 + .../compression/rtk-code-stripper.test.ts | 89 +++++++-- 18 files changed, 958 insertions(+), 88 deletions(-) create mode 100644 open-sse/services/compression/bodyAdapter.ts create mode 100644 open-sse/services/compression/rules/es/dedup.json create mode 100644 open-sse/services/compression/rules/es/ultra.json create mode 100644 tests/unit/compression/body-adapter.test.ts diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 5b53373d74..8cc02b794a 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -44,6 +44,7 @@ import { isDetailedLoggingEnabled } from "@/lib/db/detailedLogs"; import { getCallLogPipelineCaptureStreamChunks } from "@/lib/logEnv"; import { logAuditEvent } from "@/lib/compliance"; import { extractProviderWarnings } from "@/lib/compliance/providerAudit"; +import { adaptBodyForCompression } from "../services/compression/bodyAdapter.ts"; import { handleBypassRequest } from "../utils/bypassHandler.ts"; import { saveRequestUsage, @@ -1614,8 +1615,9 @@ export async function handleChatCore({ // ── Proactive Context Compression (Phase 4) ── // Check if context exceeds 70% of limit and compress proactively before sending to provider. // This prevents "prompt too long" errors for large-but-not-full contexts. + const compressionBody = body ? adaptBodyForCompression(body as Record).body : null; const allMessages = - body?.messages || body?.input || body?.contents || body?.request?.contents || []; + compressionBody?.messages || body?.contents || body?.request?.contents || []; let cavemanOutputModeApplied = false; let cavemanOutputModeIntensity: string | null = null; if (body && Array.isArray(allMessages) && allMessages.length > 0) { @@ -1898,6 +1900,7 @@ export async function handleChatCore({ 0, result.stats.originalTokens - result.stats.compressedTokens ); + const rtkPointers = result.stats.rtkRawOutputPointers ?? []; const estimatedUsdSaved = await calculateCost( provider ?? "", effectiveModel ?? "", @@ -1921,8 +1924,14 @@ export async function handleChatCore({ estimated_usd_saved: estimatedUsdSaved || null, validation_fallback: result.stats.fallbackApplied ? 1 : 0, output_mode: cavemanOutputModeApplied ? cavemanOutputModeIntensity : null, - rtk_raw_output_pointer: result.stats.rtkRawOutputPointers?.[0]?.id ?? null, - rtk_raw_output_bytes: result.stats.rtkRawOutputPointers?.[0]?.bytes ?? null, + rtk_raw_output_pointer: rtkPointers[0]?.id ?? null, + rtk_raw_output_bytes: rtkPointers[0]?.bytes ?? null, + rtk_raw_output_pointers: rtkPointers.length + ? JSON.stringify(rtkPointers.map((pointer) => pointer.id)) + : null, + rtk_raw_output_total_bytes: rtkPointers.length + ? rtkPointers.reduce((total, pointer) => total + pointer.bytes, 0) + : null, }); } catch (err) { log?.debug?.( diff --git a/open-sse/services/compression/bodyAdapter.ts b/open-sse/services/compression/bodyAdapter.ts new file mode 100644 index 0000000000..2a65c21a94 --- /dev/null +++ b/open-sse/services/compression/bodyAdapter.ts @@ -0,0 +1,160 @@ +type MessageLike = { + role?: unknown; + content?: unknown; + [COMPRESSION_INPUT_INDEX]?: number; + [key: string]: unknown; +}; + +type ResponsesItem = { + type?: unknown; + role?: unknown; + content?: unknown; + output?: unknown; + [key: string]: unknown; +}; + +const RESPONSES_MESSAGE_TYPES = new Set(["message", "function_call_output"]); +const COMPRESSION_INPUT_INDEX = Symbol("compressionInputIndex"); + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function normalizeRole(role: unknown, fallback: string): string { + return typeof role === "string" && role.length > 0 ? role : fallback; +} + +function toChatContent(content: unknown, fallbackOutput?: unknown): unknown { + return content === undefined ? fallbackOutput : content; +} + +function fromChatContent(nextContent: unknown, originalContent: unknown): unknown { + if (Array.isArray(originalContent) && typeof nextContent === "string") { + let replaced = false; + const mapped = originalContent.map((part) => { + if (!isRecord(part) || typeof part.text !== "string") return part; + if (replaced) return { ...part, text: "" }; + replaced = true; + return { ...part, text: nextContent }; + }); + return replaced ? mapped : originalContent; + } + return nextContent; +} + +function responsesItemToMessage(item: ResponsesItem): MessageLike | null { + const type = typeof item.type === "string" ? item.type : "message"; + if (!RESPONSES_MESSAGE_TYPES.has(type)) return null; + + if (type === "function_call_output") { + return { + role: "tool", + content: toChatContent(item.output ?? item.content), + }; + } + + return { + role: normalizeRole(item.role, "user"), + content: toChatContent(item.content, item.output), + }; +} + +function messageToResponsesItem(message: MessageLike, originalItem: ResponsesItem): ResponsesItem { + const type = typeof originalItem.type === "string" ? originalItem.type : "message"; + if (type === "function_call_output") { + return { + ...originalItem, + output: fromChatContent(message.content, originalItem.output), + }; + } + + return { + ...originalItem, + content: fromChatContent(message.content, originalItem.content), + }; +} + +function hasTextContent(message: MessageLike): boolean { + if (typeof message.content === "string") return message.content.length > 0; + if (!Array.isArray(message.content)) return false; + return message.content.some( + (part) => isRecord(part) && typeof part.text === "string" && part.text.length > 0 + ); +} + +export type CompressionBodyAdapter = { + body: Record; + adapted: boolean; + restore(compressedBody: Record): Record; +}; + +export function adaptBodyForCompression(body: Record): CompressionBodyAdapter { + if (Array.isArray(body.messages)) { + return { + body, + adapted: false, + restore: (compressedBody) => compressedBody, + }; + } + + if (!Array.isArray(body.input) && typeof body.input !== "string") { + return { + body, + adapted: false, + restore: (compressedBody) => compressedBody, + }; + } + + const inputItems = Array.isArray(body.input) + ? body.input + : [{ type: "message", role: "user", content: body.input }]; + const mappings: Array<{ index: number; item: ResponsesItem }> = []; + const messages: MessageLike[] = []; + + inputItems.forEach((item, index) => { + if (!isRecord(item)) return; + const message = responsesItemToMessage(item); + if (!message || !hasTextContent(message)) return; + mappings.push({ index, item: item as ResponsesItem }); + messages.push({ ...message, [COMPRESSION_INPUT_INDEX]: index }); + }); + + if (messages.length === 0) { + return { + body, + adapted: false, + restore: (compressedBody) => compressedBody, + }; + } + + const bodyWithoutInput = { ...body }; + delete bodyWithoutInput.input; + + return { + body: { ...bodyWithoutInput, messages }, + adapted: true, + restore(compressedBody) { + const compressedMessagesByIndex = new Map(); + if (Array.isArray(compressedBody.messages)) { + for (const message of compressedBody.messages as MessageLike[]) { + if (typeof message[COMPRESSION_INPUT_INDEX] === "number") { + compressedMessagesByIndex.set(message[COMPRESSION_INPUT_INDEX], message); + } + } + } + const nextInput = [...inputItems]; + mappings.forEach((mapping) => { + const compressedMessage = compressedMessagesByIndex.get(mapping.index); + if (!compressedMessage) return; + nextInput[mapping.index] = messageToResponsesItem(compressedMessage, mapping.item); + }); + const rest = { ...compressedBody }; + delete rest.messages; + if (typeof body.input === "string") { + const first = nextInput[0]; + return { ...rest, input: isRecord(first) ? (first.content ?? body.input) : body.input }; + } + return { ...rest, input: nextInput }; + }, + }; +} diff --git a/open-sse/services/compression/engines/cavemanAdapter.ts b/open-sse/services/compression/engines/cavemanAdapter.ts index 51ae2b6548..6bab174e56 100644 --- a/open-sse/services/compression/engines/cavemanAdapter.ts +++ b/open-sse/services/compression/engines/cavemanAdapter.ts @@ -3,6 +3,7 @@ import { cavemanCompress } from "../caveman.ts"; import { compressAggressive } from "../aggressive.ts"; import { ultraCompress } from "../ultra.ts"; import { createCompressionStats } from "../stats.ts"; +import { adaptBodyForCompression } from "../bodyAdapter.ts"; import { DEFAULT_AGGRESSIVE_CONFIG, DEFAULT_ULTRA_CONFIG, @@ -228,10 +229,12 @@ export const liteEngine: CompressionEngine = { stable: true, }, apply(body, options) { - return applyLiteCompression(body, { + const adapter = adaptBodyForCompression(body); + const result = applyLiteCompression(adapter.body, { ...options, preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, }); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; }, compress(body, config) { return this.apply(body, { stepConfig: config }); @@ -262,6 +265,7 @@ export const cavemanEngine: CompressionEngine = { stable: true, }, apply(body, options) { + const adapter = adaptBodyForCompression(body); const cavemanConfig = { ...(options?.config?.cavemanConfig ?? {}), ...(options?.stepConfig ?? {}), @@ -280,7 +284,11 @@ export const cavemanEngine: CompressionEngine = { } : {}), }; - return cavemanCompress(body as Parameters[0], cavemanConfig); + const result = cavemanCompress( + adapter.body as Parameters[0], + cavemanConfig + ); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; }, compress(body, config) { return this.apply(body, { stepConfig: config }); @@ -311,7 +319,8 @@ export const aggressiveEngine: CompressionEngine = { stable: true, }, apply(body, options) { - const messages = (body.messages ?? []) as Array<{ + const adapter = adaptBodyForCompression(body); + const messages = (adapter.body.messages ?? []) as Array<{ role: string; content?: string | Array<{ type: string; text?: string }>; [key: string]: unknown; @@ -325,12 +334,12 @@ export const aggressiveEngine: CompressionEngine = { preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, }; const result = compressAggressive(messages, aggressiveConfig); - const compressedBody = { ...body, messages: result.messages }; + const compressedBody = { ...adapter.body, messages: result.messages }; return { - body: compressedBody, + body: adapter.restore(compressedBody), compressed: result.stats.savingsPercent > 0, stats: createCompressionStats( - body, + adapter.body, compressedBody, "aggressive", ["aggressive"], @@ -368,7 +377,8 @@ export const ultraEngine: CompressionEngine = { stable: true, }, apply(body, options) { - const messages = (body.messages ?? []) as Array<{ + const adapter = adaptBodyForCompression(body); + const messages = (adapter.body.messages ?? []) as Array<{ role: string; content?: string | unknown[]; [key: string]: unknown; @@ -382,12 +392,12 @@ export const ultraEngine: CompressionEngine = { preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, }; const result = ultraCompress(messages, ultraConfig); - const compressedBody = { ...body, messages: result.messages }; + const compressedBody = { ...adapter.body, messages: result.messages }; return { - body: compressedBody, + body: adapter.restore(compressedBody), compressed: result.stats.savingsPercent > 0, stats: createCompressionStats( - body, + adapter.body, compressedBody, "ultra", ["ultra"], diff --git a/open-sse/services/compression/engines/rtk/codeStripper.ts b/open-sse/services/compression/engines/rtk/codeStripper.ts index 27c3512910..360b5bdd32 100644 --- a/open-sse/services/compression/engines/rtk/codeStripper.ts +++ b/open-sse/services/compression/engines/rtk/codeStripper.ts @@ -50,40 +50,6 @@ export function detectCodeLanguage(text: string): CodeLanguage { return "unknown"; } -function removeSlashComments(text: string): string { - return text - .replace(/\/\*[\s\S]*?\*\//g, "") - .replace(/^\s*\/\/.*$/gm, "") - .replace(/\s+\/\/\s.*$/gm, ""); -} - -function removeHashComments(text: string): string { - return text.replace(/^\s*#.*$/gm, "").replace(/\s+#\s.*$/gm, ""); -} - -function stripByLanguage( - text: string, - language: CodeLanguage, - options: Required -): string { - if (!options.removeComments) return text; - - if (language === "javascript" || language === "typescript" || language === "rust") { - return removeSlashComments(text); - } - if (language === "go" || language === "java") return removeSlashComments(text); - if (language === "python") { - const withoutDocstrings = options.preserveDocstrings - ? text - : text.replace(/("""[\s\S]*?"""|'''[\s\S]*?''')/g, ""); - return removeHashComments(withoutDocstrings); - } - if (language === "ruby") { - return removeHashComments(text.replace(/^=begin[\s\S]*?^=end/gm, "")); - } - return text; -} - export function stripCode( text: string, language: CodeLanguage = "unknown", @@ -101,7 +67,7 @@ export function stripCode( preserveDocstrings: options.preserveDocstrings === true, }; const originalLines = text.split(/\r?\n/).length; - let result = stripByLanguage(text, resolvedLanguage, opts); + let result = text; if (opts.removeEmptyLines) result = result.replace(/^\s*$(?:\r?\n)?/gm, ""); if (opts.collapseWhitespace) { @@ -111,7 +77,7 @@ export function stripCode( .join("\n"); } - result = result.trim(); + result = result.replace(/^\s*\n/, "").replace(/\n\s*$/, ""); const strippedLines = Math.max(0, originalLines - (result ? result.split(/\r?\n/).length : 0)); return { text: result, strippedLines, language: resolvedLanguage }; } diff --git a/open-sse/services/compression/engines/rtk/index.ts b/open-sse/services/compression/engines/rtk/index.ts index 70d6efc529..1b851bc69f 100644 --- a/open-sse/services/compression/engines/rtk/index.ts +++ b/open-sse/services/compression/engines/rtk/index.ts @@ -9,6 +9,7 @@ import { smartTruncate } from "./smartTruncate.ts"; import { normalizeCodeLanguage, stripCode } from "./codeStripper.ts"; import { maybePersistRtkRawOutput, type RtkRawOutputPointer } from "./rawOutput.ts"; import { isTextBlock } from "../../messageContent.ts"; +import { adaptBodyForCompression } from "../../bodyAdapter.ts"; type Message = { role: string; @@ -186,12 +187,81 @@ function mergeRtkConfig(base?: Partial, override?: Record isTextBlock(part) && typeof part.text === "string" && /```/.test(part.text)); +} + +function codeOnlyConfig(config: RtkConfig): boolean { + return config.applyToCodeBlocks && !config.applyToToolResults && !config.applyToAssistantMessages; +} + +function processRtkCodeBlocksOnly( + content: Message["content"], + config: RtkConfig +): { + content: Message["content"]; + compressed: boolean; + techniquesUsed: string[]; + rulesApplied: string[]; + rawOutputPointers: RtkRawOutputPointer[]; +} { + const techniquesUsed: string[] = []; + const rulesApplied: string[] = []; + const rawOutputPointers: RtkRawOutputPointer[] = []; + const processText = (text: string) => { + let compressed = false; + const nextText = text.replace(/```([\s\S]*?)```/g, (match) => { + const processed = processRtkText(match, { config }); + techniquesUsed.push(...processed.techniquesUsed); + rulesApplied.push(...processed.rulesApplied); + if (processed.rawOutputPointers) rawOutputPointers.push(...processed.rawOutputPointers); + if (!processed.compressed) return match; + compressed = true; + return processed.text; + }); + return { text: compressed ? nextText : text, compressed }; + }; + + if (typeof content === "string") { + const processed = processText(content); + return { + content: processed.text, + compressed: processed.compressed, + techniquesUsed, + rulesApplied, + rawOutputPointers, + }; + } + if (!Array.isArray(content)) { + return { content, compressed: false, techniquesUsed, rulesApplied, rawOutputPointers }; + } + let compressed = false; + const nextContent = content.map((part) => { + if (!isTextBlock(part) || !part.text) return part; + const processed = processText(part.text); + if (!processed.compressed) return part; + compressed = true; + return { ...part, text: processed.text }; + }); + return { + content: compressed ? nextContent : content, + compressed, + techniquesUsed, + rulesApplied, + rawOutputPointers, + }; +} + export function processRtkText( text: string, options: { command?: string | null; config?: Partial } = {} @@ -296,6 +366,9 @@ function processRtkContent( rulesApplied: string[]; rawOutputPointers: RtkRawOutputPointer[]; } { + if (codeOnlyConfig(config)) { + return processRtkCodeBlocksOnly(content, config); + } const techniquesUsed: string[] = []; const rulesApplied: string[] = []; const rawOutputPointers: RtkRawOutputPointer[] = []; @@ -352,7 +425,8 @@ export function applyRtkCompression( const config = mergeRtkConfig(options.config, options.stepConfig); if (!config.enabled) return { body, compressed: false, stats: null }; - const messages = body.messages as Message[] | undefined; + const adapter = adaptBodyForCompression(body); + const messages = adapter.body.messages as Message[] | undefined; if (!Array.isArray(messages) || messages.length === 0) { return { body, compressed: false, stats: null }; } @@ -373,9 +447,9 @@ export function applyRtkCompression( }; }); - const compressedBody = { ...body, messages: compressedMessages }; + const compressedBody = { ...adapter.body, messages: compressedMessages }; const stats = createCompressionStats( - body, + adapter.body, compressedBody, "rtk", [...new Set(allTechniques)], @@ -387,7 +461,7 @@ export function applyRtkCompression( stats.rtkRawOutputPointers = rawOutputPointers; } return { - body: compressedBody, + body: adapter.restore(compressedBody), compressed: stats.compressedTokens < stats.originalTokens, stats, }; diff --git a/open-sse/services/compression/outputMode.ts b/open-sse/services/compression/outputMode.ts index 9f4e7bfe45..10b4ae1d47 100644 --- a/open-sse/services/compression/outputMode.ts +++ b/open-sse/services/compression/outputMode.ts @@ -116,6 +116,7 @@ export function applyCavemanOutputMode( const messages = Array.isArray(body.messages) ? body.messages : null; if (!messages || messages.length === 0) { + const instruction = buildCavemanOutputInstruction(config, language); if (typeof body.instructions === "string") { if (body.instructions.includes(CAVEMAN_OUTPUT_MARKER)) { return { body, applied: false, skippedReason: "already_applied" }; @@ -123,11 +124,14 @@ export function applyCavemanOutputMode( return { body: { ...body, - instructions: `${body.instructions.trim()}\n\n${buildCavemanOutputInstruction(config, language)}`, + instructions: `${body.instructions.trim()}\n\n${instruction}`, }, applied: true, }; } + if (typeof body.input === "string" || Array.isArray(body.input)) { + return { body: { ...body, instructions: instruction }, applied: true }; + } return { body, applied: false, skippedReason: "no_messages" }; } diff --git a/open-sse/services/compression/rules/es/context.json b/open-sse/services/compression/rules/es/context.json index 980da607a7..0ef5a8ed33 100644 --- a/open-sse/services/compression/rules/es/context.json +++ b/open-sse/services/compression/rules/es/context.json @@ -33,6 +33,55 @@ "context": "all", "category": "context", "minIntensity": "lite" + }, + { + "name": "es_compound_collapse", + "pattern": "\\by cualquier potencial\\b", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "full" + }, + { + "name": "es_explanatory_prefix", + "pattern": "\\b(?:la funcion parece manejar|la función parece manejar|el codigo parece|el código parece|la clase es|este modulo es|este módulo es)\\b", + "replacement": "Función:", + "replacementMap": { + "la funcion parece manejar": "Función:", + "la función parece manejar": "Función:", + "el codigo parece": "Código:", + "el código parece": "Código:", + "la clase es": "Clase:", + "este modulo es": "Módulo:", + "este módulo es": "Módulo:" + }, + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "es_meta_commentary", + "pattern": "^(?:Ten en cuenta que|Recuerda que|Nota que)\\b\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "es_purpose_statement", + "pattern": "\\b(?:con el proposito de|con el propósito de|con la meta de|con el objetivo de|en un esfuerzo por|por cada)\\b", + "replacement": "para", + "replacementMap": { + "con el proposito de": "para", + "con el propósito de": "para", + "con la meta de": "para", + "con el objetivo de": "para", + "en un esfuerzo por": "para", + "por cada": "por" + }, + "context": "all", + "category": "context", + "minIntensity": "lite" } ] } diff --git a/open-sse/services/compression/rules/es/dedup.json b/open-sse/services/compression/rules/es/dedup.json new file mode 100644 index 0000000000..ac5e52239d --- /dev/null +++ b/open-sse/services/compression/rules/es/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "es", + "category": "dedup", + "rules": [ + { + "name": "es_repeated_context", + "pattern": "\\b(?:como mencionamos antes|como se menciono antes|como se mencionó antes|como dije antes|como comentamos anteriormente)\\b[,.]?\\s*", + "replacement": "Ver arriba. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "es_repeated_question", + "pattern": "\\b(?:misma pregunta que antes|pregunte esto antes|pregunté esto antes|esta es la misma pregunta|es la misma pregunta)\\b[,.]?\\s*", + "replacement": "[misma pregunta] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "es_reestablished_context", + "pattern": "\\b(?:volviendo al codigo anterior|volviendo al código anterior|refiriendome a|refiriéndome a|retomando)\\b\\s*", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "es_summary_replacement", + "pattern": "\\b(?:para resumir lo que hemos hablado|en resumen de nuestra conversacion|en resumen de nuestra conversación|recapitulando)\\b[,.]?\\s*", + "replacement": "Resumen: ", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/es/filler.json b/open-sse/services/compression/rules/es/filler.json index dee16e9e56..cf612f1ce0 100644 --- a/open-sse/services/compression/rules/es/filler.json +++ b/open-sse/services/compression/rules/es/filler.json @@ -65,6 +65,46 @@ "context": "all", "category": "filler", "minIntensity": "lite" + }, + { + "name": "es_verbose_instructions", + "pattern": "\\b(?:proporciona una explicacion detallada de|proporciona una explicación detallada de|dame una explicacion completa de|dame una explicación completa de|escribe una explicacion en profundidad de|escribe una explicación en profundidad de|crea una explicacion exhaustiva de|crea una explicación exhaustiva de|explica en detalle)\\b", + "replacement": "explica ", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "es_redundant_openers", + "pattern": "^(?:hola|buenos dias|buenos días|buenas tardes|buenas|hey)\\s*[,.!?\\s]?\\s*", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "es_excessive_gratitude", + "pattern": "\\b(?:muchisimas gracias|muchísimas gracias|gracias de antemano|te lo agradezco mucho|lo agradezco mucho)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "es_uncertainty_fillers", + "pattern": "\\b(?:supongo que|imagino que|me da la impresion de que|me da la impresión de que|quizas|quizás)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "es_assistant_fillers", + "pattern": "^(?:aqui tienes|aquí tienes|debajo esta|debajo está|esto es)\\s+(?:un|una|el|la)?\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" } ] } diff --git a/open-sse/services/compression/rules/es/structural.json b/open-sse/services/compression/rules/es/structural.json index f1d2454043..759ac33d4b 100644 --- a/open-sse/services/compression/rules/es/structural.json +++ b/open-sse/services/compression/rules/es/structural.json @@ -25,6 +25,81 @@ "context": "all", "category": "structural", "minIntensity": "lite" + }, + { + "name": "es_redundant_phrasing", + "pattern": "\\b(?:asegurate de|asegúrate de|asegurate que|asegúrate que|ten cuidado de)\\b\\s*", + "replacement": "asegura ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_redundant_because", + "pattern": "\\b(?:debido a que|por la razon de que|por la razón de que|la razon es porque|la razón es porque)\\b\\s*", + "replacement": "porque ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_redundant_directive", + "pattern": "\\b(?:es importante que|deberias|deberías|recuerda que)\\b\\s*", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_list_conjunction", + "pattern": ",\\s*y tambien\\s+|,\\s*y también\\s+|,\\s*asi como\\s+|,\\s*así como\\s+", + "replacement": ", ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_redundant_quantifiers", + "pattern": "\\b(?:cada uno de los|cada una de las|todos y cada uno|todas y cada una)\\b", + "replacement": "cada", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_all_quantifier", + "pattern": "\\b(?:cualquiera y todos|todas y cada una de las|todos y cada uno de los)\\b", + "replacement": "todos", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "es_transition_removal", + "pattern": "^(?:Por otro lado,?\\s*|En contraste,?\\s*|Sin embargo,?\\s*)", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "es_passive_voice", + "pattern": "\\b(?:esta siendo usado|está siendo usado|esta siendo llamada|está siendo llamada|esta siendo generado|está siendo generado|fue creado|fue generado|fue implementado)\\b", + "replacement": "usa", + "replacementMap": { + "esta siendo usado": "usa", + "está siendo usado": "usa", + "esta siendo llamada": "llama", + "está siendo llamada": "llama", + "esta siendo generado": "generado", + "está siendo generado": "generado", + "fue creado": "creado", + "fue generado": "generado", + "fue implementado": "implementado" + }, + "context": "all", + "category": "structural", + "minIntensity": "full" } ] } diff --git a/open-sse/services/compression/rules/es/ultra.json b/open-sse/services/compression/rules/es/ultra.json new file mode 100644 index 0000000000..bbc51cd0c0 --- /dev/null +++ b/open-sse/services/compression/rules/es/ultra.json @@ -0,0 +1,107 @@ +{ + "language": "es", + "category": "ultra", + "rules": [ + { + "name": "es_articles", + "pattern": "\\b(?:[Ee]l|[Ll]a|[Ll]os|[Ll]as|[Uu]n|[Uu]na|[Uu]nos|[Uu]nas)\\s+(?=[a-záéíóúñ])", + "flags": "g", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "es_leader_phrases", + "pattern": "^(?:voy a|puedo|podemos|vamos a|dejame|déjame|se puede)\\s+(?=[a-záéíóúñ])", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "es_ultra_database_abbreviation", + "pattern": "\\bbase de datos\\b", + "replacement": "BD", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_config_abbreviation", + "pattern": "\\bconfiguracion\\b|\\bconfiguración\\b", + "replacement": "config", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_function_abbreviation", + "pattern": "\\bfuncion\\b|\\bfunción\\b", + "replacement": "fn", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_request_abbreviation", + "pattern": "\\b(?:peticion|petición|solicitud)\\b", + "replacement": "req", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_response_abbreviation", + "pattern": "\\brespuesta\\b", + "replacement": "res", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_implementation_abbreviation", + "pattern": "\\bimplementacion\\b|\\bimplementación\\b", + "replacement": "impl", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_authentication_abbreviation", + "pattern": "\\bautenticacion\\b|\\bautenticación\\b", + "replacement": "auth", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_authorization_abbreviation", + "pattern": "\\bautorizacion\\b|\\bautorización\\b", + "replacement": "authz", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_application_abbreviation", + "pattern": "\\baplicacion\\b|\\baplicación\\b", + "replacement": "app", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "es_ultra_dependency_abbreviation", + "pattern": "\\b(?:dependencia|dependencias)\\b", + "replacement": "dep", + "replacementMap": { + "dependencia": "dep", + "dependencias": "deps" + }, + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + } + ] +} diff --git a/open-sse/services/compression/strategySelector.ts b/open-sse/services/compression/strategySelector.ts index 37bc9782ce..49edd08419 100644 --- a/open-sse/services/compression/strategySelector.ts +++ b/open-sse/services/compression/strategySelector.ts @@ -13,6 +13,7 @@ import { createCompressionStats } from "./stats.ts"; import { registerBuiltinCompressionEngines } from "./engines/index.ts"; import { getCompressionEngine } from "./engines/registry.ts"; import { applyRtkCompression } from "./engines/rtk/index.ts"; +import { adaptBodyForCompression } from "./bodyAdapter.ts"; import { detectCachingContext, getCacheAwareStrategy, @@ -73,19 +74,23 @@ export function applyCompression( if (mode === "off") { return { body, compressed: false, stats: null }; } - if (mode === "lite") { - return applyLiteCompression(body, { - ...options, - preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, - }); - } if (mode === "rtk") { return applyRtkCompression(body, { config: options?.config?.rtkConfig, }); } + const adapter = adaptBodyForCompression(body); + const compressionBody = adapter.body; + if (mode === "lite") { + const result = applyLiteCompression(compressionBody, { + ...options, + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; + } if (mode === "stacked") { - return applyStackedCompression(body, options?.config?.stackedPipeline, options); + const result = applyStackedCompression(compressionBody, options?.config?.stackedPipeline, options); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; } if (mode === "standard") { const cavemanConfig = { @@ -105,10 +110,14 @@ export function applyCompression( } : {}), }; - return cavemanCompress(body as Parameters[0], cavemanConfig); + const result = cavemanCompress( + compressionBody as Parameters[0], + cavemanConfig + ); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; } if (mode === "aggressive") { - const messages = (body.messages ?? []) as Array<{ + const messages = (compressionBody.messages ?? []) as Array<{ role: string; content?: string | Array<{ type: string; text?: string }>; [key: string]: unknown; @@ -121,12 +130,12 @@ export function applyCompression( preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, }; const result = compressAggressive(messages, aggressiveConfig); - const compressedBody = { ...body, messages: result.messages }; + const compressedBody = { ...compressionBody, messages: result.messages }; return { - body: compressedBody, + body: adapter.restore(compressedBody), compressed: result.stats.savingsPercent > 0, stats: createCompressionStats( - body, + compressionBody, compressedBody, mode, ["aggressive"], @@ -136,7 +145,7 @@ export function applyCompression( }; } if (mode === "ultra") { - const messages = (body.messages ?? []) as Array<{ + const messages = (compressionBody.messages ?? []) as Array<{ role: string; content?: string | unknown[]; [key: string]: unknown; @@ -149,12 +158,12 @@ export function applyCompression( preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, }; const result = ultraCompress(messages, ultraConfig); - const compressedBody = { ...body, messages: result.messages }; + const compressedBody = { ...compressionBody, messages: result.messages }; return { - body: compressedBody, + body: adapter.restore(compressedBody), compressed: result.stats.savingsPercent > 0, stats: createCompressionStats( - body, + compressionBody, compressedBody, mode, ["ultra"], diff --git a/src/lib/db/compressionAnalytics.ts b/src/lib/db/compressionAnalytics.ts index 16a5b51415..fd84fe35b6 100644 --- a/src/lib/db/compressionAnalytics.ts +++ b/src/lib/db/compressionAnalytics.ts @@ -26,6 +26,8 @@ export interface CompressionAnalyticsRow { output_mode?: string | null; rtk_raw_output_pointer?: string | null; rtk_raw_output_bytes?: number | null; + rtk_raw_output_pointers?: string | null; + rtk_raw_output_total_bytes?: number | null; } export interface CompressionAnalyticsSummary { @@ -118,6 +120,14 @@ function ensureCompressionAnalyticsColumns(): void { "rtk_raw_output_bytes", "ALTER TABLE compression_analytics ADD COLUMN rtk_raw_output_bytes INTEGER" ); + addColumn( + "rtk_raw_output_pointers", + "ALTER TABLE compression_analytics ADD COLUMN rtk_raw_output_pointers TEXT" + ); + addColumn( + "rtk_raw_output_total_bytes", + "ALTER TABLE compression_analytics ADD COLUMN rtk_raw_output_total_bytes INTEGER" + ); columnsEnsuredForDb = db; } @@ -131,9 +141,10 @@ export function insertCompressionAnalyticsRow(row: CompressionAnalyticsRow): voi duration_ms, request_id, actual_prompt_tokens, actual_completion_tokens, actual_total_tokens, actual_cache_read_tokens, actual_cache_write_tokens, estimated_usd_saved, mcp_description_tokens_saved, multimodal_skip_count, - receipt_source, validation_fallback, output_mode, rtk_raw_output_pointer, rtk_raw_output_bytes + receipt_source, validation_fallback, output_mode, rtk_raw_output_pointer, rtk_raw_output_bytes, + rtk_raw_output_pointers, rtk_raw_output_total_bytes ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` ).run( row.timestamp, @@ -159,7 +170,9 @@ export function insertCompressionAnalyticsRow(row: CompressionAnalyticsRow): voi row.validation_fallback ? 1 : 0, row.output_mode ?? null, row.rtk_raw_output_pointer ?? null, - row.rtk_raw_output_bytes ?? null + row.rtk_raw_output_bytes ?? null, + row.rtk_raw_output_pointers ?? null, + row.rtk_raw_output_total_bytes ?? null ); } diff --git a/tests/unit/compression/body-adapter.test.ts b/tests/unit/compression/body-adapter.test.ts new file mode 100644 index 0000000000..be584bf211 --- /dev/null +++ b/tests/unit/compression/body-adapter.test.ts @@ -0,0 +1,189 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { applyCompression } from "../../../open-sse/services/compression/strategySelector.ts"; +import { applyRtkCompression } from "../../../open-sse/services/compression/engines/rtk/index.ts"; + +describe("compression body adapter", () => { + it("applies Caveman compression to OpenAI Responses input messages", () => { + const body = { + model: "gpt-5.5-codex", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: "Please could you provide a detailed explanation of this implementation? Thank you so much for your help!", + }, + ], + }, + { + type: "function_call", + call_id: "call_1", + name: "read_file", + arguments: "{}", + }, + ], + }; + + const result = applyCompression(body, "standard", { + config: { + enabled: true, + defaultMode: "standard", + autoTriggerTokens: 0, + cacheMinutes: 5, + preserveSystemPrompt: true, + comboOverrides: {}, + cavemanConfig: { + enabled: true, + compressRoles: ["user"], + skipRules: [], + minMessageLength: 10, + preservePatterns: [], + intensity: "full", + }, + }, + }); + + assert.equal(result.compressed, true); + assert.ok(!("messages" in result.body), "Responses body must not leak synthetic messages"); + const input = result.body.input as typeof body.input; + assert.equal(input[1], body.input[1], "non-message Responses items should be preserved"); + const text = input[0].content[0].text; + assert.ok(!text.includes("Please could you")); + assert.ok(!text.includes("Thank you so much")); + assert.ok(text.includes("explain")); + }); + + it("applies RTK compression to Responses function_call_output items", () => { + const repeatedOutput = Array.from({ length: 20 }, () => "same noisy line").join("\n"); + const body = { + input: [ + { + type: "function_call_output", + call_id: "call_1", + output: repeatedOutput, + }, + ], + }; + + const result = applyRtkCompression(body); + + assert.equal(result.compressed, true); + assert.ok(!("messages" in result.body), "Responses body must not leak synthetic messages"); + const input = result.body.input as typeof body.input; + assert.match(input[0].output, /\[rtk:dropped/); + assert.equal(input[0].call_id, "call_1"); + }); + + it("restores compressed array output on Responses function_call_output items", () => { + const repeatedOutput = Array.from({ length: 20 }, () => "same noisy line").join("\n"); + const body = { + input: [ + { + type: "function_call_output", + call_id: "call_1", + output: [{ type: "input_text", text: repeatedOutput }], + }, + ], + }; + + const result = applyRtkCompression(body); + const input = result.body.input as typeof body.input; + + assert.equal(result.compressed, true); + assert.match(input[0].output[0].text, /\[rtk:dropped/); + assert.ok(!("content" in input[0]), "function_call_output should keep canonical output field"); + }); + + it("restores adapted Responses bodies even when no compression is applied", () => { + const body = { + input: [ + { + type: "message", + role: "user", + content: [{ type: "input_text", text: "short" }], + }, + ], + }; + + const result = applyCompression(body, "standard", { + config: { + enabled: true, + defaultMode: "standard", + autoTriggerTokens: 0, + cacheMinutes: 5, + preserveSystemPrompt: true, + comboOverrides: {}, + cavemanConfig: { + enabled: true, + compressRoles: ["user"], + skipRules: [], + minMessageLength: 50, + preservePatterns: [], + intensity: "full", + }, + }, + }); + + assert.equal(result.compressed, false); + assert.ok(!("messages" in result.body)); + assert.deepEqual(result.body.input, body.input); + }); + + it("does not misalign Responses input items if an engine removes a synthetic message", () => { + const body = { + input: [ + { type: "message", role: "user", content: "duplicate" }, + { type: "message", role: "user", content: "duplicate" }, + { type: "message", role: "user", content: "unique" }, + ], + }; + + const result = applyCompression(body, "lite", { + config: { + enabled: true, + defaultMode: "lite", + autoTriggerTokens: 0, + cacheMinutes: 5, + preserveSystemPrompt: true, + comboOverrides: {}, + }, + }); + + assert.equal(result.compressed, true); + assert.deepEqual(result.body.input, body.input); + }); + + it("compresses string Responses input without converting the request shape", () => { + const body = { + input: + "Please could you provide a detailed explanation of this implementation? Thank you so much for your help!", + }; + + const result = applyCompression(body, "standard", { + config: { + enabled: true, + defaultMode: "standard", + autoTriggerTokens: 0, + cacheMinutes: 5, + preserveSystemPrompt: true, + comboOverrides: {}, + cavemanConfig: { + enabled: true, + compressRoles: ["user"], + skipRules: [], + minMessageLength: 10, + preservePatterns: [], + intensity: "full", + }, + }, + }); + + assert.equal(result.compressed, true); + assert.equal(typeof result.body.input, "string"); + assert.ok(!(result.body.input as string).includes("Please could you")); + }); +}); diff --git a/tests/unit/compression/compressionAnalytics.test.ts b/tests/unit/compression/compressionAnalytics.test.ts index 12d1f55443..5f00962fbc 100644 --- a/tests/unit/compression/compressionAnalytics.test.ts +++ b/tests/unit/compression/compressionAnalytics.test.ts @@ -89,6 +89,37 @@ describe("compressionAnalytics", () => { assert.equal(summary.totalRequests, 1); }); + it("stores all RTK raw output pointers when provided", () => { + insertCompressionAnalyticsRow({ + timestamp: new Date().toISOString(), + mode: "rtk", + original_tokens: 1000, + compressed_tokens: 500, + tokens_saved: 500, + rtk_raw_output_pointer: "raw_1", + rtk_raw_output_bytes: 100, + rtk_raw_output_pointers: JSON.stringify(["raw_1", "raw_2"]), + rtk_raw_output_total_bytes: 250, + }); + + const db = getDbInstance(); + const row = db + .prepare( + "SELECT rtk_raw_output_pointer, rtk_raw_output_bytes, rtk_raw_output_pointers, rtk_raw_output_total_bytes FROM compression_analytics LIMIT 1" + ) + .get() as { + rtk_raw_output_pointer: string; + rtk_raw_output_bytes: number; + rtk_raw_output_pointers: string; + rtk_raw_output_total_bytes: number; + }; + + assert.equal(row.rtk_raw_output_pointer, "raw_1"); + assert.equal(row.rtk_raw_output_bytes, 100); + assert.equal(row.rtk_raw_output_pointers, JSON.stringify(["raw_1", "raw_2"])); + assert.equal(row.rtk_raw_output_total_bytes, 250); + }); + it("summary counts correctly after multiple inserts", () => { insertCompressionAnalyticsRow({ timestamp: new Date().toISOString(), diff --git a/tests/unit/compression/language-packs.test.ts b/tests/unit/compression/language-packs.test.ts index b551e6bfff..7172f40374 100644 --- a/tests/unit/compression/language-packs.test.ts +++ b/tests/unit/compression/language-packs.test.ts @@ -48,6 +48,28 @@ describe("Caveman language packs", () => { assert.ok(text.includes("auth")); }); + it("keeps the Spanish pack aligned with English rule categories", () => { + const esRules = loadAllRulesForLanguage("es", { refresh: true }); + assert.ok(esRules.length >= 40, `es expected 40+ rules, got ${esRules.length}`); + assert.ok(esRules.some((rule) => rule.category === "dedup"), "es missing dedup"); + assert.ok(esRules.some((rule) => rule.category === "ultra"), "es missing ultra"); + assert.ok(esRules.some((rule) => rule.category === "terse"), "es missing terse"); + }); + + it("applies expanded Spanish rules without touching technical terms", () => { + const esRules = getRulesForContext("user", "ultra", "es"); + const { text } = applyRulesToText( + "Por favor proporciona una explicación detallada de la base de datos y autenticación en src/auth.ts", + esRules + ); + + assert.ok(!text.toLowerCase().includes("por favor")); + assert.ok(!text.toLowerCase().includes("explicación detallada")); + assert.ok(text.includes("BD")); + assert.ok(text.includes("auth")); + assert.ok(text.includes("src/auth.ts")); + }); + it("builds localized output mode instructions", () => { const config = { enabled: true, intensity: "full" as const, autoClarity: true }; diff --git a/tests/unit/compression/outputMode.test.ts b/tests/unit/compression/outputMode.test.ts index 33de29d05e..fffcaa4d0c 100644 --- a/tests/unit/compression/outputMode.test.ts +++ b/tests/unit/compression/outputMode.test.ts @@ -68,6 +68,17 @@ describe("Caveman output mode", () => { assert.equal(result.body.messages?.at(-1)?.content, body.messages[0].content); }); + it("uses Responses instructions when input has no messages", () => { + const result = applyCavemanOutputMode( + { input: [{ type: "message", role: "user", content: "Summarize logs." }] }, + { enabled: true, intensity: "full", autoClarity: true } + ); + + assert.equal(result.applied, true); + assert.match(String(result.body.instructions), /Caveman Output Mode/); + assert.ok(!("messages" in result.body)); + }); + it("bypasses security, destructive, clarification, and order-sensitive prompts", () => { const cases = [ "Explain this security vulnerability in detail.", diff --git a/tests/unit/compression/rtk-code-stripper.test.ts b/tests/unit/compression/rtk-code-stripper.test.ts index ab28be648e..bba15c0264 100644 --- a/tests/unit/compression/rtk-code-stripper.test.ts +++ b/tests/unit/compression/rtk-code-stripper.test.ts @@ -16,27 +16,24 @@ describe("RTK code stripper", () => { assert.equal(detectCodeLanguage("class Main { }"), "java"); }); - it("removes single-line and multi-line comments", () => { + it("preserves comments and string literals safely", () => { const js = stripCode( - "// comment\nconst value = 1;\n/* block */\nconsole.log(value);", + "// comment\nconst url = 'https://example.com/a//b';\n/* block */\nconsole.log(url);", "javascript" ); - const py = stripCode('"""doc"""\n# comment\nprint("ok")', "python"); - const rb = stripCode("=begin\ncomment\n=end\n# comment\nputs 'ok'", "ruby"); - assert.ok(!js.text.includes("comment")); - assert.ok(!js.text.includes("block")); - assert.ok(!py.text.includes("doc")); - assert.ok(!rb.text.includes("comment")); + assert.ok(js.text.includes("// comment")); + assert.ok(js.text.includes("https://example.com/a//b")); + assert.ok(js.text.includes("/* block */")); }); - it("preserves docstrings when configured", () => { + it("preserves Python docstrings and comments", () => { const result = stripCode('"""doc"""\n# comment\nprint("ok")', "python", { preserveDocstrings: true, }); assert.ok(result.text.includes("doc")); - assert.ok(!result.text.includes("# comment")); + assert.ok(result.text.includes("# comment")); }); it("applies to fenced code blocks through RTK runtime", () => { @@ -44,7 +41,13 @@ describe("RTK code stripper", () => { messages: [ { role: "assistant", - content: "```ts\n// remove\nconst value: number = 1;\n```\nDone.", + content: `Before. + +\`\`\`txt +${Array.from({ length: 20 }, () => "same code line").join("\n")} +\`\`\` + +After.`, }, ], }; @@ -65,8 +68,68 @@ describe("RTK code stripper", () => { assert.equal(result.compressed, true); const serialized = JSON.stringify(result.body.messages); - assert.match(serialized, /const value/); - assert.doesNotMatch(serialized, /remove/); + assert.match(serialized, /Before/); + assert.match(serialized, /After/); + assert.match(serialized, /same code line/); + assert.match(serialized, /\[rtk:dropped/); assert.ok(result.stats?.techniquesUsed.includes("rtk-code-strip")); }); + + it("does not compress non-code text when only code block compression is enabled", () => { + const content = Array.from({ length: 20 }, () => "same prose line").join("\n"); + const body = { + messages: [{ role: "assistant", content }], + }; + const result = applyRtkCompression(body, { + config: { + enabled: true, + intensity: "standard", + applyToToolResults: false, + applyToAssistantMessages: false, + applyToCodeBlocks: true, + enabledFilters: [], + disabledFilters: [], + maxLinesPerResult: 100, + maxCharsPerResult: 12000, + deduplicateThreshold: 3, + }, + }); + + assert.equal(result.compressed, false); + assert.deepEqual(result.body, body); + }); + + it("does not compress fenced code when code block compression is disabled", () => { + const body = { + messages: [ + { + role: "assistant", + content: `Before. + +\`\`\`txt +${Array.from({ length: 20 }, () => "same code line").join("\n")} +\`\`\` + +After.`, + }, + ], + }; + const result = applyRtkCompression(body, { + config: { + enabled: true, + intensity: "standard", + applyToToolResults: false, + applyToAssistantMessages: false, + applyToCodeBlocks: false, + enabledFilters: [], + disabledFilters: [], + maxLinesPerResult: 100, + maxCharsPerResult: 12000, + deduplicateThreshold: 3, + }, + }); + + assert.equal(result.compressed, false); + assert.deepEqual(result.body, body); + }); });