From fb6ea295bfeeb24d477a7de72456b7530f9473af Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Wed, 22 Jul 2026 12:03:02 +0200 Subject: [PATCH] feat(compression): add Responses tool-output engine (#8010) * Add Responses tool-output compression engine * fix: enable Codex Responses stacked steps * fix(compression): share Codex tokenizer and rebase UI * fix(compression): sync MCP engine selection * fix(compression): i18n parity for codex-responses mode + rebaseline The codex-responses compression engine already imports the shared countTextTokens/resolveTokenizerEncoding from tiktokenCounter.ts (no duplicate encoder) and CompressionSettingsTab.tsx already threads the new mode through the existing useTranslations()/labelKey pattern - both pre-existing on this branch tip after rebasing onto release/v3.8.49. What was missing after the rebase: the new compressionModeCodexResponses / compressionModeCodexResponsesDesc keys existed only in en.json. Filled en-fallback into all 42 locales via scripts/i18n/fill-missing-from-en.mjs and added real pt-BR/vi translations. Also rebaselined the three files whose own growth (new codex-responses mode wiring) crossed the frozen file-size caps: open-sse/mcp-server/schemas/tools.ts, open-sse/services/ compression/strategySelector.ts, and src/lib/db/compression.ts. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: Diego Rodrigues de Sa e Souza Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- config/quality/file-size-baseline.json | 6 +- open-sse/mcp-server/schemas/tools.ts | 28 +- open-sse/mcp-server/tools/compressionTools.ts | 21 +- open-sse/services/compression/bodyAdapter.ts | 88 ++++- .../services/compression/deriveDefaultPlan.ts | 10 +- .../services/compression/engineCatalog.ts | 17 +- .../engines/codexResponses/index.ts | 314 ++++++++++++++++++ .../services/compression/engines/index.ts | 2 + open-sse/services/compression/index.ts | 3 + open-sse/services/compression/liveZone.ts | 2 + .../services/compression/strategySelector.ts | 46 ++- open-sse/services/compression/types.ts | 55 ++- .../context/combos/CompressionHub.tsx | 3 +- .../components/CompressionSettingsTab.tsx | 190 ++++++----- src/i18n/messages/ar.json | 8 +- src/i18n/messages/az.json | 8 +- src/i18n/messages/bg.json | 8 +- src/i18n/messages/bn.json | 8 +- src/i18n/messages/cs.json | 8 +- src/i18n/messages/da.json | 8 +- src/i18n/messages/de.json | 8 +- src/i18n/messages/en.json | 6 + src/i18n/messages/es.json | 8 +- src/i18n/messages/fa.json | 8 +- src/i18n/messages/fi.json | 8 +- src/i18n/messages/fr.json | 8 +- src/i18n/messages/gu.json | 8 +- src/i18n/messages/he.json | 8 +- src/i18n/messages/hi.json | 8 +- src/i18n/messages/hu.json | 8 +- src/i18n/messages/id.json | 8 +- src/i18n/messages/in.json | 8 +- src/i18n/messages/it.json | 8 +- src/i18n/messages/ja.json | 8 +- src/i18n/messages/ko.json | 8 +- src/i18n/messages/mr.json | 8 +- src/i18n/messages/ms.json | 8 +- src/i18n/messages/nl.json | 8 +- src/i18n/messages/no.json | 8 +- src/i18n/messages/phi.json | 8 +- src/i18n/messages/pl.json | 8 +- src/i18n/messages/pt-BR.json | 8 +- src/i18n/messages/pt.json | 8 +- src/i18n/messages/ro.json | 8 +- src/i18n/messages/ru.json | 8 +- src/i18n/messages/sk.json | 8 +- src/i18n/messages/sv.json | 8 +- src/i18n/messages/sw.json | 8 +- src/i18n/messages/ta.json | 8 +- src/i18n/messages/te.json | 8 +- src/i18n/messages/th.json | 8 +- src/i18n/messages/tr.json | 8 +- src/i18n/messages/uk-UA.json | 8 +- src/i18n/messages/ur.json | 8 +- src/i18n/messages/vi.json | 8 +- src/i18n/messages/zh-CN.json | 8 +- src/i18n/messages/zh-TW.json | 8 +- src/lib/db/compression.ts | 50 +++ src/lib/db/compressionCombos.ts | 2 + .../ComboCompressionModeSelect.tsx | 1 + .../validation/compressionConfigSchemas.ts | 23 ++ src/shared/validation/schemas/combo.ts | 1 + .../unit/compression/codex-responses.test.ts | 178 ++++++++++ .../compression/compressionMcpTools.test.ts | 17 + .../compression/omniglyph-registries.test.ts | 4 + tests/unit/compression/types.test.ts | 12 +- 66 files changed, 1254 insertions(+), 161 deletions(-) create mode 100644 open-sse/services/compression/engines/codexResponses/index.ts create mode 100644 tests/unit/compression/codex-responses.test.ts diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 109d9fb75a..0e5d99a37d 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -178,7 +178,9 @@ "open-sse/handlers/search.ts": 1546, "open-sse/handlers/sseParser.ts": 830, "open-sse/handlers/videoGeneration.ts": 1275, - "open-sse/mcp-server/schemas/tools.ts": 1497, + "_rebaseline_2026_07_22_8010_codex_responses_engine": "PR #8010 (@JxnLexn) own growth: open-sse/mcp-server/schemas/tools.ts 1497->1505 (+8 = threading the new \"codex-responses\" literal into the compressionConfigureInput strategy/autoTriggerMode Zod enums and setCompressionEngineInput engine enum, mirroring the existing rtk/omniglyph enum entries; no new tool). open-sse/services/compression/strategySelector.ts 1043->1054 (+11 = one new `if (mode === \"codex-responses\")` dispatch branch in runCompression that delegates 100% to the new codexResponsesEngine.apply, mirroring the existing rtk single-mode dispatch, plus threading config.codexResponsesConfig.preserveToolNames into the shared adaptBodyForCompression call at the 3 existing call sites). src/lib/db/compression.ts (untracked, new-file cap 800) 794->845 (+51 = normalizeCodexResponsesConfig, mirroring the existing normalizeRtkConfig normalizer, plus registering \"codex-responses\" in the COMPRESSION_MODES/STACKED_PIPELINE_ENGINE_IDS/SINGLE_MODE_ENGINE sets and the getCompressionSettings load/save switch) — added to the baseline at its current size. All three are cohesive dispatch/normalizer wiring at existing chokepoints (mirroring the prior compression-mode rebaselines #6534/#6556), not extractable without hiding the mode-dispatch boundary. Covered by tests/unit/compression/codex-responses.test.ts (6) + omniglyph-registries.test.ts/types.test.ts (22, updated for the new mode).", + "src/lib/db/compression.ts": 845, + "open-sse/mcp-server/schemas/tools.ts": 1505, "open-sse/mcp-server/server.ts": 1555, "open-sse/mcp-server/tools/advancedTools.ts": 1120, "_rebaseline_2026_06_27_5193_antigravity_basered": "Base-red (pre-existing release drift, fast-gate PR->release skips check:file-size): accountFallback.ts 1773->1777 and src/app/api/providers/[id]/test/route.ts 924->940 were already over their frozen caps on release/v3.8.39 independent of any antigravity change. Owner chose to rebaseline (keep the documented issue-reference comments #1846/#1449/#347 etc.) rather than accept the contributor comment-stripping in #5200/#5198. Reverted #5200 to restore the comments; bumped these two frozen caps to the actual base sizes. No logic change.", @@ -194,7 +196,7 @@ "_rebaseline_2026_06_26_fidelity_gate_extraction": "Milestone-B fidelity-gate wiring residual: bodyToText+gateAdvance extracted to fidelityGateStep.ts (889->854, -35), but the StackOptions.fidelityGate field, the `const fidelityGate` reads at the two stacked-loop dispatch chokepoints, and the import of FidelityGateConfig are irreducible wiring that cannot leave strategySelector without an architectural refactor of the pre-existing stacked pipeline. Net: 889->854 (+6 vs the pre-Milestone-B frozen 848). Covered by tests/unit/compression/*.test.ts (940 pass).", "_rebaseline_2026_06_28_5243_risk_gate_prepass": "PR #5243 (compression risk-gate pre-pass) own growth: open-sse/services/compression/strategySelector.ts 854->899 (+45). The three exported entry points (applyCompression/applyStackedCompression/applyStackedCompressionAsync) become thin wrappers over pure-extracted private bodies (runCompression/runStackedCompression/runStackedCompressionAsync) so the risk-gate mask->run->restore wrapper sits strictly OUTSIDE the per-step loop — a single universal integration point. The wrapper logic itself (resolveRiskGate/withRiskGate) lives in the new riskGate/strategyWrap.ts (960 (+61 = the opt-in result-memoization branches in applyCompression/applyCompressionAsync — principal+determinism gate, makeMemoKey lookup/store with model+supportsVision folded into the key, recompute-with-memo-off). Default off (memoizeCompressionResults), so zero behavior change. The memo helpers live in the leaf resultMemo.ts ( 800 cap). It is the vendored GCF generic-profile decoder (spec v3.2 nested flattening plus the prototype-pollution / hasOwnProperty hardening added in this PR's Gemini review). Kept as one file faithful to upstream gcf-typescript so re-vendoring stays a clean copy rather than a re-split each cycle (sibling generic.ts/scalar.ts stay < cap; extraction would also fragment the file's frozen eslint no-explicit-any suppressions). Round-trip + prototype-pollution regression coverage in tests/unit/compression/headroom-smartcrusher.test.ts. Frozen: only shrinks from here.", "open-sse/services/compression/engines/headroom/gcf/decode_generic.ts": 880, "open-sse/services/rateLimitManager.ts": 1035, diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index bf77b11577..012198baf8 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -1095,11 +1095,31 @@ export const compressionStatusTool: McpToolDefinition< export const compressionConfigureInput = z.object({ enabled: z.boolean().optional(), strategy: z - .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked", "omniglyph"]) + .enum([ + "off", + "lite", + "standard", + "aggressive", + "ultra", + "rtk", + "codex-responses", + "stacked", + "omniglyph", + ]) .optional() .describe("Compression mode"), autoTriggerMode: z - .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked", "omniglyph"]) + .enum([ + "off", + "lite", + "standard", + "aggressive", + "ultra", + "rtk", + "codex-responses", + "stacked", + "omniglyph", + ]) .optional(), maxTokens: z .number() @@ -1132,7 +1152,7 @@ export const compressionConfigureTool: McpToolDefinition< > = { name: "omniroute_compression_configure", description: - "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/codex-responses/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", inputSchema: compressionConfigureInput, outputSchema: compressionConfigureOutput, scopes: ["write:compression"], @@ -1142,7 +1162,7 @@ export const compressionConfigureTool: McpToolDefinition< }; export const setCompressionEngineInput = z.object({ - engine: z.enum(["off", "caveman", "rtk", "stacked"]).optional(), + engine: z.enum(["off", "caveman", "rtk", "codex-responses", "stacked"]).optional(), cavemanIntensity: z.enum(["lite", "full", "ultra"]).optional(), rtkIntensity: z.enum(["minimal", "standard", "aggressive"]).optional(), outputMode: z.boolean().optional(), diff --git a/open-sse/mcp-server/tools/compressionTools.ts b/open-sse/mcp-server/tools/compressionTools.ts index d9e39ffd72..1958c4736d 100644 --- a/open-sse/mcp-server/tools/compressionTools.ts +++ b/open-sse/mcp-server/tools/compressionTools.ts @@ -428,26 +428,39 @@ export async function handleSetCompressionEngine( args: z.infer ): Promise<{ success: boolean; settings: Record }> { const updates: Record = { enabled: true }; + const current = await getCompressionSettings(); if (args.engine) { updates.defaultMode = args.engine === "caveman" ? "standard" : args.engine; - if (args.engine === "off") updates.enabled = false; + if (args.engine === "off") { + updates.enabled = false; + } else if (args.engine !== "stacked") { + const selectedEngine = args.engine === "caveman" ? "caveman" : args.engine; + updates.engines = Object.fromEntries( + Object.entries(current.engines).map(([id, toggle]) => [ + id, + { + ...toggle, + enabled: id === selectedEngine, + ...(id === "caveman" && args.cavemanIntensity ? { level: args.cavemanIntensity } : {}), + ...(id === "rtk" && args.rtkIntensity ? { level: args.rtkIntensity } : {}), + }, + ]) + ); + } } if (args.cavemanIntensity) { - const current = await getCompressionSettings(); updates.cavemanConfig = { ...(current.cavemanConfig ?? {}), intensity: args.cavemanIntensity, }; } if (args.rtkIntensity) { - const current = await getCompressionSettings(); updates.rtkConfig = { ...(current.rtkConfig ?? {}), intensity: args.rtkIntensity, }; } if (args.outputMode !== undefined) { - const current = await getCompressionSettings(); updates.cavemanOutputMode = { ...(current.cavemanOutputMode ?? {}), enabled: args.outputMode, diff --git a/open-sse/services/compression/bodyAdapter.ts b/open-sse/services/compression/bodyAdapter.ts index af407c510d..d62e3410e0 100644 --- a/open-sse/services/compression/bodyAdapter.ts +++ b/open-sse/services/compression/bodyAdapter.ts @@ -6,6 +6,17 @@ type MessageLike = { [key: string]: unknown; }; +export const CODEX_RESPONSE_ITEM_META = Symbol("codexResponseItemMeta"); + +export type CodexResponseItemMeta = { + type: string; + eligible: boolean; +}; + +type CodexMessageLike = MessageLike & { + [CODEX_RESPONSE_ITEM_META]?: CodexResponseItemMeta; +}; + type ResponsesItem = { type?: unknown; role?: unknown; @@ -18,6 +29,8 @@ const RESPONSES_MESSAGE_TYPES = new Set([ "message", "function_call_output", "custom_tool_call_output", + "local_shell_call_output", + "apply_patch_call_output", ]); const COMPRESSION_INPUT_INDEX = Symbol("compressionInputIndex"); @@ -100,11 +113,16 @@ function responsesToolOutputField(item: ResponsesItem): "output" | "content" { return item.output !== null && item.output !== undefined ? "output" : "content"; } -function responsesItemToMessage(item: ResponsesItem): MessageLike | null { +function responsesItemToMessage(item: ResponsesItem): CodexMessageLike | null { const type = typeof item.type === "string" ? item.type : "message"; if (!RESPONSES_MESSAGE_TYPES.has(type)) return null; - if (type === "function_call_output" || type === "custom_tool_call_output") { + if ( + type === "function_call_output" || + type === "custom_tool_call_output" || + type === "local_shell_call_output" || + type === "apply_patch_call_output" + ) { const rawOutput = item.output ?? item.content; // OpenAI Responses shape (Codex): body.input holds Responses items. When // output is a JSON object (not a string or content array), serialise it so @@ -123,6 +141,7 @@ function responsesItemToMessage(item: ResponsesItem): MessageLike | null { : isObjectOutput ? JSON.stringify(rawOutput) : toChatContent(rawOutput), + [CODEX_RESPONSE_ITEM_META]: { type, eligible: false }, }; } @@ -132,9 +151,65 @@ function responsesItemToMessage(item: ResponsesItem): MessageLike | null { }; } +const DEFAULT_CODEX_PROTECTED_TOOL_NAMES = new Set([ + "read", + "glob", + "grep", + "write", + "edit", + "websearch", + "webfetch", + "web_search", + "web_fetch", +]); +function markCodexResponseEligibility( + messages: CodexMessageLike[], + inputItems: unknown[], + preserveToolNames: string[] = [] +): void { + const protectedNames = new Set([ + ...DEFAULT_CODEX_PROTECTED_TOOL_NAMES, + ...preserveToolNames.map((name) => name.trim().toLowerCase()), + ]); + const functionCalls = new Map(); + const skippedCallIds = new Set(); + for (const raw of inputItems) { + if (!isRecord(raw) || raw.type !== "function_call") continue; + if (typeof raw.call_id !== "string" || raw.call_id.length === 0) continue; + const name = typeof raw.name === "string" ? raw.name : ""; + functionCalls.set(raw.call_id, name); + if ( + protectedNames.has(name.trim().toLowerCase()) || + name === "headless_retrieval" || + name.endsWith("__headless_retrieval") + ) { + skippedCallIds.add(raw.call_id); + } + } + + for (const message of messages) { + const meta = message[CODEX_RESPONSE_ITEM_META]; + if (!meta) continue; + if (meta.type === "local_shell_call_output" || meta.type === "apply_patch_call_output") { + meta.eligible = true; + continue; + } + if (meta.type !== "function_call_output") continue; + const rawIndex = message[COMPRESSION_INPUT_INDEX]; + const rawItem = typeof rawIndex === "number" ? inputItems[rawIndex] : null; + const callId = isRecord(rawItem) && typeof rawItem.call_id === "string" ? rawItem.call_id : ""; + meta.eligible = callId.length > 0 && functionCalls.has(callId) && !skippedCallIds.has(callId); + } +} + function messageToResponsesItem(message: MessageLike, originalItem: ResponsesItem): ResponsesItem { const type = typeof originalItem.type === "string" ? originalItem.type : "message"; - if (type === "function_call_output" || type === "custom_tool_call_output") { + if ( + type === "function_call_output" || + type === "custom_tool_call_output" || + type === "local_shell_call_output" || + type === "apply_patch_call_output" + ) { const outputField = responsesToolOutputField(originalItem); const originalOutput = originalItem[outputField]; return { @@ -166,7 +241,10 @@ export type CompressionBodyAdapter = { restore(compressedBody: Record): Record; }; -export function adaptBodyForCompression(body: Record): CompressionBodyAdapter { +export function adaptBodyForCompression( + body: Record, + preserveToolNames: string[] = [] +): CompressionBodyAdapter { if (Array.isArray(body.messages)) { return { body, @@ -205,6 +283,8 @@ export function adaptBodyForCompression(body: Record): Compress messages.push({ ...message, [COMPRESSION_INPUT_INDEX]: index }); }); + markCodexResponseEligibility(messages, inputItems, preserveToolNames); + if (messages.length === 0) { return { body, diff --git a/open-sse/services/compression/deriveDefaultPlan.ts b/open-sse/services/compression/deriveDefaultPlan.ts index 9fa3d2c55f..3711d9f015 100644 --- a/open-sse/services/compression/deriveDefaultPlan.ts +++ b/open-sse/services/compression/deriveDefaultPlan.ts @@ -8,16 +8,12 @@ const SINGLE_MODE_OF: Record = { aggressive: "aggressive", ultra: "ultra", rtk: "rtk", + "codex-responses": "codex-responses", omniglyph: "omniglyph", }; export type CompressionSource = - | "request-header" - | "routing-override" - | "active-profile" - | "auto-trigger" - | "default" - | "off"; + "request-header" | "routing-override" | "active-profile" | "auto-trigger" | "default" | "off"; export interface DerivedPlan { mode: string; @@ -37,7 +33,7 @@ export interface DerivedPlan { */ export function deriveDefaultPlan( engines: Record, - masterEnabled: boolean, + masterEnabled: boolean ): DerivedPlan { if (!masterEnabled) return { mode: "off", stackedPipeline: [] }; diff --git a/open-sse/services/compression/engineCatalog.ts b/open-sse/services/compression/engineCatalog.ts index df43a94877..567c3dfd7f 100644 --- a/open-sse/services/compression/engineCatalog.ts +++ b/open-sse/services/compression/engineCatalog.ts @@ -22,8 +22,8 @@ export interface EngineMeta { id: string; label: string; stackPriority: number; - levels?: string[]; // intensity options; undefined = no level selector - isSingleMode: boolean; // can be the effective mode when it is the only engine on + levels?: string[]; // intensity options; undefined = no level selector + isSingleMode: boolean; // can be the effective mode when it is the only engine on description: string; guidance: EngineGuidance; } @@ -82,6 +82,19 @@ export const ENGINE_CATALOG: Record = { cacheImpact: "moderate", }, }, + "codex-responses": { + id: "codex-responses", + label: "Responses Tool Output", + stackPriority: 12, + isSingleMode: true, + description: "Conservative compression for supported Responses tool outputs.", + guidance: { + tradeoffs: + "Lossless-first JSON and bounded diagnostic compression for shell, patch, search, and build outputs. Protected tools and uncertain shapes pass through unchanged.", + lossy: true, + cacheImpact: "low", + }, + }, headroom: { id: "headroom", label: "Headroom", diff --git a/open-sse/services/compression/engines/codexResponses/index.ts b/open-sse/services/compression/engines/codexResponses/index.ts new file mode 100644 index 0000000000..30ae67acda --- /dev/null +++ b/open-sse/services/compression/engines/codexResponses/index.ts @@ -0,0 +1,314 @@ +import { createCompressionStats } from "../../stats.ts"; +import { + DEFAULT_CODEX_RESPONSES_CONFIG, + type CodexResponsesConfig, + type CompressionResult, +} from "../../types.ts"; +import type { + CompressionEngine, + CompressionEngineApplyOptions, + EngineConfigField, + EngineValidationResult, +} from "../types.ts"; +import { CODEX_RESPONSE_ITEM_META } from "../../bodyAdapter.ts"; +import { countTextTokens } from "../../../../../src/shared/utils/tiktokenCounter.ts"; + +const ENGINE_ID = "codex-responses"; + +function countCodexTokens(text: string): number { + if (!text) return 0; + return countTextTokens(text, { provider: "codex" }); +} +const SUPPORTED_TYPES = new Set([ + "function_call_output", + "local_shell_call_output", + "apply_patch_call_output", +]); + +const DIFF_HUNK_RE = + /^@@{1,2}\s+-\d+(?:,\d+)?(?:\s+-\d+(?:,\d+)?)*\s+\+\d+(?:,\d+)?(?:,\d+)?\s+@@{1,2}/m; +const SEARCH_LINE_RE = /^\s*(?:[\w./-]+:\d+(?::\d+)?:|\d+(?::\d+)?[-:])\s*\S/m; +const IMPORTANT_LOG_RE = + /(?:^|\b)(?:error|warning|warn|failed|failure|fatal|panic|exception|traceback|assertion|exit\s+code)\b/i; +const BUILD_RE = + /\b(?:build|compile|test|lint|npm|yarn|pnpm|mix|make|cargo|gradle|maven|pytest|rspec|exunit)\b/i; + +const CODEX_SCHEMA: EngineConfigField[] = [ + { key: "enabled", type: "boolean", label: "Enabled", defaultValue: false }, + { + key: "minBytes", + type: "number", + label: "Minimum output bytes", + defaultValue: DEFAULT_CODEX_RESPONSES_CONFIG.minBytes, + min: 0, + max: 2_000_000, + }, + { + key: "maxOutputBytes", + type: "number", + label: "Maximum output bytes", + defaultValue: DEFAULT_CODEX_RESPONSES_CONFIG.maxOutputBytes, + min: 1, + max: 10_000_000, + }, + { + key: "maxCandidateBytes", + type: "number", + label: "Maximum candidate bytes", + defaultValue: DEFAULT_CODEX_RESPONSES_CONFIG.maxCandidateBytes, + min: 1, + max: 2_000_000, + }, + { + key: "maxLines", + type: "number", + label: "Maximum retained lines", + defaultValue: DEFAULT_CODEX_RESPONSES_CONFIG.maxLines, + min: 1, + max: 10_000, + }, +]; + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function bounded(value: unknown, fallback: number, min: number, max: number): number { + return typeof value === "number" && Number.isFinite(value) + ? Math.min(max, Math.max(min, Math.floor(value))) + : fallback; +} + +function mergeConfig(options?: CompressionEngineApplyOptions): CodexResponsesConfig { + const source = options?.config?.codexResponsesConfig ?? {}; + const step = options?.stepConfig ?? {}; + const merged = { ...DEFAULT_CODEX_RESPONSES_CONFIG, ...source, ...step }; + return { + ...DEFAULT_CODEX_RESPONSES_CONFIG, + enabled: merged.enabled !== false, + minBytes: bounded(merged.minBytes, DEFAULT_CODEX_RESPONSES_CONFIG.minBytes, 0, 2_000_000), + maxOutputBytes: bounded( + merged.maxOutputBytes, + DEFAULT_CODEX_RESPONSES_CONFIG.maxOutputBytes, + 1, + 10_000_000 + ), + maxCandidateBytes: bounded( + merged.maxCandidateBytes, + DEFAULT_CODEX_RESPONSES_CONFIG.maxCandidateBytes, + 1, + 2_000_000 + ), + maxLines: bounded(merged.maxLines, DEFAULT_CODEX_RESPONSES_CONFIG.maxLines, 1, 10_000), + minSearchMatches: bounded( + merged.minSearchMatches, + DEFAULT_CODEX_RESPONSES_CONFIG.minSearchMatches, + 2, + 10_000 + ), + minLogLines: bounded(merged.minLogLines, DEFAULT_CODEX_RESPONSES_CONFIG.minLogLines, 2, 10_000), + preserveToolNames: Array.isArray(merged.preserveToolNames) + ? merged.preserveToolNames.filter((name): name is string => typeof name === "string") + : DEFAULT_CODEX_RESPONSES_CONFIG.preserveToolNames, + }; +} + +function minifyJson(text: string): string | null { + try { + const parsed = JSON.parse(text) as unknown; + if (!Array.isArray(parsed) && !isRecord(parsed)) return null; + return JSON.stringify(parsed); + } catch { + return null; + } +} + +function collapseLines(text: string, config: CodexResponsesConfig): string | null { + const lines = text.split(/\r?\n/); + if (lines.length < config.minLogLines) return null; + const important = lines + .map((line, index) => ({ line, index })) + .filter( + ({ line }) => IMPORTANT_LOG_RE.test(line) || /^(?:\s*(?:at\s|File\s|\S+[:(]\d+))/.test(line) + ) + .map(({ index }) => index); + if (important.length === 0 && !BUILD_RE.test(text)) return null; + const keep = new Set(); + for (let index = 0; index < Math.min(8, lines.length); index++) keep.add(index); + for (let index = Math.max(0, lines.length - 8); index < lines.length; index++) keep.add(index); + for (const index of important) { + for (let offset = -2; offset <= 2; offset++) { + if (index + offset >= 0 && index + offset < lines.length) keep.add(index + offset); + } + } + const indexes = [...keep].sort((a, b) => a - b).slice(0, config.maxLines); + const output: string[] = []; + let previous = -1; + for (const index of indexes) { + if (index - previous > 1) + output.push(`[compressed log output: omitted ${index - previous - 1} lines]`); + output.push(lines[index]); + previous = index; + } + if (previous < lines.length - 1) + output.push(`[compressed log output: omitted ${lines.length - previous - 1} lines]`); + return output.join("\n"); +} + +function compactDiff(text: string, config: CodexResponsesConfig): string | null { + if (!DIFF_HUNK_RE.test(text)) return null; + const lines = text.split(/\r?\n/); + const output: string[] = []; + let changedSinceContext = 0; + for (const line of lines) { + const changed = line.startsWith("+") || line.startsWith("-") || line.startsWith("@@"); + if (changed) { + output.push(line); + changedSinceContext = 0; + } else if (changedSinceContext < 2) { + output.push(line); + changedSinceContext++; + } + } + return output.length < lines.length && output.length <= config.maxLines * 4 + ? output.join("\n") + : null; +} + +function compactSearch(text: string, config: CodexResponsesConfig): string | null { + const lines = text.split(/\r?\n/); + const matches = lines.filter((line) => SEARCH_LINE_RE.test(line)); + if (matches.length < config.minSearchMatches) return null; + const grouped = new Map(); + for (const line of matches) { + const path = line.trim().split(/:\d/)[0].split(/-\d/)[0]; + const group = grouped.get(path) ?? []; + if (group.length < 3) group.push(line.trim()); + grouped.set(path, group); + } + const output = [...grouped.entries()] + .map(([path, entries]) => [path, ...entries].join("\n")) + .join("\n"); + return output.length < text.length ? output : null; +} + +function candidateRewrite(text: string, config: CodexResponsesConfig): string | null { + if (Buffer.byteLength(text, "utf8") < config.minBytes) return null; + if (Buffer.byteLength(text, "utf8") > config.maxCandidateBytes) return null; + const json = minifyJson(text); + const diff = compactDiff(text, config); + const search = compactSearch(text, config); + const log = collapseLines(text, config); + return ( + [json, diff, search, log] + .filter((value): value is string => typeof value === "string") + .sort((a, b) => Buffer.byteLength(a) - Buffer.byteLength(b))[0] ?? null + ); +} + +function rewriteText(value: unknown, config: CodexResponsesConfig): string | null { + if (typeof value !== "string") return null; + if (Buffer.byteLength(value, "utf8") > config.maxOutputBytes) return null; + const rewritten = candidateRewrite(value, config); + if (!rewritten || Buffer.byteLength(rewritten) >= Buffer.byteLength(value)) return null; + if (countCodexTokens(rewritten) >= countCodexTokens(value)) { + return null; + } + return rewritten; +} + +function hasLossyRewriteCandidate(text: unknown, config: CodexResponsesConfig): boolean { + if (typeof text !== "string") return false; + return ( + compactDiff(text, config) !== null || + compactSearch(text, config) !== null || + collapseLines(text, config) !== null + ); +} + +function protectedByConfig(meta: { type: string; eligible: boolean }): boolean { + return !SUPPORTED_TYPES.has(meta.type) || !meta.eligible; +} + +export const codexResponsesEngine: CompressionEngine = { + id: ENGINE_ID, + name: "Responses Tool Output", + description: "Conservative lossless-first compression for supported Responses tool outputs.", + icon: "data_object", + targets: ["tool_results"], + stackable: true, + stackPriority: 12, + metadata: { + id: ENGINE_ID, + name: "Responses Tool Output", + description: "Compresses eligible shell, patch, search, build, and JSON outputs.", + inputScope: "tool-results", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options): CompressionResult { + const config = mergeConfig(options); + if (!config.enabled || !Array.isArray(body.messages)) { + return { body, compressed: false, stats: null }; + } + let changed = false; + const messages = body.messages.map((raw) => { + if (!isRecord(raw) || raw.role !== "tool") return raw; + const meta = (raw as Record)[CODEX_RESPONSE_ITEM_META] as + { type?: unknown; eligible?: unknown } | undefined; + if (!meta || typeof meta.type !== "string" || meta.eligible !== true) return raw; + const typeMeta = { type: meta.type, eligible: true }; + if (protectedByConfig(typeMeta)) return raw; + if ( + meta.type === "local_shell_call_output" && + hasLossyRewriteCandidate(raw.content, config) + ) { + return raw; + } + const next = rewriteText(raw.content, config); + if (next === null) return raw; + changed = true; + return { ...raw, content: next }; + }); + if (!changed) return { body, compressed: false, stats: null }; + const nextBody = { ...body, messages }; + const stats = createCompressionStats(body, nextBody, "codex-responses", [ENGINE_ID]); + const originalTokens = countCodexTokens(JSON.stringify(body)); + const compressedTokens = countCodexTokens(JSON.stringify(nextBody)); + stats.originalTokens = originalTokens; + stats.compressedTokens = compressedTokens; + stats.savingsPercent = + originalTokens > 0 + ? Math.round(((originalTokens - compressedTokens) / originalTokens) * 10000) / 100 + : 0; + return { body: nextBody, compressed: stats.compressedTokens < stats.originalTokens, stats }; + }, + compress(body, config): CompressionResult { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema(): EngineConfigField[] { + return CODEX_SCHEMA; + }, + validateConfig(config): EngineValidationResult { + const errors: string[] = []; + for (const [key, min, max] of [ + ["minBytes", 0, 2_000_000], + ["maxOutputBytes", 1, 10_000_000], + ["maxCandidateBytes", 1, 2_000_000], + ["maxLines", 1, 10_000], + ["minSearchMatches", 2, 10_000], + ["minLogLines", 2, 10_000], + ] as const) { + if ( + config[key] !== undefined && + (typeof config[key] !== "number" || config[key] < min || config[key] > max) + ) { + errors.push(`${key} must be between ${min} and ${max}`); + } + } + if (config.enabled !== undefined && typeof config.enabled !== "boolean") + errors.push("enabled must be boolean"); + return { valid: errors.length === 0, errors }; + }, +}; diff --git a/open-sse/services/compression/engines/index.ts b/open-sse/services/compression/engines/index.ts index 02ba4729a4..05a1716b42 100644 --- a/open-sse/services/compression/engines/index.ts +++ b/open-sse/services/compression/engines/index.ts @@ -10,6 +10,7 @@ import { relevanceEngine } from "./relevance/index.ts"; import { llmCompressorEngine } from "./llm/index.ts"; import { readLifecycleEngine } from "./readLifecycle/index.ts"; import { omniglyphEngine } from "./omniglyphAdapter.ts"; +import { codexResponsesEngine } from "./codexResponses/index.ts"; let registered = false; @@ -27,6 +28,7 @@ export function registerBuiltinCompressionEngines(): void { { id: "aggressive", engine: aggressiveEngine }, { id: "ultra", engine: ultraEngine }, { id: "rtk", engine: rtkEngine }, + { id: "codex-responses", engine: codexResponsesEngine }, { id: "session-dedup", engine: sessionDedupEngine }, { id: "headroom", engine: headroomEngine }, { id: "ccr", engine: ccrEngine }, diff --git a/open-sse/services/compression/index.ts b/open-sse/services/compression/index.ts index 8ef9a581b5..97882eb2f2 100644 --- a/open-sse/services/compression/index.ts +++ b/open-sse/services/compression/index.ts @@ -10,6 +10,7 @@ export type { RtkConfig, RtkIntensity, RtkRawOutputRetention, + CodexResponsesConfig, CompressionEngineId, CompressionLanguageConfig, CompressionPipelineStep, @@ -27,6 +28,7 @@ export { DEFAULT_RTK_CONFIG, DEFAULT_COMPRESSION_LANGUAGE_CONFIG, DEFAULT_AGGRESSIVE_CONFIG, + DEFAULT_CODEX_RESPONSES_CONFIG, } from "./types.ts"; export { @@ -113,6 +115,7 @@ export { clearCompressionEngineRegistry, } from "./engines/registry.ts"; export { registerBuiltinCompressionEngines } from "./engines/index.ts"; +export { codexResponsesEngine } from "./engines/codexResponses/index.ts"; export { applyRtkCompression, processRtkText, rtkEngine } from "./engines/rtk/index.ts"; export { diff --git a/open-sse/services/compression/liveZone.ts b/open-sse/services/compression/liveZone.ts index d0c69cd03a..9d159b548f 100644 --- a/open-sse/services/compression/liveZone.ts +++ b/open-sse/services/compression/liveZone.ts @@ -111,6 +111,8 @@ function isToolOutputItem(value: unknown): boolean { item.role === "function" || item.role === "tool_result" || item.type === "function_call_output" || + item.type === "local_shell_call_output" || + item.type === "apply_patch_call_output" || item.type === "computer_call_output" || item.type === "tool_result" ); diff --git a/open-sse/services/compression/strategySelector.ts b/open-sse/services/compression/strategySelector.ts index e95581b814..336a982624 100644 --- a/open-sse/services/compression/strategySelector.ts +++ b/open-sse/services/compression/strategySelector.ts @@ -30,6 +30,7 @@ import { } from "./stackedStepCore.ts"; import { registerBuiltinCompressionEngines } from "./engines/index.ts"; import { getCompressionEngine, getEngineEntry } from "./engines/registry.ts"; +import { codexResponsesEngine } from "./engines/codexResponses/index.ts"; import { applyOmniglyphSingleMode } from "./engines/omniglyphSingleMode.ts"; import { applyRtkCompression } from "./engines/rtk/index.ts"; import { adaptBodyForCompression } from "./bodyAdapter.ts"; @@ -322,11 +323,26 @@ function runCompression( config: { ...(options?.config?.rtkConfig ?? {}), enabled: true }, }); } + if (mode === "codex-responses") { + const adapter = adaptBodyForCompression( + body, + options?.config?.codexResponsesConfig?.preserveToolNames + ); + const result = codexResponsesEngine.apply(adapter.body, { + ...options, + config: options?.config, + stepConfig: { enabled: true }, + }); + return adapter.adapted ? { ...result, body: adapter.restore(result.body) } : result; + } if (mode === "omniglyph") { // omniglyph is async-only — use applyCompressionAsync. Safe no-op here. return { body, compressed: false, stats: null }; } - const adapter = adaptBodyForCompression(body); + const adapter = adaptBodyForCompression( + body, + options?.config?.codexResponsesConfig?.preserveToolNames + ); const compressionBody = adapter.body; if (mode === "lite") { const result = applyLiteCompression(compressionBody, { @@ -501,7 +517,10 @@ async function runCompressionAsync( // Single-mode omniglyph (async-only) — resolution lives in engines/omniglyphSingleMode.ts. if (mode === "omniglyph") return applyOmniglyphSingleMode(body, options); if (mode === "stacked") { - const adapter = adaptBodyForCompression(body); + const adapter = adaptBodyForCompression( + body, + options?.config?.codexResponsesConfig?.preserveToolNames + ); const result = await applyStackedCompressionAsync( adapter.body, options?.config?.stackedPipeline, @@ -545,7 +564,10 @@ async function applyUltraAsync( // config.ultraEngine === "slm" and the worker backend is available). This is the // Phase-4 (B) path; it fail-opens to the heuristic and records the resolved tier. if (!modelPath) { - const adapter = adaptBodyForCompression(body); + const adapter = adaptBodyForCompression( + body, + options?.config?.codexResponsesConfig?.preserveToolNames + ); const messages = (adapter.body.messages ?? []) as Array<{ role: string; content?: string | unknown[]; @@ -622,7 +644,7 @@ async function applyUltraAsync( function normalizePipelineStep(step: CompressionPipelineStep | string): CompressionPipelineStep { if (typeof step !== "string") return step; if (step === "standard") return { engine: "caveman" }; - if (step === "rtk") return { engine: "rtk" }; + if (step === "rtk" || step === "codex-responses") return { engine: step }; if (step === "lite" || step === "aggressive" || step === "ultra") return { engine: step }; return { engine: "caveman" }; } @@ -714,14 +736,22 @@ function buildStepOptions( step: CompressionPipelineStep, options?: StackOptions ): CompressionEngineApplyOptions { + const stepConfig: Record = { + ...(step.config ?? {}), + ...(step.intensity ? { intensity: step.intensity } : {}), + }; + // Selecting an engine in an explicit stacked pipeline is itself the enablement + // signal. Preserve an explicit per-step opt-out, but do not let the standalone + // default (codexResponsesConfig.enabled=false) turn a selected stacked step into + // a no-op. + if (step.engine === "codex-responses" && stepConfig.enabled === undefined) { + stepConfig.enabled = true; + } return { ...options, compressionComboId: options?.compressionComboId ?? options?.config?.compressionComboId, principalId: options?.principalId, - stepConfig: { - ...(step.config ?? {}), - ...(step.intensity ? { intensity: step.intensity } : {}), - }, + stepConfig, }; } diff --git a/open-sse/services/compression/types.ts b/open-sse/services/compression/types.ts index ff8aedd77b..2e18c4b0ce 100644 --- a/open-sse/services/compression/types.ts +++ b/open-sse/services/compression/types.ts @@ -6,7 +6,7 @@ * Phase 2: 'standard' mode (caveman engine). * Phase 3: 'aggressive' mode (summarization + tool compression + aging). * Phase 4: 'ultra' mode (heuristic token pruning + optional SLM tier). - * Phase 5: 'rtk' and 'stacked' modes (tool-output filters + multi-engine pipeline). + * Phase 5: 'rtk', 'codex-responses', and 'stacked' modes (tool-output filters + multi-engine pipeline). */ import { ENGINE_IDS } from "./engineCatalog.ts"; @@ -25,7 +25,15 @@ import type { QuantumLockConfig, QuantumLockStats } from "./quantumLock/quantumP export { ENGINE_IDS }; export type CompressionMode = - "off" | "lite" | "standard" | "aggressive" | "ultra" | "rtk" | "omniglyph" | "stacked"; + | "off" + | "lite" + | "standard" + | "aggressive" + | "ultra" + | "rtk" + | "codex-responses" + | "omniglyph" + | "stacked"; export type CavemanIntensity = "lite" | "full" | "ultra"; export type RtkIntensity = "minimal" | "standard" | "aggressive"; export type RtkRawOutputRetention = "never" | "failures" | "always"; @@ -40,7 +48,8 @@ export type CompressionEngineId = | "ccr" | "llmlingua" | "relevance" - | "omniglyph"; + | "omniglyph" + | "codex-responses"; export interface CavemanRule { name: string; @@ -107,6 +116,18 @@ export interface RtkConfig { renderers?: string[]; } +/** Conservative, lossless-first Responses tool-output compression controls. */ +export interface CodexResponsesConfig { + enabled: boolean; + minBytes: number; + maxOutputBytes: number; + maxCandidateBytes: number; + maxLines: number; + minSearchMatches: number; + minLogLines: number; + preserveToolNames: string[]; +} + export interface RelevanceConfig { enabled: boolean; overlapThreshold: number; @@ -192,6 +213,7 @@ export interface CompressionConfig { /** Phase 4A: selected output styles (supersedes cavemanOutputMode via a back-compat shim). */ outputStyles?: OutputStyleSelectionEntry[]; rtkConfig?: RtkConfig; + codexResponsesConfig?: CodexResponsesConfig; relevanceConfig?: RelevanceConfig; languageConfig?: CompressionLanguageConfig; aggressive?: AggressiveConfig; @@ -314,6 +336,32 @@ export interface CompressionResult { stats: CompressionStats | null; } +export const DEFAULT_CODEX_RESPONSES_CONFIG: CodexResponsesConfig = { + enabled: false, + minBytes: 512, + maxOutputBytes: 2 * 1024 * 1024, + maxCandidateBytes: 512 * 1024, + maxLines: 160, + minSearchMatches: 8, + minLogLines: 24, + preserveToolNames: [ + "Read", + "Glob", + "Grep", + "Write", + "Edit", + "WebSearch", + "WebFetch", + "read", + "glob", + "grep", + "write", + "edit", + "web_search", + "web_fetch", + ], +}; + export const DEFAULT_COMPRESSION_CONFIG: CompressionConfig = { enabled: false, defaultMode: "off", @@ -334,6 +382,7 @@ export const DEFAULT_COMPRESSION_CONFIG: CompressionConfig = { ultraEngine: "heuristic", ultraSlmPrewarm: false, liveZone: { enabled: false }, + codexResponsesConfig: { ...DEFAULT_CODEX_RESPONSES_CONFIG }, }; export const DEFAULT_CAVEMAN_CONFIG: CavemanConfig = { diff --git a/src/app/(dashboard)/dashboard/context/combos/CompressionHub.tsx b/src/app/(dashboard)/dashboard/context/combos/CompressionHub.tsx index 4eae132de0..6596764e33 100644 --- a/src/app/(dashboard)/dashboard/context/combos/CompressionHub.tsx +++ b/src/app/(dashboard)/dashboard/context/combos/CompressionHub.tsx @@ -12,7 +12,8 @@ import { useTranslations } from "next-intl"; // ── Types ───────────────────────────────────────────────────────────────────── -type CompressionMode = "off" | "lite" | "standard" | "aggressive" | "ultra" | "rtk" | "stacked"; +type CompressionMode = + "off" | "lite" | "standard" | "aggressive" | "ultra" | "rtk" | "codex-responses" | "stacked"; interface CompressionSettings { enabled: boolean; diff --git a/src/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab.tsx index da4b993bd7..5c92027b05 100644 --- a/src/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab.tsx @@ -7,7 +7,8 @@ import CompressionTokenSaverCard, { type CompressionTokenSaverConfig, } from "./CompressionTokenSaverCard"; -type CompressionMode = "off" | "lite" | "standard" | "aggressive" | "ultra" | "rtk" | "stacked"; +type CompressionMode = + "off" | "lite" | "standard" | "aggressive" | "ultra" | "rtk" | "codex-responses" | "stacked"; type CavemanIntensity = "lite" | "full" | "ultra"; type RtkIntensity = "minimal" | "standard" | "aggressive"; @@ -31,6 +32,17 @@ interface RtkConfig { intensity: RtkIntensity; } +interface CodexResponsesConfig { + enabled: boolean; + minBytes: number; + maxOutputBytes: number; + maxCandidateBytes: number; + maxLines: number; + minSearchMatches: number; + minLogLines: number; + preserveToolNames: string[]; +} + interface AggressiveConfig { thresholds: { fullSummary: number; @@ -71,6 +83,7 @@ interface CompressionConfig extends CompressionTokenSaverConfig { cavemanConfig?: CavemanConfig; cavemanOutputMode?: CavemanOutputModeConfig; rtkConfig?: RtkConfig; + codexResponsesConfig?: CodexResponsesConfig; aggressive?: AggressiveConfig; ultra?: UltraConfig; } @@ -121,6 +134,12 @@ const MODES: { value: CompressionMode; labelKey: string; descKey: string; icon: descKey: "compressionModeRtkDesc", icon: "filter_list", }, + { + value: "codex-responses", + labelKey: "compressionModeCodexResponses", + descKey: "compressionModeCodexResponsesDesc", + icon: "data_object", + }, { value: "stacked", labelKey: "compressionModeStacked", @@ -161,6 +180,16 @@ export default function CompressionSettingsTab() { enabled: true, intensity: "standard", }, + codexResponsesConfig: { + enabled: false, + minBytes: 512, + maxOutputBytes: 2 * 1024 * 1024, + maxCandidateBytes: 512 * 1024, + maxLines: 160, + minSearchMatches: 8, + minLogLines: 24, + preserveToolNames: ["Read", "Glob", "Grep", "Write", "Edit", "WebSearch", "WebFetch"], + }, aggressive: { thresholds: { fullSummary: 5, moderate: 3, light: 2, verbatim: 2 }, toolStrategies: { @@ -380,10 +409,7 @@ export default function CompressionSettingsTab() { } onChange={(e) => save({ - preserveSystemPromptMode: e.target.value as - | "always" - | "whenNoCache" - | "never", + preserveSystemPromptMode: e.target.value as "always" | "whenNoCache" | "never", }) } className="w-36 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main" @@ -439,92 +465,90 @@ export default function CompressionSettingsTab() { <> -
-

{t("compressionRoles")}

-
- {ROLE_OPTIONS.map((opt) => ( - - ))} -
+
+

{t("compressionRoles")}

+
+ {ROLE_OPTIONS.map((opt) => ( + + ))}
+
- + - {/* Caveman intensity (level) is set in the panel + {/* Caveman intensity (level) is set in the panel (/dashboard/context/settings); kept out of this tab to avoid a duplicate level control. */} -
-

{t("compressionSkipRules")}

-

{t("compressionSkipRulesDesc")}

-
- {ruleMetadata.map((rule) => ( - - ))} -
+
+

{t("compressionSkipRules")}

+

{t("compressionSkipRulesDesc")}

+
+ {ruleMetadata.map((rule) => ( + + ))}
+
-
-

{t("compressionPreservePatterns")}

-

- {t("compressionPreservePatternsDesc")} -

-