diff --git a/.agents/workflows/deploy-vps-akamai.md b/.agents/workflows/deploy-vps-akamai.md index 50f92cab5f..b25ae3cd99 100644 --- a/.agents/workflows/deploy-vps-akamai.md +++ b/.agents/workflows/deploy-vps-akamai.md @@ -17,7 +17,7 @@ Deploy OmniRoute to the Akamai VPS using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to Akamai VPS and install diff --git a/.agents/workflows/deploy-vps-both.md b/.agents/workflows/deploy-vps-both.md index 161d197bd4..e1aa4d1def 100644 --- a/.agents/workflows/deploy-vps-both.md +++ b/.agents/workflows/deploy-vps-both.md @@ -22,7 +22,7 @@ Deploy OmniRoute to the production VPSs using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to both VPS and install diff --git a/.agents/workflows/deploy-vps-local.md b/.agents/workflows/deploy-vps-local.md index f348f63da5..549b1f0b2a 100644 --- a/.agents/workflows/deploy-vps-local.md +++ b/.agents/workflows/deploy-vps-local.md @@ -17,7 +17,7 @@ Deploy OmniRoute to the Local VPS using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to Local VPS and install diff --git a/.agents/workflows/generate-release.md b/.agents/workflows/generate-release.md index e127c832d4..399e12d214 100644 --- a/.agents/workflows/generate-release.md +++ b/.agents/workflows/generate-release.md @@ -226,7 +226,7 @@ git checkout main git pull origin main # Build and pack locally -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts # Deploy to LOCAL VPS (192.168.0.15) scp omniroute-*.tgz root@192.168.0.15:/tmp/ diff --git a/.agents/workflows/version-bump.md b/.agents/workflows/version-bump.md index 4b3b77a921..0667e0eb1d 100644 --- a/.agents/workflows/version-bump.md +++ b/.agents/workflows/version-bump.md @@ -20,7 +20,7 @@ Automatically bump the project version, generate CHANGELOG entries from git hist // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute CURRENT_VERSION=$(node -p "require('./package.json').version") LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") CURRENT_BRANCH=$(git branch --show-current) @@ -64,7 +64,7 @@ npm version "$VERSION" --no-git-tag-version // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null) echo "=== Commits since $LAST_TAG ===" git log "$LAST_TAG"..HEAD --pretty=format:"%h %s" --no-merges | head -100 @@ -133,7 +133,7 @@ The date must be today's date in `YYYY-MM-DD` format. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") # Update docs/openapi.yaml version @@ -156,7 +156,7 @@ echo "✓ All workspace packages synced to $VERSION" // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") OLD_VERSION_PATTERN='[0-9]\+\.[0-9]\+\.[0-9]\+' @@ -174,7 +174,7 @@ echo "✓ llm.txt → $VERSION" // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm install echo "✓ Lock file regenerated" ``` @@ -236,7 +236,7 @@ For each file in `docs/` (excluding `docs/i18n/`), review if CHANGELOG changes a // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm run lint ``` @@ -245,7 +245,7 @@ npm run lint // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm test ``` @@ -254,7 +254,7 @@ npm test // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") echo "Expected version: $VERSION" echo "" @@ -299,7 +299,7 @@ grep "^## \[" CHANGELOG.md | head -2 // turbo-all ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute git add -A VERSION=$(node -p "require('./package.json').version") git commit -m "chore(release): bump to v$VERSION — changelog, docs, version sync" diff --git a/AGENTS.md b/AGENTS.md index 3b05a3dd65..aadaf98b2e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Unified AI proxy/router — route any LLM through one endpoint. Multi-provider s with **160+ providers** (OpenAI, Anthropic, Gemini, DeepSeek, Groq, xAI, Mistral, Fireworks, Cohere, NVIDIA, Cerebras, Pollinations, Puter, Cloudflare AI, HuggingFace, DeepInfra, SambaNova, Meta Llama API, Moonshot AI, AI21 Labs, Databricks, Snowflake, and many more) -with **MCP Server** (29 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. +with **MCP Server** (37 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. ## Stack @@ -292,17 +292,28 @@ compression pipeline), and more. Modular prompt compression that runs proactively before the existing reactive context manager. -- **`strategySelector.ts`**: Selects compression mode based on config, combo overrides, auto-trigger - thresholds. Priority: combo override > auto-trigger > default mode > off. +- **`strategySelector.ts`**: Selects compression mode based on config, compression combo assignments, + combo overrides, auto-trigger thresholds, and defaults. Priority: assigned compression combo > + combo override > auto-trigger > default mode > off. - **`lite.ts`**: 5 lite-mode techniques: `collapseWhitespace`, `dedupSystemPrompt`, `compressToolResults`, `removeRedundantContent`, `replaceImageUrls`. Target: 10-15% savings at <1ms latency. +- **`caveman.ts` / `cavemanRules.ts`**: Caveman-style semantic condensation backed by built-in + rules plus file-loaded language packs under `compression/rules/`. +- **`engines/rtk/`**: Rule-based terminal/tool-output compression inspired by RTK patterns. Detects + command output classes, applies JSON filter packs, deduplicates repeated lines, strips ANSI/code + noise, and preserves errors/actionable context. The RTK JSON DSL supports replace, + match-output short-circuit, strip/keep, per-line truncation, head/tail/max-line truncation, + inline tests, trust-gated project/global custom filters, and optional redacted raw-output + retention for authenticated recovery. +- **`engines/registry.ts`**: Registers engines (`caveman`, `rtk`) and powers stacked pipelines. - **`stats.ts`**: Per-request compression stats tracking (original tokens, compressed tokens, - savings %, techniques used). -- **`types.ts`**: `CompressionMode` (off/lite/standard/aggressive/ultra), `CompressionConfig`, - `CompressionStats`, `CompressionResult`. -- DB settings in `src/lib/db/compression.ts`, API route at `src/app/api/settings/compression/`. -- Phase 1 implements lite mode only; standard/aggressive/ultra are placeholders for Phase 2. + savings %, techniques used, engine breakdown, compression combo id). +- **`types.ts`**: `CompressionMode` (off/lite/standard/aggressive/ultra/rtk/stacked), + `CompressionConfig`, `CompressionStats`, `CompressionResult`. +- DB settings in `src/lib/db/compression.ts`, compression combos in + `src/lib/db/compressionCombos.ts`, API routes under `src/app/api/settings/compression/`, + `src/app/api/context/*`, and preview/language-pack routes under `src/app/api/compression/*`. #### Combo Routing Engine (`combo.ts`) @@ -323,7 +334,7 @@ Policy engine modules: `policyEngine.ts`, `comboResolver.ts`, `costRules.ts`, ### MCP Server (`open-sse/mcp-server/`) -29 tools, 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (10 scopes), Zod schemas. +37 tools, 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (10 scopes), Zod schemas. **Core tools** (20): get_health, list_combos, get_combo_metrics, switch_combo, check_quota, route_request, cost_report, list_models_catalog, web_search, simulate_route, set_budget_guard, @@ -332,6 +343,11 @@ best_combo_for_task, explain_route, get_session_snapshot, db_health_check, sync_ **Cache tools** (2): cache_stats, cache_flush. +**Compression tools** (5): compression_status, compression_configure, set_compression_engine, +list_compression_combos, compression_combo_stats. + +**1proxy tools** (3): oneproxy_fetch, oneproxy_rotate, oneproxy_stats. + **Memory tools** (3): memory_search, memory_add, memory_clear. **Skill tools** (4): skills_list, skills_enable, skills_execute, skills_executions. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4707bf82ee..4576402473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased] +### ✨ New Features + +- **feat(compression):** add RTK tool-output compression, stacked Caveman + RTK pipelines, + compression combo assignments, dashboard context pages, MCP management tools, and + language-aware Caveman rule packs. +- **feat(compression):** expand RTK parity with a 39-filter catalog, RTK-style JSON DSL stages, + inline verify/benchmark coverage, trust-gated custom filters, expanded command detection, and + redacted raw-output recovery. + --- ## [3.7.9] — 2026-05-02 diff --git a/README.md b/README.md index 49c38eb156..c80fde019e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### Never stop coding. Save 15-75% tokens with prompt compression + auto-fallback to **FREE & low-cost AI models**. -_The most complete open-source AI proxy — **one endpoint**, **160+ providers**, **13 routing strategies**, zero downtime. Multi-platform: **Web**, **Desktop (Electron)**, **Mobile (PWA + Termux)**. Fully extensible via **MCP Server (29 tools)**, **A2A Protocol**, and **Memory/Skills** systems. Available in **40+ languages**._ +_The most complete open-source AI proxy — **one endpoint**, **160+ providers**, **13 routing strategies**, zero downtime. Multi-platform: **Web**, **Desktop (Electron)**, **Mobile (PWA + Termux)**. Fully extensible via **MCP Server (37 tools)**, **A2A Protocol**, and **Memory/Skills** systems. Available in **40+ languages**._ **Chat Completions • Responses API • Embeddings • Image Generation • Video • Music • Audio Speech/Transcription • Reranking • Moderations • Web Search • MCP Server • A2A Protocol • 4,600+ Tests • 100% TypeScript** @@ -440,16 +440,18 @@ Every request passes through the compression pipeline **transparently** — no c ``` ┌──────────────────┐ ┌─────────────────────────────┐ ┌──────────────┐ │ Client sends │────▶│ OmniRoute Compression │────▶│ Provider │ -│ full prompt │ │ Pipeline (5 modes) │ │ receives │ +│ full prompt │ │ Pipeline (7 options) │ │ receives │ │ (10,000 tok) │ │ │ │ compressed │ │ │ │ 🪶 Lite ........... ~15% │ │ (2,500 tok) │ │ │ │ 🪨 Standard ....... ~30% │ │ │ │ │ │ ⚡ Aggressive ..... ~50% │ │ 💰 75% saved │ │ │ │ 🔥 Ultra .......... ~75% │ │ │ +│ │ │ 🧰 RTK ............ 20-70% │ │ │ +│ │ │ 🔗 Stacked ........ 30-80% │ │ │ └──────────────────┘ └─────────────────────────────┘ └──────────────┘ ``` -### 5 Compression Modes +### 7 Compression Options | Mode | Savings | Technique | Best For | | ------------------------- | ------- | ----------------------------------------------------------------------------------------------- | -------------------------------------- | @@ -458,6 +460,8 @@ Every request passes through the compression pipeline **transparently** — no c | **🪨 Standard (Caveman)** | ~30% | 30+ regex rules: filler removal, context condensation, structural compression, multi-turn dedup | Daily coding with Claude/Codex | | **⚡ Aggressive** | ~50% | All standard + progressive message aging + tool result summarization + LLM-based compression | Long sessions with many tool calls | | **🔥 Ultra** | ~75% | All aggressive + heuristic token pruning + stopword removal + score-based filtering | Maximum savings when tokens are scarce | +| **🧰 RTK** | 20-70% | 39 command-aware filters, RTK-style JSON DSL, verify gate, trust-gated custom filters | Shell/test/build/git output in agents | +| **🔗 Stacked** | 30-80% | Ordered engine pipeline, usually RTK first then Caveman | Mixed prompts with tool logs + prose | ### Before & After (Standard/Caveman Mode) @@ -481,6 +485,8 @@ Request Body ├─ lite.ts ─────────────── Whitespace, dedup, image URLs, redundant content ├─ caveman.ts ──────────── 30+ regex rules via cavemanRules.ts │ └─ preservation.ts ─── Protects code blocks, URLs, JSON from compression + ├─ engines/rtk/ ────────── Command detection + JSON DSL filters + raw-output recovery + ├─ engines/registry.ts ─── Shared engine registry for caveman, RTK, and stacked ├─ aggressive.ts ───────── Summarizer + tool result compressor + progressive aging │ ├─ summarizer.ts ───── Rule-based message summarization │ ├─ toolResultCompressor.ts ── file/grep/shell/JSON/error compression @@ -492,7 +498,7 @@ Request Body ### Configuration ``` -Dashboard → Settings → Compression → Pick your mode +Dashboard → Context & Cache → Caveman / RTK / Compression Combos ``` Or per-combo override: @@ -508,9 +514,11 @@ Or per-combo override: Auto-trigger: set `autoTriggerTokens` to automatically enable compression when a request exceeds a token threshold. -> 🪨 **Fun fact:** The standard/caveman mode is inspired by [Caveman](https://github.com/JuliusBrussee/caveman) — the viral project that proved "caveman speak" cuts 65% of tokens while keeping 100% technical accuracy. OmniRoute takes this further with a **5-mode pipeline** that goes from gentle whitespace cleanup all the way to aggressive heuristic pruning. +Compression combos can also assign a named compression pipeline to routing combos, so a coding combo can use RTK + Caveman while a paid subscription combo stays on lite mode. -📖 **Full compression documentation:** [`docs/COMPRESSION_GUIDE.md`](docs/COMPRESSION_GUIDE.md) +> 🪨 **Fun fact:** The standard/caveman mode is inspired by [Caveman](https://github.com/JuliusBrussee/caveman) — the viral project that proved "caveman speak" cuts 65% of tokens while keeping 100% technical accuracy. OmniRoute takes this further with a **7-option pipeline** that goes from gentle whitespace cleanup through RTK tool-output filters and stacked multi-engine compression. + +📖 **Full compression documentation:** [`docs/COMPRESSION_GUIDE.md`](docs/COMPRESSION_GUIDE.md) • [`docs/rtk-compression.md`](docs/rtk-compression.md) • [`docs/compression-engines.md`](docs/compression-engines.md) • [`docs/compression-rules-format.md`](docs/compression-rules-format.md) • [`docs/compression-language-packs.md`](docs/compression-language-packs.md) --- @@ -933,8 +941,8 @@ Then in `/dashboard/media` → **Transcription** tab: upload any audio or video | ---------------------------------------------------------------------------------------------------- | -------------------------------- | | 🧠 **Smart 4-Tier Fallback** — Subscription → API → Cheap → Free | Never stop coding, zero downtime | | 🔄 **Format Translation** — OpenAI ↔ Claude ↔ Gemini ↔ Responses API | Works with ANY CLI tool | -| 🗜️ **Prompt Compression** — 5-mode pipeline (off → lite → standard → aggressive → ultra) | Save 15-75% tokens automatically | -| 🤖 **MCP Server** — 29 tools, 3 transports (stdio/SSE/HTTP), 10 scopes | IDE/agent tool integration | +| 🗜️ **Prompt Compression** — 7 options including Caveman, RTK, and stacked pipelines | Save 15-75% tokens automatically | +| 🤖 **MCP Server** — 37 tools, 3 transports (stdio/SSE/HTTP), 10 scopes | IDE/agent tool integration | | 🛡️ **Resilience Engine** — circuit breakers, cooldowns, TLS spoofing, anti-thundering herd | Auto-recovery from any failure | | 🎵 **10 Multi-Modal APIs** — chat, embed, images, video, music, TTS, STT, moderation, rerank, search | One endpoint for everything | | 🌍 **3-Level Proxy** — global, per-provider, per-key + 1proxy free marketplace | Access AI from any country | @@ -1142,7 +1150,7 @@ curl http://localhost:20128/.well-known/agent.json - [User Guide](docs/USER_GUIDE.md) — Providers, combos, CLI integration - [API Reference](docs/API_REFERENCE.md) — All endpoints with examples -- [MCP Server](open-sse/mcp-server/README.md) — 29 tools, IDE configs +- [MCP Server](open-sse/mcp-server/README.md) — 37 tools, IDE configs - [A2A Server](src/lib/a2a/README.md) — JSON-RPC, skills, streaming - [Environment Config](docs/ENVIRONMENT.md) — Complete `.env` reference - [VM Deployment](docs/VM_DEPLOYMENT_GUIDE.md) — VM + nginx + Cloudflare @@ -1325,16 +1333,20 @@ See the [Proxy Guide](docs/PROXY_GUIDE.md) for setup instructions. ### 🧠 Features & Architecture -| Document | Description | -| -------------------------------------------------------- | ---------------------------------------------------------------------- | -| [Architecture](docs/ARCHITECTURE.md) | System architecture, data flow, and internals | -| [Compression Guide](docs/COMPRESSION_GUIDE.md) | 5-mode pipeline: off / lite / standard / aggressive / ultra | -| [Resilience Guide](docs/RESILIENCE_GUIDE.md) | Circuit breakers, cooldowns, queue, anti-thundering herd, TLS spoofing | -| [Auto-Combo Engine](docs/AUTO-COMBO.md) | 6-factor scoring, mode packs, self-healing | -| [Proxy Guide](docs/PROXY_GUIDE.md) | 3-level proxy system, 1proxy marketplace, registry CRUD | -| [Free Tiers](docs/FREE_TIERS.md) | 25+ free API providers consolidated directory | -| [Features Gallery](docs/FEATURES.md) | Visual dashboard tour with screenshots | -| [Codebase Documentation](docs/CODEBASE_DOCUMENTATION.md) | Beginner-friendly codebase walkthrough | +| Document | Description | +| ---------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| [Architecture](docs/ARCHITECTURE.md) | System architecture, data flow, and internals | +| [Compression Guide](docs/COMPRESSION_GUIDE.md) | 7-option pipeline: off / lite / standard / aggressive / ultra / RTK / stacked | +| [RTK Compression](docs/rtk-compression.md) | Command-output compression, filters, trust, verify, raw-output recovery | +| [Compression Engines](docs/compression-engines.md) | Caveman, RTK, stacked pipelines, dashboard/API/MCP surfaces | +| [Compression Rules Format](docs/compression-rules-format.md) | JSON rule-pack schemas for Caveman and RTK filters | +| [Compression Language Packs](docs/compression-language-packs.md) | Language detection and Caveman rule-pack authoring | +| [Resilience Guide](docs/RESILIENCE_GUIDE.md) | Circuit breakers, cooldowns, queue, anti-thundering herd, TLS spoofing | +| [Auto-Combo Engine](docs/AUTO-COMBO.md) | 6-factor scoring, mode packs, self-healing | +| [Proxy Guide](docs/PROXY_GUIDE.md) | 3-level proxy system, 1proxy marketplace, registry CRUD | +| [Free Tiers](docs/FREE_TIERS.md) | 25+ free API providers consolidated directory | +| [Features Gallery](docs/FEATURES.md) | Visual dashboard tour with screenshots | +| [Codebase Documentation](docs/CODEBASE_DOCUMENTATION.md) | Beginner-friendly codebase walkthrough | ### 🤖 Protocols & APIs diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 3b4c38ec29..dfec7d1198 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -216,14 +216,32 @@ Response example: ### Settings -| Endpoint | Method | Description | -| ------------------------------- | ------------- | ---------------------- | -| `/api/settings` | GET/PUT/PATCH | General settings | -| `/api/settings/proxy` | GET/PUT | Network proxy config | -| `/api/settings/proxy/test` | POST | Test proxy connection | -| `/api/settings/ip-filter` | GET/PUT | IP allowlist/blocklist | -| `/api/settings/thinking-budget` | GET/PUT | Reasoning token budget | -| `/api/settings/system-prompt` | GET/PUT | Global system prompt | +| Endpoint | Method | Description | +| ------------------------------- | ------------- | ------------------------- | +| `/api/settings` | GET/PUT/PATCH | General settings | +| `/api/settings/proxy` | GET/PUT | Network proxy config | +| `/api/settings/proxy/test` | POST | Test proxy connection | +| `/api/settings/ip-filter` | GET/PUT | IP allowlist/blocklist | +| `/api/settings/thinking-budget` | GET/PUT | Reasoning token budget | +| `/api/settings/system-prompt` | GET/PUT | Global system prompt | +| `/api/settings/compression` | GET/PUT | Global compression config | + +### Context & Compression + +| Endpoint | Method | Description | +| -------------------------------------- | -------------- | ------------------------------------------------------------------------ | +| `/api/compression/preview` | POST | Preview off/lite/standard/aggressive/ultra/RTK/stacked compression | +| `/api/compression/language-packs` | GET | List available Caveman language packs | +| `/api/compression/rules` | GET | List Caveman rule metadata | +| `/api/context/caveman/config` | GET/PUT | Caveman-specific settings alias | +| `/api/context/rtk/config` | GET/PUT | RTK-specific settings, including custom filters and raw-output retention | +| `/api/context/rtk/filters` | GET | RTK filter catalog and custom-filter diagnostics | +| `/api/context/rtk/test` | POST | Run RTK preview/test against a text payload | +| `/api/context/rtk/raw-output/[id]` | GET | Read retained redacted raw output by pointer id | +| `/api/context/combos` | GET/POST | Compression combo list/create | +| `/api/context/combos/[id]` | GET/PUT/DELETE | Compression combo detail/update/delete | +| `/api/context/combos/[id]/assignments` | GET/PUT | Assign compression combos to routing combos | +| `/api/context/analytics` | GET | Compression analytics alias | ### Monitoring @@ -451,11 +469,12 @@ Content-Type: application/json 2. Route handler calls `handleChat`, `handleEmbedding`, `handleAudioTranscription`, or `handleImageGeneration` 3. Model is resolved (direct provider/model or alias/combo) 4. Credentials selected from local DB with account availability filtering -5. For chat: `handleChatCore` — format detection, translation, cache check, idempotency check -6. Provider executor sends upstream request -7. Response translated back to client format (chat) or returned as-is (embeddings/images/audio) -8. Usage/logging recorded -9. Fallback applies on errors according to combo rules +5. For chat: `handleChatCore` checks semantic/signature cache and resolves combo compression settings +6. Proactive compression runs before provider translation when enabled (`lite`, Caveman, RTK, or stacked) +7. Provider executor sends upstream request +8. Response translated back to client format (chat) or returned as-is (embeddings/images/audio) +9. Usage, compression analytics, and request logs are recorded +10. Fallback applies on errors according to combo rules Full architecture reference: [`ARCHITECTURE.md`](ARCHITECTURE.md) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 948004b42a..cd08c672d7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ 🌐 **Languages:** 🇺🇸 [English](ARCHITECTURE.md) | 🇧🇷 [Português (Brasil)](i18n/pt-BR/ARCHITECTURE.md) | 🇪🇸 [Español](i18n/es/ARCHITECTURE.md) | 🇫🇷 [Français](i18n/fr/ARCHITECTURE.md) | 🇮🇹 [Italiano](i18n/it/ARCHITECTURE.md) | 🇷🇺 [Русский](i18n/ru/ARCHITECTURE.md) | 🇨🇳 [中文 (简体)](i18n/zh-CN/ARCHITECTURE.md) | 🇩🇪 [Deutsch](i18n/de/ARCHITECTURE.md) | 🇮🇳 [हिन्दी](i18n/in/ARCHITECTURE.md) | 🇹🇭 [ไทย](i18n/th/ARCHITECTURE.md) | 🇺🇦 [Українська](i18n/uk-UA/ARCHITECTURE.md) | 🇸🇦 [العربية](i18n/ar/ARCHITECTURE.md) | 🇯🇵 [日本語](i18n/ja/ARCHITECTURE.md) | 🇻🇳 [Tiếng Việt](i18n/vi/ARCHITECTURE.md) | 🇧🇬 [Български](i18n/bg/ARCHITECTURE.md) | 🇩🇰 [Dansk](i18n/da/ARCHITECTURE.md) | 🇫🇮 [Suomi](i18n/fi/ARCHITECTURE.md) | 🇮🇱 [עברית](i18n/he/ARCHITECTURE.md) | 🇭🇺 [Magyar](i18n/hu/ARCHITECTURE.md) | 🇮🇩 [Bahasa Indonesia](i18n/id/ARCHITECTURE.md) | 🇰🇷 [한국어](i18n/ko/ARCHITECTURE.md) | 🇲🇾 [Bahasa Melayu](i18n/ms/ARCHITECTURE.md) | 🇳🇱 [Nederlands](i18n/nl/ARCHITECTURE.md) | 🇳🇴 [Norsk](i18n/no/ARCHITECTURE.md) | 🇵🇹 [Português (Portugal)](i18n/pt/ARCHITECTURE.md) | 🇷🇴 [Română](i18n/ro/ARCHITECTURE.md) | 🇵🇱 [Polski](i18n/pl/ARCHITECTURE.md) | 🇸🇰 [Slovenčina](i18n/sk/ARCHITECTURE.md) | 🇸🇪 [Svenska](i18n/sv/ARCHITECTURE.md) | 🇵🇭 [Filipino](i18n/phi/ARCHITECTURE.md) | 🇨🇿 [Čeština](i18n/cs/ARCHITECTURE.md) -_Last updated: 2026-04-15_ +_Last updated: 2026-05-02_ ## Executive Summary @@ -52,12 +52,13 @@ Core capabilities: - Compliance audit logging with opt-out per API key - Eval framework for LLM quality assurance - Health dashboard with real-time provider circuit breaker status -- MCP Server (29 tools) with 3 transports (stdio/SSE/Streamable HTTP) +- MCP Server (37 tools) with 3 transports (stdio/SSE/Streamable HTTP) - A2A Server (JSON-RPC 2.0 + SSE) with skills and task lifecycle - Memory system (extraction, injection, retrieval, summarization) - Skills system (registry, executor, sandbox, built-in skills) - MITM proxy with certificate management and DNS handling - Prompt injection guard middleware +- Prompt compression pipeline with Caveman, RTK, stacked pipelines, compression combos, language packs, and analytics - ACP (Agent Communication Protocol) registry - Modular OAuth providers (13 individual modules under `src/lib/oauth/providers/`) - Uninstall/full-uninstall scripts @@ -112,6 +113,9 @@ Main pages under `src/app/(dashboard)/dashboard/`: - `/dashboard/health` — uptime, circuit breakers, rate limits, quota-monitored sessions - `/dashboard/logs` — request/proxy/audit/console logs - `/dashboard/settings` — system settings tabs (general, routing, combo defaults, etc.) +- `/dashboard/context/caveman` — Caveman compression rules, language packs, preview, and output mode +- `/dashboard/context/rtk` — RTK command-output filters, preview, and runtime safety settings +- `/dashboard/context/combos` — named compression pipelines assigned to routing combos - `/dashboard/api-manager` — API key lifecycle and model permissions ## High-Level System Context @@ -203,6 +207,8 @@ Management domains: - IP filter: `src/app/api/settings/ip-filter` (GET/PUT) - Thinking budget: `src/app/api/settings/thinking-budget` (GET/PUT) - System prompt: `src/app/api/settings/system-prompt` (GET/PUT) +- Compression: `src/app/api/settings/compression`, `src/app/api/compression/*`, and + `src/app/api/context/*` - Sessions: `src/app/api/sessions` (GET) - Rate limits: `src/app/api/rate-limits` (GET) - Resilience: `src/app/api/resilience` (GET/PATCH) — request queue, connection cooldown, provider breaker, wait-for-cooldown config @@ -252,6 +258,8 @@ Services (business logic): - Rate limit management: `open-sse/services/rateLimitManager.ts` - Circuit breaker: `open-sse/services/circuitBreaker.ts` - Context handoff: `open-sse/services/contextHandoff.ts` — handoff summary generation and injection for context-relay strategy +- Compression: `open-sse/services/compression/*` — proactive compression before provider translation; + includes Caveman rules, RTK filters, stacked pipelines, compression combos, stats, and validation - Codex quota fetcher: `open-sse/services/codexQuotaFetcher.ts` — fetches Codex quota for context-relay handoff decisions - Cooldown-aware retry: `src/sse/services/cooldownAwareRetry.ts` — per-model cooldown retries with configurable `requestRetry` / `maxRetryIntervalSec` - Safe outbound fetch: `src/shared/network/safeOutboundFetch.ts` — guarded provider/model fetch with SSRF guard, private-URL blocking, retry, and timeout @@ -630,6 +638,12 @@ flowchart LR - `src/app/api/settings/ip-filter`: IP allowlist/blocklist (GET/PUT) - `src/app/api/settings/thinking-budget`: thinking token budget config (GET/PUT) - `src/app/api/settings/system-prompt`: global system prompt (GET/PUT) +- `src/app/api/settings/compression`: global compression settings (GET/PUT) +- `src/app/api/compression/*`: compression preview, rule metadata, and language packs +- `src/app/api/context/caveman/config`: Caveman settings alias (GET/PUT) +- `src/app/api/context/rtk/*`: RTK config, filter catalog, test endpoint, and raw-output recovery +- `src/app/api/context/combos*`: compression combo CRUD and routing-combo assignments +- `src/app/api/context/analytics`: compression analytics alias - `src/app/api/sessions`: active session listing (GET) - `src/app/api/rate-limits`: per-account rate limit status (GET) - `src/app/api/sync/tokens`: sync token CRUD (GET/POST) diff --git a/docs/COMPRESSION_GUIDE.md b/docs/COMPRESSION_GUIDE.md index a6db971396..f3fd118fa0 100644 --- a/docs/COMPRESSION_GUIDE.md +++ b/docs/COMPRESSION_GUIDE.md @@ -19,6 +19,8 @@ Client Request → Standard: Caveman-speak filler removal (~30%) → Aggressive: History aging + summarization (~50%) → Ultra: Heuristic pruning + code-block thinning (~75%) + → RTK: Command-aware terminal/tool-output filtering (20-70%) + → Stacked: Ordered multi-engine pipeline, usually RTK then Caveman (30-80%) → Compressed Request → Provider ``` @@ -77,6 +79,35 @@ Maximum compression for token-critical scenarios: **Best for:** When you're hitting context limits repeatedly. +### RTK Mode (20-70% savings) + +RTK mode is optimized for verbose tool outputs that appear in coding-agent sessions: + +- Detects command/output classes such as `git status`, `git diff`, `git log`, test runners, + TypeScript/Vite/Webpack builds, ESLint/Biome/Prettier, npm audit/installs, Docker logs, infra + output, and generic shell output +- Applies JSON filter packs from `open-sse/services/compression/engines/rtk/filters/` +- Ships 39 built-in filters with inline verify samples +- Removes ANSI control sequences, progress bars, repeated lines, and non-actionable noise +- Preserves failures, errors, warnings, changed files, summaries, and the tail of long output +- Supports trust-gated project filters, global filters, and optional redacted raw-output recovery + +**Best for:** Agent sessions with shell, build, test, git, grep, and file-output transcripts. + +### Stacked Mode (30-80% savings) + +Stacked mode runs multiple compression engines in a deterministic order. The default pipeline is: + +```txt +RTK -> Caveman +``` + +That order keeps terminal/tool output compact first, then applies Caveman semantic condensation to +the remaining natural-language prompt. Stacked pipelines can be configured globally or through +compression combos assigned to routing combos. + +**Best for:** Mixed context with large tool logs plus human instructions or assistant summaries. + --- ## Token Savings Visualization @@ -87,6 +118,8 @@ With Lite: 40K tokens sent (15% saved — safe, always-on) With Standard: 33K tokens sent (30% saved — caveman-speak rules) With Aggressive: 24K tokens sent (50% saved — aging + summarization) With Ultra: 12K tokens sent (75% saved — heuristic pruning) +With RTK: varies by output (20-70% saved — command-aware filters) +With Stacked: varies by pipeline (30-80% saved — multi-engine execution) ``` --- @@ -95,25 +128,29 @@ With Ultra: 12K tokens sent (75% saved — heuristic pruning) ### Dashboard -Navigate to `Dashboard → Settings → Compression`: +Navigate to `Dashboard → Context & Cache`: -- **Default Mode** — sets the system-wide compression mode +- **Caveman** — mode selection, language packs, preview, and global defaults +- **RTK** — command-filter preview, RTK safety settings, and filter catalog +- **Compression Combos** — named engine pipelines assigned to routing combos - **Auto-Trigger Threshold** — automatically engage compression when token count exceeds threshold -- **Per-Combo Override** — each combo can have its own compression mode ### Per-Combo Override -In `Dashboard → Combos → [Your Combo] → Advanced`, set compression mode per combo: +In `Dashboard → Context & Cache → Compression Combos`, assign a compression combo to a routing +combo: ```txt Combo: "free-forever" - Mode: Standard + Compression Combo: "coding-agent-stack" + Pipeline: RTK -> Caveman Targets: 1. gc/gemini-3-flash 2. if/kimi-k2-thinking ``` -This lets you use aggressive compression on free providers while keeping lite mode on paid subscriptions. +This lets you use stacked compression on free/coding providers while keeping lite mode on paid +subscriptions. ### API @@ -124,7 +161,20 @@ curl http://localhost:20128/api/settings/compression # Update compression settings curl -X PUT http://localhost:20128/api/settings/compression \ -H "Content-Type: application/json" \ - -d '{"defaultMode":"lite","autoTriggerThreshold":32000}' + -d '{"defaultMode":"stacked","autoTriggerMode":"stacked","autoTriggerTokens":32000}' + +# Preview a specific RTK/stacked payload +curl -X POST http://localhost:20128/api/compression/preview \ + -H "Content-Type: application/json" \ + -d '{"mode":"rtk","messages":[{"role":"tool","content":"npm test output here"}]}' + +# List RTK filter packs +curl http://localhost:20128/api/context/rtk/filters + +# Test RTK directly with optional command metadata +curl -X POST http://localhost:20128/api/context/rtk/test \ + -H "Content-Type: application/json" \ + -d '{"command":"npm test","text":"FAIL tests/example.test.ts\nError: boom"}' ``` --- @@ -136,11 +186,14 @@ The compression engine **always preserves:** - ✅ Code blocks (fenced and inline) - ✅ URLs and file paths - ✅ JSON structures and structured data -- ✅ API keys, tokens, and identifiers +- ✅ Identifiers and protected technical tokens - ✅ Mathematical expressions - ✅ Tool/function call definitions - ✅ System prompts (in lite mode) +RTK raw-output recovery redacts common API keys, bearer tokens, Slack tokens, AWS access keys, +passwords, tokens, and secrets before anything is persisted. + --- ## Compression Stats @@ -154,7 +207,10 @@ Every compressed request includes stats in the server logs: "savingsPercent": 15.0, "techniquesUsed": ["collapseWhitespace", "dedupSystemPrompt"], "mode": "lite", - "latencyMs": 0.8 + "engine": "caveman", + "compressionComboId": "coding-agent-stack", + "durationMs": 0.8, + "rtkRawOutputPointers": [] } ``` @@ -166,7 +222,8 @@ Every compressed request includes stats in the server logs: | ------- | ------------------------------------ | ---------- | | Phase 1 | Off, Lite | ✅ Shipped | | Phase 2 | Standard, Aggressive, Ultra | ✅ Shipped | -| Phase 3 | Per-model adaptive, ML-based pruning | 🗓️ Planned | +| Phase 3 | RTK, Stacked, Compression Combos | ✅ Shipped | +| Phase 4 | Per-model adaptive, ML-based pruning | 🗓️ Planned | --- @@ -181,3 +238,7 @@ Standard mode compression rules are inspired by **[Caveman](https://github.com/J - [Environment Config](ENVIRONMENT.md) — Compression environment variables - [Architecture Guide](ARCHITECTURE.md) — Compression pipeline internals - [User Guide](USER_GUIDE.md) — Getting started with compression +- [RTK Compression](rtk-compression.md) — RTK filters, trust model, verify gate, raw-output recovery +- [Compression Engines](compression-engines.md) — Caveman, RTK, stacked, APIs, MCP, dashboard +- [Compression Rules Format](compression-rules-format.md) — JSON rule-pack format +- [Compression Language Packs](compression-language-packs.md) — Language-specific Caveman rules diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 02e7c2843b..066d800705 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -497,6 +497,12 @@ The logging system writes to both stdout and rotated log files. All configuratio | `CONTEXT_LENGTH_DEFAULT` | `128000` | Global fallback max context length for models without explicit config. | | `USAGE_TOKEN_BUFFER` | `100` | Extra token headroom reserved when tracking usage quotas. | +### Compression + +| Variable | Default | Description | +| ------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------- | +| `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS` | unset | Trust project `.rtk/filters.json` without a `.rtk/trust.json` hash. Use only in controlled local development. | + ### Low-RAM Docker Example ```bash diff --git a/docs/FEATURES.md b/docs/FEATURES.md index d68c6a6db9..ff2ec2e3c8 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -113,6 +113,20 @@ Currently supports Codex account rotation. See [Context Relay documentation](fea --- +## 🗜️ Prompt Compression _(v3.7.9+)_ + +Context & Cache now exposes dedicated pages for Caveman, RTK, and Compression Combos: + +- **Caveman** — language-aware rule packs, preview, output-mode controls, and analytics +- **RTK** — command-aware compression for shell, git, test, build, package, Docker, infra, JSON, and stack-trace output +- **Compression Combos** — named pipelines such as `rtk -> caveman` assigned to routing combos +- **Raw-output recovery** — optional redacted RTK raw-output pointers for debugging compressed failures + +See [Compression Guide](COMPRESSION_GUIDE.md), [RTK Compression](rtk-compression.md), and +[Compression Engines](compression-engines.md). + +--- + ## 🛡️ Proxy Hardening _(v3.5.5+)_ Comprehensive proxy configuration enforcement across the entire request pipeline: diff --git a/docs/MCP-SERVER.md b/docs/MCP-SERVER.md index 2f8e6bab3f..0483ef19f8 100644 --- a/docs/MCP-SERVER.md +++ b/docs/MCP-SERVER.md @@ -1,6 +1,6 @@ # OmniRoute MCP Server Documentation -> Model Context Protocol server with 16 intelligent tools +> Model Context Protocol server with 37 tools across routing, cache, compression, memory, skills, and proxy operations ## Installation @@ -49,20 +49,50 @@ See [IDE Configs](integrations/ide-configs.md) for Antigravity, Cursor, Copilot, | `omniroute_explain_route` | Explain a past routing decision | | `omniroute_get_session_snapshot` | Full session state: costs, tokens, errors | +## Cache Tools (2) + +| Tool | Description | +| :---------------------- | :-------------------------------------------------- | +| `omniroute_cache_stats` | Semantic cache, prompt-cache, and idempotency stats | +| `omniroute_cache_flush` | Flush cache globally or by signature/model | + +## Compression Tools (5) + +| Tool | Description | +| :---------------------------------- | :------------------------------------------------------------- | +| `omniroute_compression_status` | Compression settings, analytics summary, and cache-aware stats | +| `omniroute_compression_configure` | Configure compression mode, threshold, and runtime options | +| `omniroute_set_compression_engine` | Set Caveman, RTK, or stacked compression mode and pipeline | +| `omniroute_list_compression_combos` | List named compression combos and routing assignments | +| `omniroute_compression_combo_stats` | Analytics grouped by compression combo and engine | + +See [Compression Engines](compression-engines.md) and [RTK Compression](rtk-compression.md) for +the runtime compression model behind these tools. + +## Other Tool Groups + +The remaining MCP surface includes 1proxy tools, memory tools, and skill tools. The live source of +truth is `open-sse/mcp-server/tools/` and `open-sse/mcp-server/schemas/tools.ts`. + ## Authentication MCP tools are authenticated via API key scopes. Each tool requires specific scopes: -| Scope | Tools | -| :------------- | :----------------------------------------------- | -| `read:health` | get_health, get_provider_metrics | -| `read:combos` | list_combos, get_combo_metrics | -| `write:combos` | switch_combo | -| `read:quota` | check_quota | -| `write:route` | route_request, simulate_route, test_combo | -| `read:usage` | cost_report, get_session_snapshot, explain_route | -| `write:config` | set_budget_guard, set_resilience_profile | -| `read:models` | list_models_catalog, best_combo_for_task | +| Scope | Tools | +| :-------------------- | :------------------------------------------------------------------- | +| `read:health` | get_health, get_provider_metrics | +| `read:combos` | list_combos, get_combo_metrics | +| `write:combos` | switch_combo | +| `read:quota` | check_quota | +| `write:route` | route_request, simulate_route, test_combo | +| `read:usage` | cost_report, get_session_snapshot, explain_route | +| `write:config` | set_budget_guard, set_resilience_profile | +| `read:models` | list_models_catalog, best_combo_for_task | +| `read:cache` | cache_stats | +| `write:cache` | cache_flush | +| `read:compression` | compression_status, list_compression_combos, compression_combo_stats | +| `write:compression` | compression_configure, set_compression_engine | +| `execute:completions` | route_request, test_combo | ## Audit Logging @@ -74,10 +104,10 @@ Every tool call is logged to `mcp_tool_audit` with: ## Files -| File | Purpose | -| :------------------------------------------- | :------------------------------------------ | -| `open-sse/mcp-server/server.ts` | MCP server creation + 16 tool registrations | -| `open-sse/mcp-server/transport.ts` | Stdio + HTTP transport | -| `open-sse/mcp-server/auth.ts` | API key + scope validation | -| `open-sse/mcp-server/audit.ts` | Tool call audit logging | -| `open-sse/mcp-server/tools/advancedTools.ts` | 8 advanced tool handlers | +| File | Purpose | +| :------------------------------------------- | :------------------------------------------------ | +| `open-sse/mcp-server/server.ts` | MCP server creation and scoped tool registrations | +| `open-sse/mcp-server/transport.ts` | Stdio + HTTP transport | +| `open-sse/mcp-server/auth.ts` | API key + scope validation | +| `open-sse/mcp-server/audit.ts` | Tool call audit logging | +| `open-sse/mcp-server/tools/advancedTools.ts` | 8 advanced tool handlers | diff --git a/docs/SETUP_GUIDE.md b/docs/SETUP_GUIDE.md index 9e78e203e9..26c332a5ab 100644 --- a/docs/SETUP_GUIDE.md +++ b/docs/SETUP_GUIDE.md @@ -138,7 +138,7 @@ Add to your MCP settings: } ``` -**Full MCP documentation:** [MCP Server README](../open-sse/mcp-server/README.md) — 29 tools, IDE configs, Python/TS/Go clients. +**Full MCP documentation:** [MCP Server README](../open-sse/mcp-server/README.md) — 37 tools, IDE configs, Python/TS/Go clients. ### A2A Setup (Agent-to-Agent Protocol) diff --git a/docs/compression-engines.md b/docs/compression-engines.md new file mode 100644 index 0000000000..12e18d60c6 --- /dev/null +++ b/docs/compression-engines.md @@ -0,0 +1,127 @@ +# Compression Engines + +OmniRoute compression is built around engine contracts. A mode can run one engine directly +(`caveman` or `rtk`) or a deterministic stacked pipeline that executes multiple engines in order. + +## Modes + +| Mode | Engine path | Intended input | +| ------------ | ---------------------------------- | -------------------------------------------- | +| `off` | none | Exact prompt preservation | +| `lite` | Caveman lite helpers | Low-risk always-on cleanup | +| `standard` | Caveman | Natural-language prompt condensation | +| `aggressive` | Caveman + history/tool summarizers | Long chat sessions | +| `ultra` | Caveman + pruning helpers | Context-limit recovery | +| `rtk` | RTK | Terminal, shell, build, test, and git output | +| `stacked` | Pipeline, default `rtk -> caveman` | Mixed tool logs and prose | + +## Engine Registry + +The registry lives in `open-sse/services/compression/engines/registry.ts`. Engines expose a shared +contract: + +- `id`: stable engine id such as `caveman` or `rtk` +- `label`: dashboard-readable name +- `supports(mode)`: whether the engine can execute a compression mode +- `compress(input)`: transforms text/messages and returns stats + +`strategySelector.ts` registers the built-in engines before compression runs. This lets preview, +runtime compression, stacked mode, tests, and future engines use the same execution path. + +## Caveman + +Caveman mode focuses on semantic condensation of normal prose: + +- preserves code blocks, URLs, JSON, paths, and structured data +- removes filler, hedging, repeated context, and verbose connective phrasing +- supports language-aware file rule packs in `open-sse/services/compression/rules/` +- remains available through the legacy `standard`, `aggressive`, and `ultra` modes + +The dashboard surface is `Dashboard -> Context & Cache -> Caveman`. + +## RTK + +RTK mode focuses on command and tool output: + +- detects output classes such as `git status`, `git branch`, `git diff`, Vitest/Jest/Pytest, + Cargo/Go tests, TypeScript/Vite/Webpack builds, ESLint, npm audit/installs, Docker logs, + shell `find`/`grep`, stack traces, and generic logs +- applies 39 JSON filters from `open-sse/services/compression/engines/rtk/filters/` +- supports the RTK-style declarative pipeline: ANSI stripping, replace, match-output short-circuit, + strip/keep lines, per-line truncation, head/tail/max-line truncation, and on-empty fallback +- supports trust-gated project filters in `.rtk/filters.json` and global filters in + `DATA_DIR/rtk/filters.json` +- strips ANSI sequences, progress noise, repeated lines, and unhelpful boilerplate +- preserves actionable failures, warnings, summaries, changed files, and tail context +- can optionally retain redacted raw output for recovery/debugging through authenticated management + routes + +The dashboard surface is `Dashboard -> Context & Cache -> RTK`. + +Operational details for custom filters, trust, verify, and raw-output recovery live in +[`rtk-compression.md`](rtk-compression.md). + +## Stacked Pipelines + +Stacked mode runs pipeline steps in order. The default is: + +```txt +rtk -> caveman +``` + +Use this for coding-agent sessions where a prompt combines command output with human or assistant +prose. RTK reduces noisy tool logs first, then Caveman compresses remaining natural language. + +Pipeline steps are configured with `stackedPipeline` in compression settings or through compression +combos. + +## Compression Combos + +Compression combos are named compression profiles that can be assigned to routing combos: + +- `compression_combos`: stores mode, pipeline, RTK config, language config, and default marker +- `compression_combo_assignments`: maps a compression combo to a routing combo +- runtime integration resolves an assigned compression combo before generic combo overrides +- analytics include `compression_combo_id` and `engine` + +Dashboard surface: `Dashboard -> Context & Cache -> Compression Combos`. + +## API Surface + +| Route | Purpose | +| -------------------------------------- | ------------------------------------------ | +| `/api/settings/compression` | Global compression settings | +| `/api/compression/preview` | Preview any compression mode | +| `/api/compression/language-packs` | List available Caveman language packs | +| `/api/context/caveman/config` | Caveman settings alias | +| `/api/context/rtk/config` | RTK defaults and settings | +| `/api/context/rtk/filters` | RTK filter catalog | +| `/api/context/rtk/test` | RTK preview/test endpoint | +| `/api/context/rtk/raw-output/[id]` | Authenticated redacted raw-output recovery | +| `/api/context/combos` | Compression combo CRUD | +| `/api/context/combos/[id]/assignments` | Routing-combo assignment CRUD | +| `/api/context/analytics` | Compression analytics alias | + +Management routes require management authentication or API-key policy checks. + +## MCP Tools + +Compression exposes five MCP tools: + +| Tool | Scope | Purpose | +| ----------------------------------- | ------------------- | -------------------------------- | +| `omniroute_compression_status` | `read:compression` | Settings, analytics, cache stats | +| `omniroute_compression_configure` | `write:compression` | Update global settings | +| `omniroute_set_compression_engine` | `write:compression` | Set mode and optional pipeline | +| `omniroute_list_compression_combos` | `read:compression` | List compression combos | +| `omniroute_compression_combo_stats` | `read:compression` | Read combo/engine analytics | + +## Validation + +The focused gates for this area are: + +```bash +node --import tsx/esm --test tests/unit/compression/rtk-*.test.ts tests/unit/compression/pipeline-integration.test.ts tests/unit/compression/context-compression-api.test.ts +node --import tsx/esm --test tests/unit/compression/*.test.ts tests/golden-set/*.test.ts tests/integration/compression-pipeline.test.ts tests/unit/api/compression/compression-api.test.ts +npm run typecheck:core +``` diff --git a/docs/compression-language-packs.md b/docs/compression-language-packs.md new file mode 100644 index 0000000000..f797865697 --- /dev/null +++ b/docs/compression-language-packs.md @@ -0,0 +1,96 @@ +# Compression Language Packs + +Caveman compression can load language-specific rule packs in addition to the built-in English rules. +This keeps the core engine stable while allowing Portuguese, Spanish, German, French, Japanese, and +future language packs to evolve independently. + +## Location + +Language packs live under: + +```txt +open-sse/services/compression/rules// +``` + +Current shipped packs include: + +| Language | Directory | +| ------------------- | -------------- | +| English | `rules/en/` | +| Portuguese (Brazil) | `rules/pt-BR/` | +| Spanish | `rules/es/` | +| German | `rules/de/` | +| French | `rules/fr/` | +| Japanese | `rules/ja/` | + +## Language Detection + +`languageDetector.ts` uses lightweight heuristics to infer the language from prompt text. The +configured default language is still respected, and detection can be disabled by config when exact +control is required. + +Detection output is used only to choose rule packs. It does not change provider routing, locale +selection, or UI language. + +## Config Shape + +Compression settings can include: + +```json +{ + "languageConfig": { + "enabled": true, + "defaultLanguage": "en", + "autoDetect": true, + "enabledPacks": ["en", "pt-BR", "es", "de", "fr", "ja"] + }, + "cavemanConfig": { + "language": "en", + "autoDetectLanguage": true, + "enabledLanguagePacks": ["en", "pt-BR", "es", "de", "fr", "ja"] + } +} +``` + +`languageConfig` controls dashboard/preview defaults. `cavemanConfig` is the runtime engine config +used when Caveman compresses message text. + +## Adding a Language Pack + +1. Create `open-sse/services/compression/rules//.json`. +2. Use the Caveman rule format from `docs/compression-rules-format.md`. +3. Keep replacements conservative and avoid changing code, identifiers, URLs, or JSON. +4. Add or update tests for language selection and replacement behavior. +5. Expose new dashboard/i18n labels if the language appears in UI selectors. + +## API + +Available packs can be queried with: + +```bash +curl http://localhost:20128/api/compression/language-packs +``` + +The preview endpoint accepts language config overrides: + +```bash +curl -X POST http://localhost:20128/api/compression/preview \ + -H "Content-Type: application/json" \ + -d '{ + "mode": "standard", + "text": "Por favor, eu gostaria que voce basicamente resumisse isso.", + "config": { + "languageConfig": { + "defaultLanguage": "pt-BR", + "autoDetect": true + } + } + }' +``` + +## Operational Notes + +- English built-in rules remain the fallback when a language pack is missing. +- Invalid built-in JSON packs fail validation so release assets do not silently degrade. +- Rule packs are data-only and should not import code or run arbitrary logic. +- The compression analytics layer records the selected mode and engine, not full prompt text. diff --git a/docs/compression-rules-format.md b/docs/compression-rules-format.md new file mode 100644 index 0000000000..a3187b9c8e --- /dev/null +++ b/docs/compression-rules-format.md @@ -0,0 +1,177 @@ +# Compression Rules Format + +Compression rules are JSON files loaded at runtime. They are intentionally data-only so new +language packs and RTK command filters can be reviewed without changing engine code. + +## Caveman Rule Packs + +Caveman rule packs live under: + +```txt +open-sse/services/compression/rules//.json +``` + +Each pack contains replacements that apply to normal prose after protected regions are isolated. + +```json +{ + "language": "en", + "category": "filler", + "rules": [ + { + "name": "remove_basically", + "pattern": "\\bbasically\\b", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite", + "description": "Remove filler adverb." + } + ] +} +``` + +### Caveman Fields + +| Field | Required | Description | +| ---------------------- | -------- | ---------------------------------------------------------------- | +| `language` | yes | BCP-47-like language key such as `en`, `pt-BR`, `es` | +| `category` | yes | Pack category filename/category, for example `filler` or `dedup` | +| `rules` | yes | Array of regex replacement rules | +| `rules[].name` | yes | Stable rule name | +| `rules[].pattern` | yes | JavaScript regex source, compiled with `gi` | +| `rules[].replacement` | yes | Replacement string | +| `rules[].context` | no | `all`, `user`, `assistant`, or `system`; default `all` | +| `rules[].category` | no | `filler`, `context`, `structural`, `dedup`, `terse`, or `ultra` | +| `rules[].minIntensity` | no | `lite`, `full`, or `ultra`; default `lite` | +| `rules[].description` | no | Human-readable rule summary | + +Caveman file packs are data-only. Replacement functions are reserved for built-in TypeScript rules; +JSON packs use string replacements only. + +## RTK Filter Packs + +RTK filters live under: + +```txt +open-sse/services/compression/engines/rtk/filters/.json +``` + +Each filter describes how to recognize and compress a command-output family. + +```json +{ + "id": "test-vitest", + "label": "Vitest output", + "category": "test", + "priority": 92, + "match": { + "outputTypes": ["test-vitest"], + "commands": ["vitest", "npm test", "npm run test"], + "patterns": ["\\bFAIL\\b", "\\bPASS\\b", "\\bTest Files\\b"] + }, + "rules": { + "stripAnsi": true, + "replace": [{ "pattern": "\\s+\\[[0-9]+ms\\]", "replacement": "" }], + "matchOutput": [ + { "pattern": "All tests passed", "message": "vitest: ok", "unless": "FAIL|Error:" } + ], + "includePatterns": ["FAIL", "Error:", "Test Files", "Tests"], + "dropPatterns": ["^\\s*$", "Duration\\s+\\d+"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "truncateLineAt": 240, + "maxLines": 160, + "headLines": 24, + "tailLines": 40, + "onEmpty": "vitest: ok", + "filterStderr": false + }, + "preserve": { + "errorPatterns": ["FAIL", "Error:", "AssertionError"], + "summaryPatterns": ["Test Files", "Tests", "Snapshots"] + }, + "tests": [ + { + "name": "keeps failing tests", + "command": "vitest", + "input": "FAIL test/a.test.ts\\nError: boom\\nTest Files 1 failed", + "expected": "FAIL test/a.test.ts\\nError: boom\\nTest Files 1 failed" + } + ] +} +``` + +### RTK Fields + +| Field | Required | Description | +| -------------------------- | -------- | ------------------------------------------------------------------------------ | +| `id` | yes | Stable filter id | +| `label` | yes | Dashboard-readable name | +| `category` | yes | Filter family: git, test, build, shell, docker, package, infra, cloud, generic | +| `priority` | no | Higher priority wins when multiple filters match | +| `match.outputTypes` | no | Detector output ids that select this filter | +| `match.commands` | no | Command tokens that select this filter | +| `match.patterns` | no | Regex patterns that select this filter from output text | +| `rules.stripAnsi` | no | Remove ANSI escape sequences before regex stages | +| `rules.replace` | no | Ordered regex substitutions applied line by line | +| `rules.matchOutput` | no | Short-circuit output rules with optional `unless` guard | +| `rules.includePatterns` | no | Lines to prefer preserving | +| `rules.dropPatterns` | no | Lines to remove as noise | +| `rules.collapsePatterns` | no | Repeated matching lines that can be collapsed | +| `rules.deduplicate` | no | Collapse duplicate normalized lines | +| `rules.truncateLineAt` | no | Unicode-safe per-line character limit | +| `rules.maxLines` | no | Maximum retained lines before tail preservation | +| `rules.headLines` | no | Head lines retained during truncation | +| `rules.tailLines` | no | Tail lines retained for recent context | +| `rules.onEmpty` | no | Fallback message when filtering removes all content | +| `rules.filterStderr` | no | Normalize common stderr prefixes before later filtering stages | +| `preserve.errorPatterns` | no | Error lines that should survive truncation | +| `preserve.summaryPatterns` | no | Summary lines that should survive truncation | +| `tests[]` | no | Inline verification samples used by the RTK verify gate | + +RTK applies declarative stages in this order: `stripAnsi`, `filterStderr`, `replace`, +`matchOutput`, `dropPatterns`/`includePatterns`, `truncateLineAt`, `headLines`/`tailLines`, +`maxLines`, and `onEmpty`. + +Custom filters can be loaded from: + +1. Project `.rtk/filters.json` files only after a matching `.rtk/trust.json` hash is present or + `trustProjectFilters` is enabled. +2. Global `DATA_DIR/rtk/filters.json`. +3. Built-in filters. + +Project/global custom files may contain one filter object or an array of filter objects. Invalid +custom filters are skipped with diagnostics; invalid built-in filters fail validation. + +Project trust file: + +```json +{ + "filtersSha256": "0123456789abcdef..." +} +``` + +The environment override `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=1` trusts project filters without a +hash and should be limited to controlled local development. + +## Safety Rules + +- Keep rules idempotent: running the same filter twice should not corrupt output. +- Preserve exact error text, file paths, line numbers, and command summaries where possible. +- Avoid rules that modify code blocks, JSON payloads, URLs, or secrets. +- Add unit coverage for new command families in detector/filter tests. +- Add `tests[]` samples to every built-in filter and to shared custom filters. + +## Validation + +Rule packs are validated before use. Built-in Caveman packs and built-in RTK filters fail fast +during validation so broken release assets are caught before shipment. Custom RTK filters are +skipped with diagnostics when parsing or trust validation fails. + +Focused validation: + +```bash +node --import tsx/esm --test tests/unit/compression/rule-loader.test.ts tests/unit/compression/language-packs.test.ts +node --import tsx/esm --test tests/unit/compression/rtk-verify.test.ts tests/unit/compression/rtk-dsl-pipeline.test.ts +``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e4277d69a5..bea18f734e 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -51,6 +51,8 @@ tags: description: Routing combo management - name: Settings description: Application settings + - name: Compression + description: Prompt compression, RTK filters, Caveman rules, and compression combos - name: Usage description: Usage analytics and logs - name: Translator @@ -734,6 +736,202 @@ paths: "200": description: Updated settings + /api/settings/compression: + get: + tags: [Compression] + summary: Get global compression settings + security: + - ManagementSessionAuth: [] + responses: + "200": + description: Current compression settings + put: + tags: [Compression] + summary: Update global compression settings + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + defaultMode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + autoTriggerMode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + autoTriggerTokens: + type: integer + minimum: 0 + rtkConfig: + type: object + additionalProperties: true + stackedPipeline: + type: array + items: + type: object + responses: + "200": + description: Updated compression settings + + /api/compression/preview: + post: + tags: [Compression] + summary: Preview compression for a message payload + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [messages, mode] + properties: + mode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + messages: + type: array + items: + type: object + required: [role, content] + properties: + role: + type: string + content: + oneOf: + - type: string + - type: array + items: {} + config: + type: object + additionalProperties: true + responses: + "200": + description: Compression preview with diff, validation, and stats + + /api/compression/language-packs: + get: + tags: [Compression] + summary: List Caveman compression language packs + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + responses: + "200": + description: Available languages and rule-pack metadata + + /api/compression/rules: + get: + tags: [Compression] + summary: List Caveman compression rule metadata + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + responses: + "200": + description: Caveman rule metadata + + /api/context/rtk/config: + get: + tags: [Compression] + summary: Get RTK compression settings + security: + - ManagementSessionAuth: [] + responses: + "200": + description: Current RTK config + put: + tags: [Compression] + summary: Update RTK compression settings + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + intensity: + type: string + enum: [minimal, standard, aggressive] + customFiltersEnabled: + type: boolean + trustProjectFilters: + type: boolean + rawOutputRetention: + type: string + enum: [never, failures, always] + rawOutputMaxBytes: + type: integer + responses: + "200": + description: Updated RTK config + + /api/context/rtk/filters: + get: + tags: [Compression] + summary: List RTK filters and load diagnostics + security: + - ManagementSessionAuth: [] + responses: + "200": + description: RTK filter catalog and diagnostics + + /api/context/rtk/test: + post: + tags: [Compression] + summary: Run RTK compression preview for text + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [text] + properties: + text: + type: string + command: + type: string + config: + type: object + additionalProperties: true + responses: + "200": + description: Detection and RTK compression result + + /api/context/rtk/raw-output/{id}: + get: + tags: [Compression] + summary: Read retained redacted RTK raw output + security: + - ManagementSessionAuth: [] + parameters: + - in: path + name: id + required: true + schema: + type: string + pattern: "^[a-f0-9]{24}$" + responses: + "200": + description: Raw output text + "404": + description: Raw output not found + /api/settings/payload-rules: get: tags: [Settings] diff --git a/docs/rtk-compression.md b/docs/rtk-compression.md new file mode 100644 index 0000000000..20feebef65 --- /dev/null +++ b/docs/rtk-compression.md @@ -0,0 +1,223 @@ +# RTK Compression + +RTK compression is OmniRoute's command-aware compression engine for terminal and tool output. It is +designed for coding-agent sessions where most context growth comes from test logs, build output, +package manager noise, shell transcripts, Docker output, git output, and stack traces. + +RTK can run directly with `defaultMode: "rtk"` or as the first step in a stacked pipeline, usually: + +```txt +rtk -> caveman +``` + +That order compresses noisy machine output first, then lets Caveman condense remaining prose. + +## What It Compresses + +The built-in catalog currently ships 39 filters across these categories: + +| Category | Examples | +| --------- | ------------------------------------------------------------- | +| `git` | `git status`, `git branch`, `git diff`, `git log` | +| `test` | Vitest, Jest, Pytest, Playwright, Go tests, Cargo tests | +| `build` | TypeScript, ESLint, Biome, Prettier, Vite, Webpack, Turbo, Nx | +| `package` | `npm install`, `npm audit`, `pip`, `uv sync`, Poetry, Bundler | +| `shell` | `ls`, `find`, `grep`, generic shell logs | +| `docker` | `docker ps`, Docker logs | +| `infra` | Terraform, OpenTofu, `systemctl status` | +| `generic` | JSON output, stack traces, generic output fallback | + +The detector in `open-sse/services/compression/engines/rtk/commandDetector.ts` classifies output +before filter selection. Filters can also match by command pattern or output regex when a command +class is not enough. + +## Filter Resolution + +RTK loads filters in this order: + +1. Project filters from `.rtk/filters.json`, only when trusted. +2. Global filters from `DATA_DIR/rtk/filters.json`. +3. Built-in filters from `open-sse/services/compression/engines/rtk/filters/`. + +Project filters are intentionally trust-gated because regex filters can change how tool output is +shown to agents. A project filter file is accepted when one of these is true: + +- `rtkConfig.trustProjectFilters` is `true`. +- `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=1` is set. +- `.rtk/trust.json` contains the SHA-256 hash of `.rtk/filters.json`. + +Trust file example: + +```json +{ + "filtersSha256": "0123456789abcdef..." +} +``` + +Custom filters can be one filter object or an array of filter objects. Invalid custom filters are +skipped and reported by `/api/context/rtk/filters` diagnostics. Invalid built-in filters fail fast. + +## Filter DSL + +Filters use the JSON schema described in [Compression Rules Format](compression-rules-format.md). +The runtime applies these stages in order: + +```txt +stripAnsi -> filterStderr -> replace -> matchOutput -> drop/include lines + -> truncateLineAt -> head/tail/maxLines -> onEmpty +``` + +Important fields: + +| Field | Purpose | +| ---------------------------- | -------------------------------------------------------------- | +| `rules.stripAnsi` | Remove terminal color/control sequences before matching | +| `rules.filterStderr` | Normalize common stderr prefixes before matching/filtering | +| `rules.replace` | Apply ordered regex replacements | +| `rules.matchOutput` | Return a compact summary when output matches a known condition | +| `rules.matchOutput[].unless` | Skip the shortcut when an error/failure pattern is present | +| `rules.dropPatterns` | Remove noisy lines | +| `rules.includePatterns` | Prefer actionable lines | +| `rules.collapsePatterns` | Collapse repeated matching lines | +| `rules.truncateLineAt` | Unicode-safe per-line truncation | +| `rules.onEmpty` | Fallback message if all lines are filtered out | +| `tests[]` | Inline samples used by the verify gate | + +Built-in filters are expected to include inline `tests[]` samples. Custom filters should include +them too, especially when they are shared across projects. + +## Configuration + +Global settings are available through `/api/settings/compression`. RTK-specific settings are also +available through `/api/context/rtk/config`. + +```json +{ + "defaultMode": "stacked", + "autoTriggerMode": "stacked", + "autoTriggerTokens": 32000, + "stackedPipeline": [ + { "engine": "rtk", "intensity": "standard" }, + { "engine": "caveman", "intensity": "full" } + ], + "rtkConfig": { + "enabled": true, + "intensity": "standard", + "applyToToolResults": true, + "applyToCodeBlocks": false, + "applyToAssistantMessages": false, + "enabledFilters": [], + "disabledFilters": [], + "maxLinesPerResult": 120, + "maxCharsPerResult": 12000, + "deduplicateThreshold": 3, + "customFiltersEnabled": true, + "trustProjectFilters": false, + "rawOutputRetention": "never", + "rawOutputMaxBytes": 1048576 + } +} +``` + +`enabledFilters` and `disabledFilters` use filter ids, for example `test-vitest` or `git-diff`. + +## API + +| Route | Method | Purpose | +| ---------------------------------- | ------ | -------------------------------------------- | +| `/api/context/rtk/config` | GET | Read RTK config | +| `/api/context/rtk/config` | PUT | Update RTK config | +| `/api/context/rtk/filters` | GET | List filter catalog and load diagnostics | +| `/api/context/rtk/test` | POST | Preview RTK compression for one text payload | +| `/api/context/rtk/raw-output/[id]` | GET | Read retained redacted raw output | +| `/api/compression/preview` | POST | Preview any compression mode | + +RTK test payload: + +```json +{ + "command": "npm test", + "text": "FAIL tests/example.test.ts\nAssertionError: expected true\nTest Files 1 failed", + "config": { + "intensity": "standard" + } +} +``` + +Compression preview payload: + +```json +{ + "mode": "stacked", + "messages": [ + { + "role": "tool", + "content": "FAIL tests/example.test.ts\nAssertionError: expected true\nTest Files 1 failed" + } + ], + "config": { + "rtkConfig": { + "rawOutputRetention": "failures" + } + } +} +``` + +Management routes require dashboard management auth or the matching API-key policy. + +## Raw Output Recovery + +RTK normally returns only compressed text. For debugging, `rawOutputRetention` can retain redacted +raw output: + +| Value | Behavior | +| ---------- | ------------------------------------------------------- | +| `never` | Do not retain raw output | +| `failures` | Retain only likely failure output | +| `always` | Retain every compressed RTK raw output, after redaction | + +Retained files are written under: + +```txt +DATA_DIR/rtk/raw-output/ +``` + +Secrets are redacted before persistence, including common bearer tokens, API keys, Slack tokens, +AWS access keys, and assignment-style `token=...`, `secret=...`, `password=...` values. Analytics +stores only the pointer id, size, and hash metadata. + +## Verify Gate + +The focused verify gate runs built-in inline filter tests without shelling out to external commands: + +```bash +node --import tsx/esm --test tests/unit/compression/rtk-verify.test.ts +``` + +The broader RTK gate is: + +```bash +node --import tsx/esm --test \ + tests/unit/compression/rtk-*.test.ts \ + tests/unit/compression/pipeline-integration.test.ts \ + tests/unit/compression/context-compression-api.test.ts +``` + +Run the broad compression gate before release: + +```bash +node --import tsx/esm --test \ + tests/unit/compression/*.test.ts \ + tests/golden-set/*.test.ts \ + tests/integration/compression-pipeline.test.ts \ + tests/unit/api/compression/compression-api.test.ts +``` + +## Extending RTK + +1. Add or update a filter JSON file. +2. Include at least one `tests[]` sample that proves the important behavior. +3. Add a fixture under `tests/unit/compression/fixtures/rtk/` for new command families. +4. Add command detection coverage when introducing a new output class. +5. Run the verify and broad RTK gates. +6. If the filter is project-local, commit `.rtk/filters.json` and refresh `.rtk/trust.json` only after review. diff --git a/next.config.mjs b/next.config.mjs index be2b7dd40c..38acb2cc93 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -79,9 +79,13 @@ const nextConfig = { }, outputFileTracingRoot: projectRoot, outputFileTracingIncludes: { - // Migration SQL files are read via fs.readFileSync at runtime and are NOT - // auto-traced by webpack/turbopack — include them explicitly. - "/*": ["./src/lib/db/migrations/**/*"], + // Migration SQL and compression rule/filter JSON files are read via fs at + // runtime and are NOT always auto-traced by webpack/turbopack. + "/*": [ + "./src/lib/db/migrations/**/*", + "./open-sse/services/compression/engines/rtk/filters/**/*.json", + "./open-sse/services/compression/rules/**/*.json", + ], }, outputFileTracingExcludes: { // Planning/task docs are not runtime assets and can break standalone copies @@ -136,97 +140,6 @@ const nextConfig = { images: { unoptimized: true, }, - webpack: (config, { isServer, webpack }) => { - if (isServer) { - // Webpack IgnorePlugin: skip thread-stream test files that contain - // intentionally broken syntax/imports (they cause Turbopack build errors) - config.plugins.push( - new webpack.IgnorePlugin({ - resourceRegExp: /\/test\//, - contextRegExp: /thread-stream/, - }) - ); - - // ── Turbopack / Next.js 16 module-hash patch (#394, #396, #398) ──────── - // - // Next.js 16 (with or without Turbopack) compiles the instrumentation hook - // into a separate chunk and emits hashed require() calls such as: - // require('better-sqlite3-90e2652d1716b047') - // require('zod-dcb22c6336e0bc69') - // require('pino-28069d5257187539') - // - // These hashed names don't exist in node_modules and cause a 500 at - // startup on all npm global installs (issues #394, #396, #398). - // - // We use two strategies: - // 1. Exact-name externals for all known server-side packages. - // 2. Hash-strip catch-all: any require('-<16hexchars>[/subpath]') - // strips the hash suffix and falls through to the real package name. - // - const HASH_PATTERN = /^(.+)-[0-9a-f]{16}(\/.*)?$/; - - const KNOWN_EXTERNALS = new Set([ - "better-sqlite3", - "keytar", - "@ngrok/ngrok", - "wreq-js", - "zod", - "pino", - "pino-pretty", - "pino-abstract-transport", - "child_process", - "fs", - "path", - "os", - "crypto", - "net", - "tls", - "http", - "https", - "stream", - "buffer", - "util", - "process", - ]); - - const prev = config.externals ?? []; - const prevArr = Array.isArray(prev) ? prev : [prev]; - config.externals = [ - ...prevArr, - ({ request }, callback) => { - // Case 1: Exact known package — treat as external - if (KNOWN_EXTERNALS.has(request)) { - return callback(null, `commonjs ${request}`); - } - // Case 2: Hash-suffixed name — strip hash, preserve subpath - // e.g. "better-sqlite3-90e2652d1716b047" → "better-sqlite3" - // "zod-dcb22c6336e0bc69" → "zod" - // "zod-dcb22c6336e0bc69/v3" → "zod/v3" - // "zod-dcb22c6336e0bc69/v4-mini" → "zod/v4-mini" - const hashMatch = request?.match?.(HASH_PATTERN); - if (hashMatch) { - const resolved = hashMatch[2] ? `${hashMatch[1]}${hashMatch[2]}` : hashMatch[1]; - return callback(null, `commonjs ${resolved}`); - } - callback(); - }, - ]; - } else { - // Ignore native Node.js modules in browser bundle - config.resolve.fallback = { - ...config.resolve.fallback, - fs: false, - path: false, - child_process: false, - net: false, - tls: false, - crypto: false, - process: false, - os: false, - }; - } - return config; - }, async headers() { return [ diff --git a/open-sse/config/embeddingRegistry.ts b/open-sse/config/embeddingRegistry.ts index 178db272cf..c2526af4a3 100644 --- a/open-sse/config/embeddingRegistry.ts +++ b/open-sse/config/embeddingRegistry.ts @@ -205,6 +205,15 @@ function resolveEmbeddingProviderId(providerId: string): string { } function normalizeProviderScopedModelId(providerId: string, modelId: string): string { + const resolvedProvider = resolveEmbeddingProviderId(providerId); + const provider = EMBEDDING_PROVIDERS[resolvedProvider]; + if (provider?.models.some((model) => model.id === modelId)) return modelId; + + const providerScopedModelId = `${resolvedProvider}/${modelId}`; + if (provider?.models.some((model) => model.id === providerScopedModelId)) { + return providerScopedModelId; + } + return modelId.startsWith(`${providerId}/`) ? modelId.slice(providerId.length + 1) : modelId; } diff --git a/open-sse/config/rerankRegistry.ts b/open-sse/config/rerankRegistry.ts index e47366c4e3..b075bf4357 100644 --- a/open-sse/config/rerankRegistry.ts +++ b/open-sse/config/rerankRegistry.ts @@ -83,6 +83,15 @@ function resolveRerankProviderId(providerId) { } function normalizeProviderScopedModelId(providerId, modelId) { + const resolvedProvider = resolveRerankProviderId(providerId); + const provider = RERANK_PROVIDERS[resolvedProvider]; + if (provider?.models.some((model) => model.id === modelId)) return modelId; + + const providerScopedModelId = `${resolvedProvider}/${modelId}`; + if (provider?.models.some((model) => model.id === providerScopedModelId)) { + return providerScopedModelId; + } + return modelId.startsWith(`${providerId}/`) ? modelId.slice(providerId.length + 1) : modelId; } diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 9759e1e967..99564b41f7 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1578,30 +1578,6 @@ export async function handleChatCore({ ); } - if (compressionSettings?.cavemanOutputMode?.enabled) { - try { - const { applyCavemanOutputMode } = await import("../services/compression/outputMode.ts"); - const outputMode = applyCavemanOutputMode( - body as Parameters[0], - compressionSettings.cavemanOutputMode - ); - if (outputMode.applied) { - body = outputMode.body as typeof body; - cavemanOutputModeApplied = true; - cavemanOutputModeIntensity = compressionSettings.cavemanOutputMode.intensity; - estimatedTokens = estimateTokens(JSON.stringify(body?.messages ?? body?.input ?? [])); - log?.debug?.("COMPRESSION", "Caveman output mode instruction applied"); - } else if (outputMode.skippedReason && outputMode.skippedReason !== "disabled") { - log?.debug?.("COMPRESSION", `Caveman output mode skipped: ${outputMode.skippedReason}`); - } - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Caveman output mode skipped: " + (err instanceof Error ? err.message : String(err)) - ); - } - } - // --- Modular Compression Pipeline (Phase 1 Lite + Phase 2 Standard/Caveman + Phase 3 Aggressive) --- // Runs BEFORE the existing reactive compressContext() to proactively reduce tokens. try { @@ -1642,7 +1618,9 @@ export async function handleChatCore({ comboMode === "lite" || comboMode === "standard" || comboMode === "aggressive" || - comboMode === "ultra" + comboMode === "ultra" || + comboMode === "rtk" || + comboMode === "stacked" ) { config = { ...config, @@ -1654,6 +1632,74 @@ export async function handleChatCore({ }; compressionComboKey = comboName; } + const routingComboIds = [ + comboConfig?.id, + comboName, + comboName.startsWith("combo/") ? comboName.substring(6) : null, + ].filter((id): id is string => typeof id === "string" && id.length > 0); + if (routingComboIds.length > 0) { + const { getCompressionComboForRoutingCombo } = + await import("../../src/lib/db/compressionCombos.ts"); + const assignedCompressionCombo = + routingComboIds + .map((id) => getCompressionComboForRoutingCombo(id)) + .find((combo) => combo !== null) ?? null; + if (assignedCompressionCombo && assignedCompressionCombo.pipeline.length > 0) { + const comboLanguagePacks = [ + ...new Set( + assignedCompressionCombo.languagePacks + .map((pack) => pack.trim()) + .filter((pack) => pack.length > 0) + ), + ]; + const comboOutputIntensity = ( + ["lite", "full", "ultra"].includes(assignedCompressionCombo.outputModeIntensity) + ? assignedCompressionCombo.outputModeIntensity + : (config.cavemanOutputMode?.intensity ?? "full") + ) as "lite" | "full" | "ultra"; + const comboDefaultLanguage = + comboLanguagePacks.find( + (pack) => pack === config.languageConfig?.defaultLanguage + ) ?? + comboLanguagePacks[0] ?? + config.languageConfig?.defaultLanguage ?? + "en"; + config = { + ...config, + compressionComboId: assignedCompressionCombo.id, + stackedPipeline: assignedCompressionCombo.pipeline, + languageConfig: { + ...(config.languageConfig ?? { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], + }), + enabled: true, + defaultLanguage: comboDefaultLanguage, + enabledPacks: + comboLanguagePacks.length > 0 + ? comboLanguagePacks + : (config.languageConfig?.enabledPacks ?? ["en"]), + }, + cavemanOutputMode: { + ...(config.cavemanOutputMode ?? { + enabled: false, + intensity: "full", + autoClarity: true, + }), + enabled: assignedCompressionCombo.outputMode, + intensity: comboOutputIntensity, + }, + comboOverrides: { + ...(config.comboOverrides ?? {}), + ...(comboName ? { [comboName]: "stacked" as const } : {}), + ...(comboConfig?.id ? { [comboConfig.id]: "stacked" as const } : {}), + }, + }; + compressionComboKey = comboName; + } + } } catch (err) { log?.debug?.( "COMPRESSION", @@ -1662,6 +1708,32 @@ export async function handleChatCore({ ); } } + if (config.cavemanOutputMode?.enabled) { + try { + const { applyCavemanOutputMode } = await import("../services/compression/outputMode.ts"); + const outputModeLanguage = + config.languageConfig?.enabled === true ? config.languageConfig.defaultLanguage : "en"; + const outputMode = applyCavemanOutputMode( + body as Parameters[0], + config.cavemanOutputMode, + outputModeLanguage + ); + if (outputMode.applied) { + body = outputMode.body as typeof body; + cavemanOutputModeApplied = true; + cavemanOutputModeIntensity = config.cavemanOutputMode.intensity; + estimatedTokens = estimateTokens(JSON.stringify(body?.messages ?? body?.input ?? [])); + log?.debug?.("COMPRESSION", "Caveman output mode instruction applied"); + } else if (outputMode.skippedReason && outputMode.skippedReason !== "disabled") { + log?.debug?.("COMPRESSION", `Caveman output mode skipped: ${outputMode.skippedReason}`); + } + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Caveman output mode skipped: " + (err instanceof Error ? err.message : String(err)) + ); + } + } const compressionInputBody = body as Record; const mode = selectCompressionStrategy( config, @@ -1706,6 +1778,9 @@ export async function handleChatCore({ combo_id: comboName ?? null, provider: provider ?? null, mode, + engine: result.stats.engine ?? mode, + compression_combo_id: + result.stats.compressionComboId ?? config.compressionComboId ?? null, original_tokens: result.stats.originalTokens, compressed_tokens: result.stats.compressedTokens, tokens_saved: tokensSaved, @@ -1714,6 +1789,8 @@ export async function handleChatCore({ estimated_usd_saved: estimatedUsdSaved || null, validation_fallback: result.stats.fallbackApplied ? 1 : 0, output_mode: cavemanOutputModeApplied ? cavemanOutputModeIntensity : null, + rtk_raw_output_pointer: result.stats.rtkRawOutputPointers?.[0]?.id ?? null, + rtk_raw_output_bytes: result.stats.rtkRawOutputPointers?.[0]?.bytes ?? null, }); } catch (err) { log?.debug?.( @@ -1776,6 +1853,8 @@ export async function handleChatCore({ combo_id: comboName ?? null, provider: provider ?? null, mode: "output-caveman", + engine: "caveman-output", + compression_combo_id: config.compressionComboId ?? null, original_tokens: estimatedTokens, compressed_tokens: estimatedTokens, tokens_saved: 0, diff --git a/open-sse/mcp-server/README.md b/open-sse/mcp-server/README.md index 43b9852940..80803a6013 100644 --- a/open-sse/mcp-server/README.md +++ b/open-sse/mcp-server/README.md @@ -145,12 +145,15 @@ omniroute --mcp ### Cache and Compression Tools -| # | Tool | Scopes | Description | -| --- | --------------------------------- | ------------------- | ---------------------------------------------------------------------------- | -| 21 | `omniroute_cache_stats` | `read:cache` | Semantic cache, prompt-cache, and idempotency statistics | -| 22 | `omniroute_cache_flush` | `write:cache` | Flush cache entries globally or by signature/model | -| 23 | `omniroute_compression_status` | `read:compression` | Compression settings, analytics summary, and provider-aware cache statistics | -| 24 | `omniroute_compression_configure` | `write:compression` | Configure compression mode and trigger thresholds at runtime | +| # | Tool | Scopes | Description | +| --- | ----------------------------------- | ------------------- | ---------------------------------------------------------------------------- | +| 21 | `omniroute_cache_stats` | `read:cache` | Semantic cache, prompt-cache, and idempotency statistics | +| 22 | `omniroute_cache_flush` | `write:cache` | Flush cache entries globally or by signature/model | +| 23 | `omniroute_compression_status` | `read:compression` | Compression settings, analytics summary, and provider-aware cache statistics | +| 24 | `omniroute_compression_configure` | `write:compression` | Configure compression mode and trigger thresholds at runtime | +| 25 | `omniroute_set_compression_engine` | `write:compression` | Set Caveman, RTK, or stacked compression mode and pipeline | +| 26 | `omniroute_list_compression_combos` | `read:compression` | List named compression combos and routing assignments | +| 27 | `omniroute_compression_combo_stats` | `read:compression` | Read analytics grouped by compression combo and engine | --- @@ -540,12 +543,12 @@ The MCP server supports **fine-grained scope enforcement** for multi-tenant envi | `read:usage` | `cost_report`, `explain_route`, `get_session_snapshot` | | `read:models` | `list_models_catalog` | | `read:cache` | `cache_stats` | -| `read:compression` | `compression_status` | +| `read:compression` | `compression_status`, `list_compression_combos`, `compression_combo_stats` | | `write:combos` | `switch_combo` | | `write:budget` | `set_budget_guard` | | `write:resilience` | `set_resilience_profile` | | `write:cache` | `cache_flush` | -| `write:compression` | `compression_configure` | +| `write:compression` | `compression_configure`, `set_compression_engine` | | `execute:completions` | `route_request`, `test_combo` | **Wildcard scopes:** Use `read:*` to grant all read scopes, or `*` for full access. @@ -581,7 +584,7 @@ mcp-server/ ├── audit.ts # SQLite audit logger (SHA-256 input hashing) ├── scopeEnforcement.ts # Fine-grained scope enforcement ├── schemas/ -│ ├── tools.ts # Zod schemas for all 16 tools (input/output/scopes) +│ ├── tools.ts # Zod schemas for core, cache, compression, and proxy tools │ ├── a2a.ts # A2A protocol types (Agent Card, Task, JSON-RPC) │ ├── audit.ts # Audit & routing decision types + hash helpers │ └── index.ts # Schema barrel export diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index fc9d0deef9..cacce07272 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -1064,10 +1064,12 @@ export const compressionStatusTool: McpToolDefinition< export const compressionConfigureInput = z.object({ enabled: z.boolean().optional(), strategy: z - .enum(["off", "lite", "standard", "aggressive", "ultra"]) + .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked"]) .optional() .describe("Compression mode"), - autoTriggerMode: z.enum(["off", "lite", "standard", "aggressive", "ultra"]).optional(), + autoTriggerMode: z + .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked"]) + .optional(), maxTokens: z .number() .int() @@ -1099,7 +1101,7 @@ export const compressionConfigureTool: McpToolDefinition< > = { name: "omniroute_compression_configure", description: - "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", inputSchema: compressionConfigureInput, outputSchema: compressionConfigureOutput, scopes: ["write:compression"], @@ -1108,6 +1110,72 @@ export const compressionConfigureTool: McpToolDefinition< sourceEndpoints: ["/api/compression/configure"], }; +export const setCompressionEngineInput = z.object({ + engine: z.enum(["off", "caveman", "rtk", "stacked"]).optional(), + cavemanIntensity: z.enum(["lite", "full", "ultra"]).optional(), + rtkIntensity: z.enum(["minimal", "standard", "aggressive"]).optional(), + outputMode: z.boolean().optional(), +}); + +export const setCompressionEngineOutput = z.object({ + success: z.boolean(), + settings: z.record(z.string(), z.unknown()), +}); + +export const setCompressionEngineTool: McpToolDefinition< + typeof setCompressionEngineInput, + typeof setCompressionEngineOutput +> = { + name: "omniroute_set_compression_engine", + description: "Set the active compression engine and Caveman/RTK runtime options.", + inputSchema: setCompressionEngineInput, + outputSchema: setCompressionEngineOutput, + scopes: ["write:compression"], + auditLevel: "full", + phase: 2, + sourceEndpoints: ["/api/settings/compression", "/api/context/rtk/config"], +}; + +export const listCompressionCombosInput = z.object({}); +export const listCompressionCombosOutput = z.object({ + combos: z.array(z.record(z.string(), z.unknown())), +}); + +export const listCompressionCombosTool: McpToolDefinition< + typeof listCompressionCombosInput, + typeof listCompressionCombosOutput +> = { + name: "omniroute_list_compression_combos", + description: "List compression combos and their engine pipelines.", + inputSchema: listCompressionCombosInput, + outputSchema: listCompressionCombosOutput, + scopes: ["read:compression"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/context/combos"], +}; + +export const compressionComboStatsInput = z.object({ + comboId: z.string().optional(), + since: z.enum(["24h", "7d", "30d", "all"]).optional(), +}); + +export const compressionComboStatsOutput = z.record(z.string(), z.unknown()); + +export const compressionComboStatsTool: McpToolDefinition< + typeof compressionComboStatsInput, + typeof compressionComboStatsOutput +> = { + name: "omniroute_compression_combo_stats", + description: "Get compression analytics grouped by engine and compression combo.", + inputSchema: compressionComboStatsInput, + outputSchema: compressionComboStatsOutput, + scopes: ["read:compression"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/context/analytics"], +}; + // ============ 1proxy Tools ============ export const oneproxyFetchInput = z.object({ @@ -1245,6 +1313,9 @@ export const MCP_TOOLS = [ cacheFlushTool, compressionStatusTool, compressionConfigureTool, + setCompressionEngineTool, + listCompressionCombosTool, + compressionComboStatsTool, oneproxyFetchTool, oneproxyRotateTool, oneproxyStatsTool, diff --git a/open-sse/mcp-server/tools/compressionTools.ts b/open-sse/mcp-server/tools/compressionTools.ts index 8732d09376..3e8c4d1ed2 100644 --- a/open-sse/mcp-server/tools/compressionTools.ts +++ b/open-sse/mcp-server/tools/compressionTools.ts @@ -13,6 +13,7 @@ import { } from "../../../src/lib/db/compression.ts"; import { getCompressionAnalyticsSummary } from "../../../src/lib/db/compressionAnalytics.ts"; import { getCacheStatsSummary } from "../../../src/lib/db/compressionCacheStats.ts"; +import { listCompressionCombos } from "../../../src/lib/db/compressionCombos.ts"; import type { McpToolExtraLike } from "../scopeEnforcement.ts"; import { getMcpDescriptionCompressionStats } from "../descriptionCompressor.ts"; @@ -38,6 +39,8 @@ export async function handleCompressionStatus( tokensSaved: number; avgCompressionRatio: number; byMode: Record; + byEngine: Record; + byCompressionCombo: Record; validationFallbacks: number; requestsWithReceipts: number; realUsage: { @@ -89,6 +92,8 @@ export async function handleCompressionStatus( tokensSaved: analyticsSummary.totalTokensSaved, avgCompressionRatio: analyticsSummary.avgSavingsPct, byMode: analyticsSummary.byMode ?? {}, + byEngine: analyticsSummary.byEngine ?? {}, + byCompressionCombo: analyticsSummary.byCompressionCombo ?? {}, validationFallbacks: analyticsSummary.validationFallbacks, requestsWithReceipts: analyticsSummary.realUsage.requestsWithReceipts, realUsage: analyticsSummary.realUsage, @@ -213,7 +218,64 @@ export async function handleCompressionConfigure( } import { z } from "zod"; -import { compressionStatusInput, compressionConfigureInput } from "../schemas/tools.ts"; +import { + compressionStatusInput, + compressionConfigureInput, + setCompressionEngineInput, + listCompressionCombosInput, + compressionComboStatsInput, +} from "../schemas/tools.ts"; + +export async function handleSetCompressionEngine( + args: z.infer +): Promise<{ success: boolean; settings: Record }> { + const updates: Record = { enabled: true }; + if (args.engine) { + updates.defaultMode = args.engine === "caveman" ? "standard" : args.engine; + if (args.engine === "off") updates.enabled = false; + } + if (args.cavemanIntensity) { + const current = await getCompressionSettings(); + updates.cavemanConfig = { + ...(current.cavemanConfig ?? {}), + intensity: args.cavemanIntensity, + }; + } + if (args.rtkIntensity) { + const current = await getCompressionSettings(); + updates.rtkConfig = { + ...(current.rtkConfig ?? {}), + intensity: args.rtkIntensity, + }; + } + if (args.outputMode !== undefined) { + const current = await getCompressionSettings(); + updates.cavemanOutputMode = { + ...(current.cavemanOutputMode ?? {}), + enabled: args.outputMode, + }; + } + const settings = await updateCompressionSettings(updates); + return { success: true, settings: settings as unknown as Record }; +} + +export async function handleListCompressionCombos(): Promise<{ + combos: ReturnType; +}> { + return { combos: listCompressionCombos() }; +} + +export async function handleCompressionComboStats( + args: z.infer +): Promise> { + const summary = getCompressionAnalyticsSummary(args.since === "all" ? undefined : args.since); + if (!args.comboId) return summary as unknown as Record; + return { + comboId: args.comboId, + summary, + combo: summary.byCompressionCombo[args.comboId] ?? { count: 0, tokensSaved: 0 }, + }; +} export const compressionTools = { omniroute_compression_status: { @@ -226,8 +288,27 @@ export const compressionTools = { omniroute_compression_configure: { name: "omniroute_compression_configure", description: - "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", inputSchema: compressionConfigureInput, handler: (args: z.infer) => handleCompressionConfigure(args), }, + omniroute_set_compression_engine: { + name: "omniroute_set_compression_engine", + description: "Set the active compression engine and Caveman/RTK runtime options.", + inputSchema: setCompressionEngineInput, + handler: (args: z.infer) => handleSetCompressionEngine(args), + }, + omniroute_list_compression_combos: { + name: "omniroute_list_compression_combos", + description: "List compression combos and their engine pipelines.", + inputSchema: listCompressionCombosInput, + handler: (_args: z.infer) => handleListCompressionCombos(), + }, + omniroute_compression_combo_stats: { + name: "omniroute_compression_combo_stats", + description: "Get compression analytics grouped by engine and compression combo.", + inputSchema: compressionComboStatsInput, + handler: (args: z.infer) => + handleCompressionComboStats(args), + }, }; diff --git a/open-sse/services/AGENTS.md b/open-sse/services/AGENTS.md index b7b99308d0..d13f4b1ea4 100644 --- a/open-sse/services/AGENTS.md +++ b/open-sse/services/AGENTS.md @@ -51,12 +51,15 @@ ### Prompt Compression Pipeline - **`compression/`** — Modular prompt compression running proactively before `contextManager.ts`. - - `strategySelector.ts` — Selects mode (off/lite/standard/aggressive/ultra) with combo overrides and auto-trigger. + - `strategySelector.ts` — Selects mode (off/lite/standard/aggressive/ultra/rtk/stacked) with compression combo assignments, combo overrides, and auto-trigger. - `lite.ts` — 5 lite techniques: whitespace collapse, system prompt dedup, tool result truncation, redundant removal, image URL placeholder. + - `caveman.ts` / `cavemanRules.ts` — Caveman-style semantic condensation with file-loaded rule packs and language-aware rule selection. + - `engines/registry.ts` — Engine registry used by standalone RTK/Caveman execution and stacked pipelines. + - `engines/rtk/` — RTK tool-output compression: command detection, JSON filter packs, deduplication, smart truncation, ANSI/code noise stripping. - `stats.ts` — Per-request compression stats (original/compressed tokens, savings %, techniques). - `types.ts` — Shared types (`CompressionMode`, `CompressionConfig`, `CompressionStats`, `CompressionResult`). - `index.ts` — Barrel re-exports. - - Phase 1: lite mode only. Standard/aggressive/ultra = Phase 2. + - Dashboard/API surface: `/dashboard/context/caveman`, `/dashboard/context/rtk`, `/dashboard/context/combos`, `/api/context/*`, and `/api/compression/preview`. ### Auto-Routing & Adaptive diff --git a/open-sse/services/compression/caveman.ts b/open-sse/services/compression/caveman.ts index 464a6ac3a8..a57195b02c 100644 --- a/open-sse/services/compression/caveman.ts +++ b/open-sse/services/compression/caveman.ts @@ -1,10 +1,17 @@ -import type { CavemanConfig, CavemanRule, CompressionResult, CompressionMode } from "./types.ts"; +import type { + CavemanConfig, + CavemanRule, + CompressionResult, + CompressionMode, + CompressionStats, +} from "./types.ts"; import { DEFAULT_CAVEMAN_CONFIG } from "./types.ts"; import { CAVEMAN_RULES, getRulesForContext } from "./cavemanRules.ts"; import { extractPreservedBlocks, restorePreservedBlocks } from "./preservation.ts"; import { createCompressionStats, estimateCompressionTokens } from "./stats.ts"; import { validateCompression } from "./validation.ts"; import { mapTextContent } from "./messageContent.ts"; +import { detectCompressionLanguage } from "./languageDetector.ts"; interface ChatMessage { role: string; @@ -18,15 +25,9 @@ interface ChatRequestBody { } const RULE_KEYWORDS: Record = { - redundant_phrasing: [ - "make sure", - "be sure", - "due to the fact", - "the reason is because", - "it is important", - "you should", - "remember to", - ], + redundant_phrasing: ["make sure", "be sure"], + redundant_because: ["due to the fact", "the reason is because"], + redundant_directive: ["it is important", "you should", "remember to"], pleasantries: ["sure", "certainly", "of course", "happy to"], polite_framing: [ "please", @@ -72,6 +73,9 @@ const RULE_KEYWORDS: Record = { self_reference: ["i am trying to", "i am working on", "i have been"], excessive_gratitude: ["thank you so much", "thanks in advance", "i really appreciate"], qualifier_removal: ["a bit", "a little", "somewhat", "kind of", "sort of"], + softeners: ["if possible", "when you get a chance", "at your convenience", "just wondering"], + uncertainty_fillers: ["i guess", "i suppose", "more or less", "in a way"], + assistant_fillers: ["here's", "below is", "this is"], compound_collapse: ["and any potential"], explanatory_prefix: [ "the function appears to be handling", @@ -98,6 +102,7 @@ const RULE_KEYWORDS: Record = { list_conjunction: ["and also", "as well as"], purpose_phrases: ["in order to", "so as to"], redundant_quantifiers: ["each and every", "any and all"], + all_quantifier: ["any and all"], verbose_connectors: ["furthermore", "additionally", "moreover", "in addition"], transition_removal: ["on the other hand", "in contrast", "however"], emphasis_removal: ["very", "really", "extremely", "highly", "quite"], @@ -122,49 +127,36 @@ const RULE_KEYWORDS: Record = { ], reestablished_context: ["going back to the code above", "referring back to", "returning to"], summary_replacement: ["to summarize", "in summary of our conversation", "to recap"], - ultra_abbreviations: [ - "database", - "configuration", - "function", - "request", - "response", + ultra_abbreviations: ["database"], + ultra_config_abbreviation: ["configuration"], + ultra_function_abbreviation: ["function"], + ultra_request_abbreviation: ["request"], + ultra_response_abbreviation: ["response"], + ultra_implementation_abbreviation: ["implementation"], + ultra_authentication_abbreviation: ["authentication"], + ultra_authorization_abbreviation: ["authorization"], + ultra_application_abbreviation: ["application"], + ultra_dependency_abbreviation: ["dependency", "dependencies"], + ultra_common_abbreviations: [ "implementation", "authentication", "authorization", "application", "dependency", + "dependencies", ], }; -const KEYWORD_WORDS = new Map(); +const ARTICLE_HINT_RE = /\b(?:a|an|the)\b/; -function extractLowerWords(lowerText: string): Set { - return new Set(lowerText.match(/[a-z]+(?:'[a-z]+)?/g) ?? []); -} - -function getKeywordWords(keyword: string): string[] { - const cached = KEYWORD_WORDS.get(keyword); - if (cached) return cached; - const words = keyword.match(/[a-z]+(?:'[a-z]+)?/g) ?? []; - KEYWORD_WORDS.set(keyword, words); - return words; -} - -function hasKeywordHint(keyword: string, words: Set): boolean { - const keywordWords = getKeywordWords(keyword); - return keywordWords.length === 0 || keywordWords.every((word) => words.has(word)); -} - -function shouldAttemptRule(ruleName: string, lowerText: string, words: Set): boolean { +function shouldAttemptRule(ruleName: string, lowerText: string): boolean { if (ruleName === "articles") { - return words.has("a") || words.has("an") || words.has("the"); + ARTICLE_HINT_RE.lastIndex = 0; + return ARTICLE_HINT_RE.test(lowerText); } const keywords = RULE_KEYWORDS[ruleName]; - return ( - !keywords || - keywords.some((keyword) => hasKeywordHint(keyword, words) && lowerText.includes(keyword)) - ); + return !keywords || keywords.some((keyword) => lowerText.includes(keyword)); } export function applyRulesToText( @@ -173,11 +165,10 @@ export function applyRulesToText( ): { text: string; appliedRules: string[] } { let result = text; const lowerResult = text.toLowerCase(); - const lowerWords = extractLowerWords(lowerResult); const appliedRules: string[] = []; for (const rule of rules) { - if (!shouldAttemptRule(rule.name, lowerResult, lowerWords)) continue; + if (!shouldAttemptRule(rule.name, lowerResult)) continue; const before = result; const { pattern, replacement } = rule; @@ -217,6 +208,29 @@ function recapitalizeSentences(text: string): string { }); } +function createCavemanStats( + originalTokens: number, + compressedTokens: number, + techniquesUsed: string[], + rulesApplied: string[] | undefined, + durationMs: number +): CompressionStats { + const savingsPercent = + originalTokens > 0 + ? Math.round(((originalTokens - compressedTokens) / originalTokens) * 10000) / 100 + : 0; + return { + originalTokens, + compressedTokens, + savingsPercent, + techniquesUsed, + mode: "standard", + timestamp: Date.now(), + ...(rulesApplied && rulesApplied.length > 0 ? { rulesApplied } : {}), + durationMs, + }; +} + function compileUserPreservePatterns(patterns: string[]): { patterns: RegExp[]; warnings: string[]; @@ -320,7 +334,16 @@ export function cavemanCompress( : { text: textPart, blocks: [] }; preservedBlockCount += blocks.length; - const rules = getRulesForContext(msg.role, config.intensity).filter( + const detectedLanguage = config.autoDetectLanguage + ? detectCompressionLanguage(textPart) + : (config.language ?? "en"); + const enabledPacks = config.enabledLanguagePacks ?? ["en", detectedLanguage]; + const language = enabledPacks.includes(detectedLanguage) + ? detectedLanguage + : enabledPacks.includes("en") + ? "en" + : detectedLanguage; + const rules = getRulesForContext(msg.role, config.intensity, language).filter( (rule) => !config.skipRules.includes(rule.name) ); const { text: rulesApplied, appliedRules } = applyRulesToText(extractedText, rules); @@ -365,10 +388,9 @@ export function cavemanCompress( const durationMs = performance.now() - startMs; const uniqueRules = [...new Set(allAppliedRules)]; - const stats = createCompressionStats( - body as unknown as Record, - { ...body, messages: compressedMessages } as unknown as Record, - "standard" as CompressionMode, + const stats = createCavemanStats( + totalOriginalTokens, + totalCompressedTokens, uniqueRules.length > 0 ? ["caveman-rules"] : [], uniqueRules.length > 0 ? uniqueRules : undefined, Math.round(durationMs * 100) / 100 diff --git a/open-sse/services/compression/cavemanRules.ts b/open-sse/services/compression/cavemanRules.ts index 906f64fc28..9e74aa3de9 100644 --- a/open-sse/services/compression/cavemanRules.ts +++ b/open-sse/services/compression/cavemanRules.ts @@ -1,4 +1,5 @@ import type { CavemanRule } from "./types.ts"; +import { loadAllRulesForLanguage } from "./ruleLoader.ts"; const CAVEMAN_RULES: CavemanRule[] = [ // ── Category 1: Filler Removal (10+ rules) ────────────────────────── @@ -392,13 +393,22 @@ const INTENSITY_RANK = { lite: 0, full: 1, ultra: 2 } as const; export function getRulesForContext( context: string, - intensity: "lite" | "full" | "ultra" = "full" + intensity: "lite" | "full" | "ultra" = "full", + language = "en" ): CavemanRule[] { const rank = INTENSITY_RANK[intensity] ?? INTENSITY_RANK.full; - return CAVEMAN_RULES.filter((rule) => { + const fileRules = language ? loadAllRulesForLanguage(language) : []; + const rules = fileRules.length > 0 ? fileRules : CAVEMAN_RULES; + const selected = rules.filter((rule) => { const minRank = INTENSITY_RANK[rule.minIntensity ?? "lite"]; return (rule.context === "all" || rule.context === context) && minRank <= rank; }); + return selected.length > 0 + ? selected + : CAVEMAN_RULES.filter((rule) => { + const minRank = INTENSITY_RANK[rule.minIntensity ?? "lite"]; + return (rule.context === "all" || rule.context === context) && minRank <= rank; + }); } export function getRuleByName(name: string): CavemanRule | undefined { diff --git a/open-sse/services/compression/diffHelper.ts b/open-sse/services/compression/diffHelper.ts index 80632c7e51..5dbebd7b1d 100644 --- a/open-sse/services/compression/diffHelper.ts +++ b/open-sse/services/compression/diffHelper.ts @@ -16,10 +16,31 @@ export interface CompressionPreviewDiff { fallbackApplied: boolean; } +export interface CompressionPreviewDiffOptions { + maxTokenProduct?: number; +} + +export const DEFAULT_MAX_PREVIEW_DIFF_TOKEN_PRODUCT = 1_000_000; + function tokenize(text: string): string[] { return text.match(/\s+|[^\s]+/g) ?? []; } +function getDiffSkipWarning( + original: string, + compressed: string, + options: CompressionPreviewDiffOptions = {} +): string | null { + const maxTokenProduct = options.maxTokenProduct ?? DEFAULT_MAX_PREVIEW_DIFF_TOKEN_PRODUCT; + if (maxTokenProduct <= 0) return null; + + const originalTokens = tokenize(original).length; + const compressedTokens = tokenize(compressed).length; + if (originalTokens * compressedTokens <= maxTokenProduct) return null; + + return `Preview diff omitted because token product ${originalTokens}x${compressedTokens} exceeds safe limit ${maxTokenProduct}.`; +} + export function buildCompressionDiff( original: string, compressed: string @@ -69,19 +90,27 @@ export function buildCompressionDiff( export function buildCompressionPreviewDiff( original: string, compressed: string, - stats: CompressionStats | null | undefined + stats: CompressionStats | null | undefined, + options: CompressionPreviewDiffOptions = {} ): CompressionPreviewDiff { const validation = validateCompression(original, compressed); const preserved = extractPreservedBlocks(original).blocks.map((block) => ({ kind: block.kind, preview: block.content.replace(/\s+/g, " ").slice(0, 120), })); + const diffSkipWarning = getDiffSkipWarning(original, compressed, options); return { - segments: buildCompressionDiff(original, compressed), + segments: diffSkipWarning + ? [{ type: "same", text: "[diff omitted: input too large]" }] + : buildCompressionDiff(original, compressed), preservedBlocks: preserved, ruleRemovals: stats?.rulesApplied ?? [], - validationWarnings: [...(stats?.validationWarnings ?? []), ...validation.warnings], + validationWarnings: [ + ...(stats?.validationWarnings ?? []), + ...validation.warnings, + ...(diffSkipWarning ? [diffSkipWarning] : []), + ], validationErrors: [...(stats?.validationErrors ?? []), ...validation.errors], fallbackApplied: Boolean(stats?.fallbackApplied || validation.fallbackApplied), }; diff --git a/open-sse/services/compression/engines/cavemanAdapter.ts b/open-sse/services/compression/engines/cavemanAdapter.ts new file mode 100644 index 0000000000..27bb44d87d --- /dev/null +++ b/open-sse/services/compression/engines/cavemanAdapter.ts @@ -0,0 +1,254 @@ +import { applyLiteCompression } from "../lite.ts"; +import { cavemanCompress } from "../caveman.ts"; +import { compressAggressive } from "../aggressive.ts"; +import { ultraCompress } from "../ultra.ts"; +import { createCompressionStats } from "../stats.ts"; +import type { CavemanIntensity } from "../types.ts"; +import type { CompressionEngine, EngineConfigField, EngineValidationResult } from "./types.ts"; + +const CAVEMAN_INTENSITIES: CavemanIntensity[] = ["lite", "full", "ultra"]; + +const CAVEMAN_SCHEMA: EngineConfigField[] = [ + { + key: "intensity", + type: "select", + label: "Intensity", + defaultValue: "full", + options: CAVEMAN_INTENSITIES.map((value) => ({ value, label: value })), + }, + { + key: "minMessageLength", + type: "number", + label: "Minimum message length", + defaultValue: 50, + min: 0, + max: 10000, + }, + { + key: "enabled", + type: "boolean", + label: "Enabled", + defaultValue: true, + }, +]; + +function ok(): EngineValidationResult { + return { valid: true, errors: [] }; +} + +function validateCavemanLikeConfig(config: Record): EngineValidationResult { + const errors: string[] = []; + if ( + config.intensity !== undefined && + !CAVEMAN_INTENSITIES.includes(config.intensity as CavemanIntensity) + ) { + errors.push("intensity must be lite, full, or ultra"); + } + if ( + config.minMessageLength !== undefined && + (typeof config.minMessageLength !== "number" || config.minMessageLength < 0) + ) { + errors.push("minMessageLength must be a non-negative number"); + } + if (config.enabled !== undefined && typeof config.enabled !== "boolean") { + errors.push("enabled must be a boolean"); + } + return { valid: errors.length === 0, errors }; +} + +export const liteEngine: CompressionEngine = { + id: "lite", + name: "Lite", + description: "Fast whitespace, tool-result and image URL reduction.", + icon: "compress", + targets: ["messages", "tool_results"], + stackable: true, + stackPriority: 5, + metadata: { + id: "lite", + name: "Lite", + description: "Fast whitespace, tool-result and image URL reduction.", + inputScope: "messages", + targetLatencyMs: 1, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + return applyLiteCompression(body, { + ...options, + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; + +export const cavemanEngine: CompressionEngine = { + id: "caveman", + name: "Caveman", + description: "Rule-based message compression with preservation and validation.", + icon: "compress", + targets: ["messages"], + stackable: true, + stackPriority: 20, + metadata: { + id: "caveman", + name: "Caveman", + description: "Rule-based message compression with preservation and validation.", + inputScope: "messages", + targetLatencyMs: 1, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const cavemanConfig = { + ...(options?.config?.cavemanConfig ?? {}), + ...(options?.stepConfig ?? {}), + ...(options?.config?.languageConfig?.enabled + ? { + language: options.config.languageConfig.defaultLanguage, + autoDetectLanguage: options.config.languageConfig.autoDetect, + enabledLanguagePacks: options.config.languageConfig.enabledPacks, + } + : {}), + ...(options?.config?.preserveSystemPrompt !== false + ? { + compressRoles: (options?.config?.cavemanConfig?.compressRoles ?? ["user"]).filter( + (role) => role !== "system" + ), + } + : {}), + }; + return cavemanCompress(body as Parameters[0], cavemanConfig); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return CAVEMAN_SCHEMA; + }, + validateConfig(config) { + return validateCavemanLikeConfig(config); + }, +}; + +export const aggressiveEngine: CompressionEngine = { + id: "aggressive", + name: "Aggressive", + description: "Summarization, tool result compression and progressive aging.", + icon: "speed", + targets: ["messages", "tool_results"], + stackable: true, + stackPriority: 30, + metadata: { + id: "aggressive", + name: "Aggressive", + description: "Summarization, tool result compression and progressive aging.", + inputScope: "messages", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const messages = (body.messages ?? []) as Array<{ + role: string; + content?: string | Array<{ type: string; text?: string }>; + [key: string]: unknown; + }>; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + const aggressiveConfig = { + ...(options?.config?.aggressive ?? {}), + ...(options?.stepConfig ?? {}), + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }; + const result = compressAggressive(messages, aggressiveConfig); + const compressedBody = { ...body, messages: result.messages }; + return { + body: compressedBody, + compressed: result.stats.savingsPercent > 0, + stats: createCompressionStats( + body, + compressedBody, + "aggressive", + ["aggressive"], + result.stats.rulesApplied, + result.stats.durationMs + ), + }; + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; + +export const ultraEngine: CompressionEngine = { + id: "ultra", + name: "Ultra", + description: "Heuristic token pruning with optional local SLM fallback.", + icon: "bolt", + targets: ["messages"], + stackable: true, + stackPriority: 40, + metadata: { + id: "ultra", + name: "Ultra", + description: "Heuristic token pruning with optional local SLM fallback.", + inputScope: "messages", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const messages = (body.messages ?? []) as Array<{ + role: string; + content?: string | unknown[]; + [key: string]: unknown; + }>; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + const ultraConfig = { + ...(options?.config?.ultra ?? {}), + ...(options?.stepConfig ?? {}), + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }; + const result = ultraCompress(messages, ultraConfig); + const compressedBody = { ...body, messages: result.messages }; + return { + body: compressedBody, + compressed: result.stats.savingsPercent > 0, + stats: createCompressionStats( + body, + compressedBody, + "ultra", + ["ultra"], + result.stats.rulesApplied, + result.stats.durationMs + ), + }; + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; diff --git a/open-sse/services/compression/engines/index.ts b/open-sse/services/compression/engines/index.ts new file mode 100644 index 0000000000..07094ee9ac --- /dev/null +++ b/open-sse/services/compression/engines/index.ts @@ -0,0 +1,14 @@ +import { registerCompressionEngine, getCompressionEngine } from "./registry.ts"; +import { aggressiveEngine, cavemanEngine, liteEngine, ultraEngine } from "./cavemanAdapter.ts"; +import { rtkEngine } from "./rtk/index.ts"; + +let registered = false; + +export function registerBuiltinCompressionEngines(): void { + const engines = [liteEngine, cavemanEngine, aggressiveEngine, ultraEngine, rtkEngine]; + if (registered && engines.every((engine) => getCompressionEngine(engine.id))) return; + for (const engine of engines) { + if (!getCompressionEngine(engine.id)) registerCompressionEngine(engine); + } + registered = true; +} diff --git a/open-sse/services/compression/engines/registry.ts b/open-sse/services/compression/engines/registry.ts new file mode 100644 index 0000000000..85d6a50cdd --- /dev/null +++ b/open-sse/services/compression/engines/registry.ts @@ -0,0 +1,87 @@ +import type { CompressionEngine, EngineRegistryEntry, EngineValidationResult } from "./types.ts"; + +const ENGINES = new Map(); + +function assertValidEngine(engine: CompressionEngine): void { + if ( + !engine?.id || + typeof engine.apply !== "function" || + typeof engine.compress !== "function" || + typeof engine.getConfigSchema !== "function" || + typeof engine.validateConfig !== "function" + ) { + throw new Error("Invalid compression engine registration"); + } +} + +export function registerEngine( + engine: CompressionEngine, + defaultConfig: Record = {} +): void { + assertValidEngine(engine); + const validation = engine.validateConfig(defaultConfig); + if (!validation.valid) { + throw new Error(`Invalid default config for ${engine.id}: ${validation.errors.join("; ")}`); + } + ENGINES.set(engine.id, { + engine, + enabled: true, + config: { ...defaultConfig }, + }); +} + +export function registerCompressionEngine(engine: CompressionEngine): void { + registerEngine(engine); +} + +export function unregisterCompressionEngine(id: string): boolean { + return ENGINES.delete(id); +} + +export function getEngine(id: string): CompressionEngine | null { + return ENGINES.get(id)?.engine ?? null; +} + +export function getCompressionEngine(id: string): CompressionEngine | null { + return getEngine(id); +} + +export function getEngineEntry(id: string): EngineRegistryEntry | null { + return ENGINES.get(id) ?? null; +} + +export function listEngines(): EngineRegistryEntry[] { + return Array.from(ENGINES.values()); +} + +export function listCompressionEngines(): CompressionEngine[] { + return listEngines().map((entry) => entry.engine); +} + +export function listEnabledEngines(): EngineRegistryEntry[] { + return listEngines().filter((entry) => entry.enabled); +} + +export function setEngineEnabled(id: string, enabled: boolean): boolean { + const entry = ENGINES.get(id); + if (!entry) return false; + entry.enabled = enabled; + return true; +} + +export function updateEngineConfig( + id: string, + config: Record +): EngineValidationResult { + const entry = ENGINES.get(id); + if (!entry) return { valid: false, errors: [`Unknown compression engine: ${id}`] }; + const nextConfig = { ...entry.config, ...config }; + const validation = entry.engine.validateConfig(nextConfig); + if (!validation.valid) return validation; + entry.config = nextConfig; + return { valid: true, errors: [] }; +} + +export function clearCompressionEngineRegistry(): void { + ENGINES.clear(); +} diff --git a/open-sse/services/compression/engines/rtk/codeStripper.ts b/open-sse/services/compression/engines/rtk/codeStripper.ts new file mode 100644 index 0000000000..27c3512910 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/codeStripper.ts @@ -0,0 +1,128 @@ +export type CodeLanguage = + | "javascript" + | "typescript" + | "python" + | "rust" + | "go" + | "ruby" + | "java" + | "unknown"; + +export interface CodeStripperOptions { + removeComments?: boolean; + removeEmptyLines?: boolean; + collapseWhitespace?: boolean; + preserveDocstrings?: boolean; +} + +const LANGUAGE_ALIASES: Record = { + js: "javascript", + jsx: "javascript", + javascript: "javascript", + ts: "typescript", + tsx: "typescript", + typescript: "typescript", + py: "python", + python: "python", + rs: "rust", + rust: "rust", + go: "go", + rb: "ruby", + ruby: "ruby", + java: "java", +}; + +export function normalizeCodeLanguage(language?: string | null): CodeLanguage { + if (!language) return "unknown"; + return LANGUAGE_ALIASES[language.trim().toLowerCase()] ?? "unknown"; +} + +export function detectCodeLanguage(text: string): CodeLanguage { + if (/\b(?:interface|type)\s+\w+\s*=|:\s*(?:string|number|boolean)\b/.test(text)) { + return "typescript"; + } + if (/\b(?:const|let|function|import|export)\b|=>/.test(text)) return "javascript"; + if (/\bdef\s+\w+\(|\bimport\s+\w+|print\(/.test(text)) return "python"; + if (/\bfn\s+\w+\(|\blet\s+mut\b|println!\(/.test(text)) return "rust"; + if (/\bfunc\s+\w+\(|package\s+\w+/.test(text)) return "go"; + if (/\bclass\s+\w+|System\.out\.println/.test(text)) return "java"; + if (/\bdef\s+\w+|puts\s+|end\s*$/.test(text)) return "ruby"; + return "unknown"; +} + +function removeSlashComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/\s+\/\/\s.*$/gm, ""); +} + +function removeHashComments(text: string): string { + return text.replace(/^\s*#.*$/gm, "").replace(/\s+#\s.*$/gm, ""); +} + +function stripByLanguage( + text: string, + language: CodeLanguage, + options: Required +): string { + if (!options.removeComments) return text; + + if (language === "javascript" || language === "typescript" || language === "rust") { + return removeSlashComments(text); + } + if (language === "go" || language === "java") return removeSlashComments(text); + if (language === "python") { + const withoutDocstrings = options.preserveDocstrings + ? text + : text.replace(/("""[\s\S]*?"""|'''[\s\S]*?''')/g, ""); + return removeHashComments(withoutDocstrings); + } + if (language === "ruby") { + return removeHashComments(text.replace(/^=begin[\s\S]*?^=end/gm, "")); + } + return text; +} + +export function stripCode( + text: string, + language: CodeLanguage = "unknown", + options: CodeStripperOptions = {} +): { + text: string; + strippedLines: number; + language: CodeLanguage; +} { + const resolvedLanguage = language === "unknown" ? detectCodeLanguage(text) : language; + const opts: Required = { + removeComments: options.removeComments !== false, + removeEmptyLines: options.removeEmptyLines !== false, + collapseWhitespace: options.collapseWhitespace !== false, + preserveDocstrings: options.preserveDocstrings === true, + }; + const originalLines = text.split(/\r?\n/).length; + let result = stripByLanguage(text, resolvedLanguage, opts); + + if (opts.removeEmptyLines) result = result.replace(/^\s*$(?:\r?\n)?/gm, ""); + if (opts.collapseWhitespace) { + result = result + .split(/\r?\n/) + .map((line) => line.replace(/[ \t]+$/g, "")) + .join("\n"); + } + + result = result.trim(); + const strippedLines = Math.max(0, originalLines - (result ? result.split(/\r?\n/).length : 0)); + return { text: result, strippedLines, language: resolvedLanguage }; +} + +export function stripCodeComments( + text: string, + language = "typescript" +): { + text: string; + stripped: boolean; +} { + const result = stripCode(text, normalizeCodeLanguage(language)); + return { text: result.text, stripped: result.strippedLines > 0 || result.text !== text }; +} diff --git a/open-sse/services/compression/engines/rtk/commandDetector.ts b/open-sse/services/compression/engines/rtk/commandDetector.ts new file mode 100644 index 0000000000..31806c65b7 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/commandDetector.ts @@ -0,0 +1,465 @@ +export interface CommandDetectionResult { + type: string; + command: string | null; + confidence: number; + category: + | "git" + | "test" + | "build" + | "shell" + | "docker" + | "package" + | "infra" + | "cloud" + | "generic"; + matchedPatterns: string[]; +} + +type Detector = { + type: string; + category: CommandDetectionResult["category"]; + commandPatterns: RegExp[]; + contentPatterns: RegExp[]; +}; + +const COMMAND_PREFIXES = [ + "git", + "make", + "terraform", + "tofu", + "opentofu", + "systemctl", + "npm", + "pnpm", + "yarn", + "vitest", + "jest", + "pytest", + "python", + "go", + "cargo", + "tsc", + "eslint", + "webpack", + "vite", + "biome", + "prettier", + "turbo", + "nx", + "playwright", + "ruff", + "mypy", + "pip", + "uv", + "poetry", + "golangci-lint", + "bundle", + "rubocop", + "docker", + "aws", + "gcloud", + "ssh", + "rsync", + "curl", + "wget", + "ls", + "find", + "grep", + "rg", + "ag", + "ps", + "df", + "du", +]; + +const COMMAND_PREFIX_PATTERN = new RegExp(`^(?:${COMMAND_PREFIXES.join("|")})\\b`); + +const DETECTORS: Detector[] = [ + { + type: "git-status", + category: "git", + commandPatterns: [/^git\s+status\b/i], + contentPatterns: [ + /^On branch /m, + /^Changes (?:not staged|to be committed)/m, + /^Untracked files:/m, + ], + }, + { + type: "git-branch", + category: "git", + commandPatterns: [/^git\s+branch\b/i, /^git\s+checkout\b/i, /^git\s+switch\b/i], + contentPatterns: [/^\*\s+\S+/m, /Switched to (?:a new )?branch/i, /Already on ['"][^'"]+['"]/i], + }, + { + type: "git-diff", + category: "git", + commandPatterns: [/^git\s+diff\b/i, /^git\s+show\b/i], + contentPatterns: [/^diff --git /m, /^@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@/m], + }, + { + type: "git-log", + category: "git", + commandPatterns: [/^git\s+log\b/i], + contentPatterns: [/^commit [0-9a-f]{7,40}/m, /^Author: /m], + }, + { + type: "make", + category: "build", + commandPatterns: [/^make\b/i], + contentPatterns: [/^make\[\d+\]: (?:Entering|Leaving) directory/m, /make: \*\*\* /], + }, + { + type: "terraform-plan", + category: "infra", + commandPatterns: [/^terraform\s+plan\b/i], + contentPatterns: [/Terraform will perform the following actions:/, /Plan: \d+ to add/i], + }, + { + type: "tofu-plan", + category: "infra", + commandPatterns: [/^(?:tofu|opentofu)\s+plan\b/i], + contentPatterns: [/OpenTofu will perform the following actions:/, /Plan: \d+ to add/i], + }, + { + type: "systemctl-status", + category: "infra", + commandPatterns: [/^systemctl\s+status\b/i], + contentPatterns: [/^\s*Loaded:\s+/m, /^\s*Active:\s+/m, /^●\s+\S+\.service/m], + }, + { + type: "test-vitest", + category: "test", + commandPatterns: [/^vitest\b/i, /^npm\s+(?:run\s+)?test:vitest\b/i], + contentPatterns: [/\bvitest\b/i, /^ ✓ /m, /^ ❯ /m, /Test Files\s+\d+\s+(?:passed|failed)/i], + }, + { + type: "test-jest", + category: "test", + commandPatterns: [/^jest\b/i, /^npm\s+(?:run\s+)?test\b/i], + contentPatterns: [/Test Suites:\s+\d+/i, /Tests:\s+\d+/i, /^PASS\s+/m, /^FAIL\s+/m], + }, + { + type: "test-pytest", + category: "test", + commandPatterns: [/^pytest\b/i, /^python\s+-m\s+pytest\b/i], + contentPatterns: [/=+\s+(?:\d+\s+)?(?:passed|failed|errors?)/i, /^E\s+/m, /^FAILED /m], + }, + { + type: "test-cargo", + category: "test", + commandPatterns: [/^cargo\s+test\b/i, /^cargo\s+nextest\b/i], + contentPatterns: [ + /^running \d+ tests?/m, + /^test\s+[\w:.-]+\s+\.\.\.\s+(?:ok|FAILED|ignored)/m, + /test result:\s+(?:ok|FAILED)/i, + ], + }, + { + type: "test-go", + category: "test", + commandPatterns: [/^go\s+test\b/i], + contentPatterns: [/^(?:ok|FAIL)\s+[\w./-]+\s+[\d.]+s/m, /^--- FAIL: /m, /^panic: /m], + }, + { + type: "build-typescript", + category: "build", + commandPatterns: [/^tsc\b/i, /^npm\s+run\s+typecheck\b/i], + contentPatterns: [/TS\d{4}:/, /error TS\d{4}/i], + }, + { + type: "build-eslint", + category: "build", + commandPatterns: [/^eslint\b/i, /^npm\s+run\s+lint\b/i], + contentPatterns: [/\s+\d+:\d+\s+(?:error|warning)\s+/, /✖\s+\d+\s+problems?/], + }, + { + type: "build-webpack", + category: "build", + commandPatterns: [/^webpack\b/i, /^npx\s+webpack\b/i, /^npm\s+run\s+build:webpack\b/i], + contentPatterns: [ + /webpack\s+\d/i, + /compiled (?:successfully|with \d+ errors?)/i, + /asset .+\.js/i, + ], + }, + { + type: "build-vite", + category: "build", + commandPatterns: [/^vite\s+build\b/i, /^npm\s+run\s+build\b/i, /^pnpm\s+build\b/i], + contentPatterns: [/vite v[\d.]+/i, /✓ built in/i, /transforming \(\d+\)/i], + }, + { + type: "biome", + category: "build", + commandPatterns: [/^biome\b/i, /^npx\s+biome\b/i], + contentPatterns: [/lint\/[A-Za-z0-9/.-]+/, /Checked \d+ files? in/i], + }, + { + type: "prettier", + category: "build", + commandPatterns: [/^prettier\b/i, /^npx\s+prettier\b/i], + contentPatterns: [/^Checking formatting\.\.\./m, /Code style issues found/i], + }, + { + type: "turbo", + category: "build", + commandPatterns: [/^turbo\b/i, /^npx\s+turbo\b/i], + contentPatterns: [/^• Packages in scope:/m, /^Tasks:\s+\d+\s+successful/m], + }, + { + type: "nx", + category: "build", + commandPatterns: [/^nx\b/i, /^npx\s+nx\b/i], + contentPatterns: [/^NX\s+/m, /^> nx run /m], + }, + { + type: "playwright", + category: "test", + commandPatterns: [/^playwright\s+test\b/i, /^npx\s+playwright\s+test\b/i], + contentPatterns: [/Running \d+ tests? using \d+ workers?/i, /^\s+\d+ failed/m], + }, + { + type: "npm-install", + category: "package", + commandPatterns: [/^(?:npm|pnpm|yarn)\s+(?:install|add|update)\b/i], + contentPatterns: [ + /added \d+ packages/i, + /packages are looking for funding/i, + /audited \d+ packages/i, + ], + }, + { + type: "npm-audit", + category: "package", + commandPatterns: [/^(?:npm|pnpm|yarn)\s+audit\b/i], + contentPatterns: [/found \d+ vulnerabilities/i, /\b(?:low|moderate|high|critical)\b/i], + }, + { + type: "ruff", + category: "build", + commandPatterns: [/^ruff\b/i, /^uv\s+run\s+ruff\b/i], + contentPatterns: [/^[\w./-]+\.py:\d+:\d+:\s+[A-Z]\d+/m, /Found \d+ errors?\./i], + }, + { + type: "mypy", + category: "build", + commandPatterns: [/^mypy\b/i, /^python\s+-m\s+mypy\b/i], + contentPatterns: [/^[\w./-]+\.py:\d+:\s+error:/m, /Found \d+ errors? in \d+ files?/i], + }, + { + type: "pip", + category: "package", + commandPatterns: [/^pip\s+(?:install|download|uninstall)\b/i, /^python\s+-m\s+pip\b/i], + contentPatterns: [/^Collecting /m, /^Successfully installed /m], + }, + { + type: "uv-sync", + category: "package", + commandPatterns: [/^uv\s+sync\b/i, /^uv\s+pip\s+install\b/i], + contentPatterns: [/^Resolved \d+ packages?/m, /^Installed \d+ packages?/m], + }, + { + type: "poetry-install", + category: "package", + commandPatterns: [/^poetry\s+install\b/i], + contentPatterns: [/^Installing dependencies from lock file/m, /^Package operations:/m], + }, + { + type: "golangci-lint", + category: "build", + commandPatterns: [/^golangci-lint\b/i], + contentPatterns: [/^[\w./-]+\.go:\d+:\d+:/m, /^\d+ issues?:/m], + }, + { + type: "bundle-install", + category: "package", + commandPatterns: [/^bundle\s+install\b/i], + contentPatterns: [/^Fetching gem metadata from /m, /^Bundle complete!/m], + }, + { + type: "rubocop", + category: "build", + commandPatterns: [/^rubocop\b/i, /^bundle\s+exec\s+rubocop\b/i], + contentPatterns: [/^Inspecting \d+ files/m, /^[\w./-]+\.rb:\d+:\d+:\s+[A-Z]:/m], + }, + { + type: "docker-ps", + category: "docker", + commandPatterns: [/^docker\s+ps\b/i], + contentPatterns: [/^CONTAINER ID\s+IMAGE\s+COMMAND/m], + }, + { + type: "docker-logs", + category: "docker", + commandPatterns: [/^docker\s+logs\b/i, /^docker\s+compose\s+logs\b/i], + contentPatterns: [ + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/m, + /\b(?:ERROR|WARN|INFO)\b/, + /^Attaching to /m, + ], + }, + { + type: "aws", + category: "cloud", + commandPatterns: [/^aws\b/i], + contentPatterns: [/An error occurred \([A-Za-z0-9]+\) when calling/, /^(?:upload|download): /m], + }, + { + type: "gcloud", + category: "cloud", + commandPatterns: [/^gcloud\b/i], + contentPatterns: [/^ERROR: \(gcloud\./m, /^Updated property \[/m], + }, + { + type: "ssh", + category: "cloud", + commandPatterns: [/^ssh\b/i], + contentPatterns: [ + /Permission denied \(/, + /Host key verification failed/, + /Connection timed out/, + ], + }, + { + type: "rsync", + category: "cloud", + commandPatterns: [/^rsync\b/i], + contentPatterns: [/^sending incremental file list/m, /^rsync error:/m], + }, + { + type: "curl", + category: "cloud", + commandPatterns: [/^curl\b/i], + contentPatterns: [/curl: \(\d+\)/, /^HTTP\/\d(?:\.\d)? \d{3}/m], + }, + { + type: "wget", + category: "cloud", + commandPatterns: [/^wget\b/i], + contentPatterns: [/^--\d{4}-\d{2}-\d{2}/m, /^ERROR \d{3}:/m], + }, + { + type: "json-output", + category: "generic", + commandPatterns: [/^jq\b/i, /^cat\s+.*\.json\b/i], + contentPatterns: [/^\s*[\[{][\s\S]*[\]}]\s*$/], + }, + { + type: "shell-ls", + category: "shell", + commandPatterns: [/^ls(?:\s+-[A-Za-z]+)?\b/i], + contentPatterns: [/^total \d+/m, /^\S+\s+\S+\s+\d+\s+\w+\s+\d{1,2}\s+/m], + }, + { + type: "shell-find", + category: "shell", + commandPatterns: [/^find\b/i], + contentPatterns: [/^(?:\.{1,2}|\/|[\w.-]+\/).+/m], + }, + { + type: "shell-grep", + category: "shell", + commandPatterns: [/^(?:grep|rg|ag)\b/i], + contentPatterns: [ + /^[\w./-]+\.(?:ts|tsx|js|jsx|py|go|rs|java|rb|md|json|ya?ml|txt):\d*:/m, + /^[\w./-]+\/[\w./-]+:\d*:/m, + ], + }, + { + type: "shell-ps", + category: "shell", + commandPatterns: [/^ps\b/i], + contentPatterns: [/^(?:USER\s+PID|\s*PID\s+)/m], + }, + { + type: "shell-df", + category: "shell", + commandPatterns: [/^df\b/i], + contentPatterns: [/^Filesystem\s+.*Use%/m], + }, + { + type: "shell-du", + category: "shell", + commandPatterns: [/^du\b/i], + contentPatterns: [/^\d+(?:\.\d+)?[KMGTP]?\s+\S+/m], + }, + { + type: "error-stacktrace", + category: "generic", + commandPatterns: [], + contentPatterns: [ + /Traceback \(most recent call last\):/, + /^\s+at\s+\S+\s+\(.+:\d+:\d+\)/m, + /^panic: /m, + /^thread '[^']+' panicked at/m, + ], + }, + { + type: "generic-error", + category: "generic", + commandPatterns: [], + contentPatterns: [/Error:/, /Exception:/, /Traceback \(most recent call last\):/], + }, +]; + +export function detectCommandFromText(text: string): string | null { + const firstLines = text.split(/\r?\n/).slice(0, 4); + for (const line of firstLines) { + const trimmed = line.trim().replace(/^\$\s+/, ""); + if (!trimmed) continue; + if (COMMAND_PREFIX_PATTERN.test(trimmed)) { + return trimmed; + } + } + return null; +} + +export function detectCommandType(text: string, command?: string | null): CommandDetectionResult { + const detectedCommand = command?.trim() || detectCommandFromText(text); + let best: CommandDetectionResult | null = null; + + for (const detector of DETECTORS) { + const matchedPatterns: string[] = []; + const commandMatched = + Boolean(detectedCommand) && + detector.commandPatterns.some((pattern) => { + const matched = pattern.test(detectedCommand ?? ""); + if (matched) matchedPatterns.push(pattern.source); + return matched; + }); + const contentMatches = detector.contentPatterns.filter((pattern) => { + const matched = pattern.test(text); + if (matched) matchedPatterns.push(pattern.source); + return matched; + }).length; + if (!commandMatched && contentMatches === 0) continue; + + const confidence = Math.min(1, (commandMatched ? 0.55 : 0) + contentMatches * 0.25); + if (!best || confidence > best.confidence) { + best = { + type: detector.type, + command: detectedCommand, + confidence, + category: detector.category, + matchedPatterns, + }; + } + } + + return ( + best ?? { + type: "unknown", + command: detectedCommand, + confidence: detectedCommand ? 0.35 : 0.1, + category: "generic", + matchedPatterns: [], + } + ); +} + +export function listCommandTypes(): string[] { + return DETECTORS.map((detector) => detector.type); +} + +export const detectCommandOutput = detectCommandType; diff --git a/open-sse/services/compression/engines/rtk/deduplicator.ts b/open-sse/services/compression/engines/rtk/deduplicator.ts new file mode 100644 index 0000000000..ea473097fd --- /dev/null +++ b/open-sse/services/compression/engines/rtk/deduplicator.ts @@ -0,0 +1,37 @@ +export interface DeduplicateOptions { + threshold?: number; +} + +export function deduplicateRepeatedLines( + text: string, + options: DeduplicateOptions = {} +): { + text: string; + collapsed: number; +} { + const threshold = Math.max(2, Math.floor(options.threshold ?? 3)); + const lines = text.split(/\r?\n/); + const output: string[] = []; + let collapsed = 0; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + let runLength = 1; + while (index + runLength < lines.length && lines[index + runLength] === line) { + runLength++; + } + + if (line.trim() && runLength >= threshold) { + output.push(line); + output.push(`[line repeated ${runLength - 1}x]`); + output.push(`[rtk:dropped ${runLength - 1} repeated lines]`); + collapsed += runLength - 1; + index += runLength - 1; + continue; + } + + output.push(line); + } + + return { text: output.join("\n"), collapsed }; +} diff --git a/open-sse/services/compression/engines/rtk/filterLoader.ts b/open-sse/services/compression/engines/rtk/filterLoader.ts new file mode 100644 index 0000000000..8c776a9b85 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filterLoader.ts @@ -0,0 +1,220 @@ +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import os from "node:os"; +import crypto from "node:crypto"; +import { detectCommandType } from "./commandDetector.ts"; +import { validateRtkFilter, type RtkFilterDefinition } from "./filterSchema.ts"; + +let cache: RtkFilterDefinition[] | null = null; +let cacheKey: string | null = null; +let diagnostics: RtkFilterLoadDiagnostic[] = []; + +export interface RtkFilterLoadDiagnostic { + source: "project" | "global" | "builtin"; + path?: string; + level: "warning" | "error"; + message: string; +} + +interface FilterSource { + source: "project" | "global" | "builtin"; + path: string; + trusted: boolean; +} + +interface RtkFilterLoadOptions { + refresh?: boolean; + customFiltersEnabled?: boolean; + trustProjectFilters?: boolean; +} + +function getFiltersDir(): string { + const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + const candidates = [ + path.join(moduleDir, "filters"), + path.join(moduleDir, "..", "services", "compression", "engines", "rtk", "filters"), + path.join(process.cwd(), "open-sse", "services", "compression", "engines", "rtk", "filters"), + path.join( + process.cwd(), + "app", + "open-sse", + "services", + "compression", + "engines", + "rtk", + "filters" + ), + ]; + return ( + candidates.find((candidate, index) => { + return candidates.indexOf(candidate) === index && fs.existsSync(candidate); + }) ?? candidates[0] + ); +} + +function getDataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +function sha256(value: string): string { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function projectFiltersTrusted( + filtersPath: string, + trustProjectFilters = false +): boolean | "changed" { + if (trustProjectFilters) return true; + if (process.env.OMNIROUTE_RTK_TRUST_PROJECT_FILTERS === "1") return true; + const trustPath = path.join(path.dirname(filtersPath), "trust.json"); + if (!fs.existsSync(trustPath)) return false; + try { + const filtersHash = sha256(fs.readFileSync(filtersPath, "utf8")); + const trust = JSON.parse(fs.readFileSync(trustPath, "utf8")) as Record; + const trustedHash = + typeof trust.filtersSha256 === "string" + ? trust.filtersSha256 + : typeof trust.trustedFiltersSha256 === "string" + ? trust.trustedFiltersSha256 + : null; + if (!trustedHash) return false; + return trustedHash === filtersHash ? true : "changed"; + } catch { + return false; + } +} + +function collectFilterSources(options: RtkFilterLoadOptions = {}): FilterSource[] { + const sources: FilterSource[] = []; + const projectPath = path.join(process.cwd(), ".rtk", "filters.json"); + if (options.customFiltersEnabled !== false && fs.existsSync(projectPath)) { + const trusted = projectFiltersTrusted(projectPath, options.trustProjectFilters === true); + if (trusted === true) { + sources.push({ source: "project", path: projectPath, trusted: true }); + } else { + diagnostics.push({ + source: "project", + path: projectPath, + level: "warning", + message: + trusted === "changed" + ? "Project RTK filters changed after trust and were skipped" + : "Project RTK filters are untrusted and were skipped", + }); + } + } + + const globalPath = path.join(getDataDir(), "rtk", "filters.json"); + if (options.customFiltersEnabled !== false && fs.existsSync(globalPath)) { + sources.push({ source: "global", path: globalPath, trusted: true }); + } + + const builtinDir = getFiltersDir(); + if (fs.existsSync(builtinDir)) { + for (const file of fs + .readdirSync(builtinDir) + .filter((entry) => entry.endsWith(".json")) + .sort()) { + sources.push({ + source: "builtin", + path: path.join(builtinDir, file), + trusted: true, + }); + } + } + + return sources; +} + +function parseFilterFile(source: FilterSource): RtkFilterDefinition[] { + try { + const parsed = JSON.parse(fs.readFileSync(source.path, "utf8")); + const entries = Array.isArray(parsed) ? parsed : [parsed]; + return entries.map(validateRtkFilter); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (source.source === "builtin") { + throw new Error(`Invalid RTK filter ${path.basename(source.path)}: ${message}`); + } + diagnostics.push({ + source: source.source, + path: source.path, + level: "warning", + message: `Invalid custom RTK filter skipped: ${message}`, + }); + return []; + } +} + +export function loadRtkFilters(options: RtkFilterLoadOptions = {}): RtkFilterDefinition[] { + const currentCacheKey = [ + process.cwd(), + getDataDir(), + options.customFiltersEnabled === false ? "builtin-only" : "custom", + options.trustProjectFilters === true ? "trusted-project" : "trust-file", + process.env.OMNIROUTE_RTK_TRUST_PROJECT_FILTERS === "1" ? "env-trust" : "env-normal", + ].join("|"); + if (cache && cacheKey === currentCacheKey && !options.refresh) return cache; + diagnostics = []; + const filters: RtkFilterDefinition[] = []; + for (const source of collectFilterSources(options)) { + filters.push(...parseFilterFile(source)); + } + + const sorted = filters.sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id)); + cache = sorted; + cacheKey = currentCacheKey; + return sorted; +} + +export function getRtkFilterLoadDiagnostics(): RtkFilterLoadDiagnostic[] { + loadRtkFilters(); + return diagnostics.map((entry) => ({ ...entry })); +} + +export function getRtkFilterCatalog(): Array< + Pick< + RtkFilterDefinition, + "id" | "name" | "description" | "commandTypes" | "category" | "priority" + > +> { + return loadRtkFilters().map((filter) => ({ + id: filter.id, + name: filter.name, + description: filter.description, + commandTypes: filter.commandTypes, + category: filter.category, + priority: filter.priority, + })); +} + +export function matchRtkFilter( + text: string, + command?: string | null, + options: RtkFilterLoadOptions = {} +): RtkFilterDefinition | null { + const detection = detectCommandType(text, command); + const detectedCommand = detection.command ?? command ?? ""; + const matchesPattern = (pattern: string, value: string): boolean => { + try { + return new RegExp(pattern, "im").test(value); + } catch { + return false; + } + }; + const filters = loadRtkFilters(options); + return ( + filters.find((filter) => filter.commandTypes.includes(detection.type)) ?? + filters.find( + (filter) => + detectedCommand && + filter.commandPatterns.some((pattern) => matchesPattern(pattern, detectedCommand)) + ) ?? + filters.find((filter) => + filter.matchPatterns.some((pattern) => matchesPattern(pattern, text)) + ) ?? + filters.find((filter) => filter.commandTypes.includes("generic-output")) ?? + null + ); +} diff --git a/open-sse/services/compression/engines/rtk/filterSchema.ts b/open-sse/services/compression/engines/rtk/filterSchema.ts new file mode 100644 index 0000000000..bb5f67d974 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filterSchema.ts @@ -0,0 +1,182 @@ +import { z } from "zod"; + +const rtkFilterCategorySchema = z.enum([ + "git", + "test", + "build", + "shell", + "docker", + "package", + "infra", + "cloud", + "generic", +]); + +const rtkReplaceRuleSchema = z + .object({ + pattern: z.string().min(1), + replacement: z.string(), + }) + .strict(); + +const rtkMatchOutputRuleSchema = z + .object({ + pattern: z.string().min(1), + message: z.string(), + unless: z.string().min(1).optional(), + }) + .strict(); + +const rtkInlineTestSchema = z + .object({ + name: z.string().min(1), + input: z.string(), + expected: z.string(), + command: z.string().optional(), + }) + .strict(); + +const rtkFilterMatchSchema = z + .object({ + commands: z.array(z.string().min(1)).default([]), + patterns: z.array(z.string().min(1)).default([]), + outputTypes: z.array(z.string().min(1)).default([]), + }) + .strict(); + +const rtkFilterRulesSchema = z + .object({ + stripAnsi: z.boolean().default(false), + replace: z.array(rtkReplaceRuleSchema).default([]), + matchOutput: z.array(rtkMatchOutputRuleSchema).default([]), + includePatterns: z.array(z.string()).default([]), + dropPatterns: z.array(z.string()).default([]), + collapsePatterns: z.array(z.string()).default([]), + deduplicate: z.boolean().default(false), + truncateLineAt: z.number().int().min(0).default(0), + maxLines: z.number().int().min(0).default(0), + headLines: z.number().int().min(0).default(20), + tailLines: z.number().int().min(0).default(20), + onEmpty: z.string().default(""), + filterStderr: z.boolean().default(false), + }) + .strict(); + +const rtkFilterPreserveSchema = z + .object({ + errorPatterns: z.array(z.string()).default([]), + summaryPatterns: z.array(z.string()).default([]), + }) + .strict(); + +export const rtkFilterPackSchema = z + .object({ + id: z.string().min(1), + label: z.string().min(1), + description: z.string().default(""), + category: rtkFilterCategorySchema, + priority: z.number().int().min(0).max(100).default(50), + match: rtkFilterMatchSchema, + rules: rtkFilterRulesSchema.default({}), + preserve: rtkFilterPreserveSchema.default({}), + tests: z.array(rtkInlineTestSchema).default([]), + }) + .strict(); + +const legacyRtkFilterSchema = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + description: z.string().default(""), + commandTypes: z.array(z.string().min(1)).min(1), + category: rtkFilterCategorySchema, + priority: z.number().int().min(0).max(100).default(50), + stripPatterns: z.array(z.string()).default([]), + keepPatterns: z.array(z.string()).default([]), + priorityPatterns: z.array(z.string()).default([]), + collapsePatterns: z.array(z.string()).default([]), + stripAnsi: z.boolean().default(false), + replace: z.array(rtkReplaceRuleSchema).default([]), + matchOutput: z.array(rtkMatchOutputRuleSchema).default([]), + truncateLineAt: z.number().int().min(0).default(0), + onEmpty: z.string().default(""), + filterStderr: z.boolean().default(false), + maxLines: z.number().int().min(0).default(0), + preserveHead: z.number().int().min(0).default(20), + preserveTail: z.number().int().min(0).default(20), + tests: z.array(rtkInlineTestSchema).default([]), + }) + .strict(); + +export const rtkFilterSchema = z.union([rtkFilterPackSchema, legacyRtkFilterSchema]); + +export type RtkFilterPack = z.infer; + +export interface RtkFilterDefinition { + id: string; + name: string; + description: string; + commandTypes: string[]; + commandPatterns: string[]; + matchPatterns: string[]; + category: z.infer; + priority: number; + stripPatterns: string[]; + keepPatterns: string[]; + priorityPatterns: string[]; + collapsePatterns: string[]; + stripAnsi: boolean; + replace: Array<{ pattern: string; replacement: string }>; + matchOutput: Array<{ pattern: string; message: string; unless?: string }>; + truncateLineAt: number; + onEmpty: string; + filterStderr: boolean; + deduplicate: boolean; + maxLines: number; + preserveHead: number; + preserveTail: number; + tests: Array<{ name: string; input: string; expected: string; command?: string }>; +} + +function isCanonicalFilter(value: z.infer): value is RtkFilterPack { + return "label" in value && "match" in value && "rules" in value; +} + +export function validateRtkFilter(value: unknown): RtkFilterDefinition { + const parsed = rtkFilterSchema.parse(value); + if (!isCanonicalFilter(parsed)) { + return { + ...parsed, + commandPatterns: [], + matchPatterns: [], + deduplicate: parsed.collapsePatterns.length > 0, + }; + } + + const preservePatterns = [...parsed.preserve.errorPatterns, ...parsed.preserve.summaryPatterns]; + return { + id: parsed.id, + name: parsed.label, + description: parsed.description, + commandTypes: parsed.match.outputTypes, + commandPatterns: parsed.match.commands, + matchPatterns: parsed.match.patterns, + category: parsed.category, + priority: parsed.priority, + stripPatterns: parsed.rules.dropPatterns, + keepPatterns: parsed.rules.includePatterns, + priorityPatterns: preservePatterns, + collapsePatterns: parsed.rules.collapsePatterns, + stripAnsi: parsed.rules.stripAnsi, + replace: parsed.rules.replace, + matchOutput: parsed.rules.matchOutput, + truncateLineAt: parsed.rules.truncateLineAt, + onEmpty: parsed.rules.onEmpty, + filterStderr: parsed.rules.filterStderr, + deduplicate: parsed.rules.deduplicate, + maxLines: parsed.rules.maxLines, + preserveHead: parsed.rules.headLines, + preserveTail: parsed.rules.tailLines, + tests: parsed.tests, + }; +} diff --git a/open-sse/services/compression/engines/rtk/filters/aws.json b/open-sse/services/compression/engines/rtk/filters/aws.json new file mode 100644 index 0000000000..7a3f82d887 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/aws.json @@ -0,0 +1,34 @@ +{ + "id": "aws", + "label": "AWS CLI", + "description": "Compress AWS CLI progress output while preserving errors and completed transfers.", + "category": "cloud", + "priority": 76, + "match": { + "outputTypes": ["aws"], + "commands": ["^aws\\b"], + "patterns": ["An error occurred \\([A-Za-z0-9]+\\) when calling", "^(?:upload|download): "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Completed \\d", "^\\s*$"], + "includePatterns": ["^(?:upload|download): ", "An error occurred \\("], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["An error occurred", "AccessDenied", "Throttling", "ValidationError"], + "summaryPatterns": ["^(?:upload|download): "] + }, + "tests": [ + { + "name": "inline sample", + "command": "aws s3 cp", + "input": "Completed 1.0 MiB/2.0 MiB\nupload: ./app.log to s3://bucket/app.log\nAn error occurred (AccessDenied) when calling the PutObject operation: denied\n", + "expected": "upload: ./app.log to s3://bucket/app.log\nAn error occurred (AccessDenied) when calling the PutObject operation: denied" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/biome.json b/open-sse/services/compression/engines/rtk/filters/biome.json new file mode 100644 index 0000000000..d0c7564b39 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/biome.json @@ -0,0 +1,35 @@ +{ + "id": "biome", + "label": "Biome", + "description": "Compact Biome lint/format output.", + "category": "build", + "priority": 80, + "match": { + "outputTypes": [], + "commands": ["^biome\\b", "^npx\\s+biome\\b"], + "patterns": ["Checked \\d+ files?", "biome"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^Checked \\d+ files? in"], + "includePatterns": ["error", "warning", "✖", "Checked", "Fixed", "^\\s*>"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40, + "onEmpty": "biome: ok" + }, + "preserve": { + "errorPatterns": ["error", "✖"], + "summaryPatterns": ["Checked", "Fixed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "biome", + "input": "Checked 10 files in 20ms\nerror lint/a.ts:1:1 boom\nFixed 1 file\n", + "expected": "error lint/a.ts:1:1 boom\nFixed 1 file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-eslint.json b/open-sse/services/compression/engines/rtk/filters/build-eslint.json new file mode 100644 index 0000000000..610e616363 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-eslint.json @@ -0,0 +1,37 @@ +{ + "id": "build-eslint", + "label": "ESLint", + "description": "Keep lint diagnostics and problem summary.", + "category": "build", + "priority": 88, + "match": { + "outputTypes": ["build-eslint"], + "commands": ["^(?:eslint|npm\\s+run\\s+lint)\\b"], + "patterns": ["\\s+\\d+:\\d+\\s+(?:error|warning)\\s+", "✖\\s+\\d+\\s+problems?"] + }, + "rules": { + "includePatterns": [ + "\\s+\\d+:\\d+\\s+(error|warning)\\s+", + "✖\\s+\\d+\\s+problems?", + "^/.*\\.(ts|tsx|js|jsx)$" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["error", "warning", "problems?"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:eslint|npm run lint)", + "input": "/repo/a.ts\n 1:1 error boom no-boom\n✖ 1 problem (1 error, 0 warnings)\n", + "expected": "/repo/a.ts\n 1:1 error boom no-boom\n✖ 1 problem (1 error, 0 warnings)" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-typescript.json b/open-sse/services/compression/engines/rtk/filters/build-typescript.json new file mode 100644 index 0000000000..5ef3e314c5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-typescript.json @@ -0,0 +1,33 @@ +{ + "id": "build-typescript", + "label": "TypeScript", + "description": "Keep TypeScript diagnostics and file locations.", + "category": "build", + "priority": 93, + "match": { + "outputTypes": ["build-typescript"], + "commands": ["^(?:tsc|npm\\s+run\\s+typecheck)\\b"], + "patterns": ["TS\\d{4}:", "error TS\\d{4}"] + }, + "rules": { + "includePatterns": ["TS\\d{4}:", "error TS\\d{4}", "^Found \\d+ errors?"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["TS\\d{4}", "Found \\d+"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:tsc|npm run typecheck)", + "input": "src/a.ts:1:1 - error TS2322: Type string is not assignable\nFound 1 error.\n", + "expected": "src/a.ts:1:1 - error TS2322: Type string is not assignable\nFound 1 error." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-vite.json b/open-sse/services/compression/engines/rtk/filters/build-vite.json new file mode 100644 index 0000000000..7e81845b81 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-vite.json @@ -0,0 +1,41 @@ +{ + "id": "build-vite", + "label": "Vite Build", + "description": "Keep Vite errors and final build summary.", + "category": "build", + "priority": 83, + "match": { + "outputTypes": ["build-vite"], + "commands": ["^vite\\s+build\\b", "^npm\\s+run\\s+build\\b", "^pnpm\\s+build\\b"], + "patterns": ["vite v[\\d.]+", "✓ built in"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "✓ built in", + "message": "vite: build ok", + "unless": "error|failed" + } + ], + "includePatterns": ["error", "failed", "✓ built in", "dist/", "rendering chunks"], + "dropPatterns": ["^transforming ", "^✓ \\d+ modules transformed", "^\\s*$"], + "collapsePatterns": ["^dist/"], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["✓ built in"] + }, + "tests": [ + { + "name": "inline sample", + "command": "vite build", + "input": "vite v5.0.0 building for production...\n✓ 10 modules transformed.\n✓ built in 100ms\n", + "expected": "vite: build ok" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-webpack.json b/open-sse/services/compression/engines/rtk/filters/build-webpack.json new file mode 100644 index 0000000000..55e1eb845b --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-webpack.json @@ -0,0 +1,34 @@ +{ + "id": "build-webpack", + "label": "Webpack Build", + "description": "Keep webpack errors/warnings and compilation summary.", + "category": "build", + "priority": 84, + "match": { + "outputTypes": ["build-webpack"], + "commands": ["^webpack\\b", "^npx\\s+webpack\\b"], + "patterns": ["webpack \\d", "compiled (?:successfully|with)"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": ["ERROR in", "WARNING in", "compiled ", "asset ", "Entrypoint"], + "dropPatterns": ["^\\s*$", "^cacheable modules"], + "collapsePatterns": ["^asset "], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["ERROR in"], + "summaryPatterns": ["compiled "] + }, + "tests": [ + { + "name": "inline sample", + "command": "webpack", + "input": "asset main.js 10 KiB\nERROR in ./src/a.ts\ncompiled with 1 error\n", + "expected": "asset main.js 10 KiB\nERROR in ./src/a.ts\ncompiled with 1 error" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/bundle-install.json b/open-sse/services/compression/engines/rtk/filters/bundle-install.json new file mode 100644 index 0000000000..7b126ae117 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/bundle-install.json @@ -0,0 +1,35 @@ +{ + "id": "bundle-install", + "label": "Bundle Install", + "description": "Strip bundle install noise and keep final status.", + "category": "package", + "priority": 70, + "match": { + "outputTypes": [], + "commands": ["^bundle\\s+install\\b"], + "patterns": ["Bundle complete", "Using "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Using ", "^Fetching ", "^Installing ", "^\\s*$"], + "includePatterns": ["Bundle complete", "Your bundle is complete", "ERROR", "Gem::"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "bundle: ok" + }, + "preserve": { + "errorPatterns": ["ERROR", "Gem::"], + "summaryPatterns": ["Bundle complete", "Your bundle is complete"] + }, + "tests": [ + { + "name": "inline sample", + "command": "bundle install", + "input": "Using rake 13.0\nInstalling pg 1.5\nBundle complete! 4 Gemfile dependencies\n", + "expected": "Bundle complete! 4 Gemfile dependencies" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/curl.json b/open-sse/services/compression/engines/rtk/filters/curl.json new file mode 100644 index 0000000000..936f290706 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/curl.json @@ -0,0 +1,34 @@ +{ + "id": "curl", + "label": "curl", + "description": "Remove curl progress noise and keep HTTP status/errors.", + "category": "cloud", + "priority": 70, + "match": { + "outputTypes": ["curl"], + "commands": ["^curl\\b"], + "patterns": ["curl: \\(\\d+\\)", "^HTTP/\\d(?:\\.\\d)? \\d{3}"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*% Total\\s+% Received", "^\\s*\\d+\\s+\\d+", "^\\s*$"], + "includePatterns": ["^HTTP/\\d(?:\\.\\d)? \\d{3}", "^curl: \\(\\d+\\)", "^[<{]"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["curl: \\(\\d+\\)", "^HTTP/\\d(?:\\.\\d)? [45]\\d\\d"], + "summaryPatterns": ["^HTTP/\\d(?:\\.\\d)? \\d{3}"] + }, + "tests": [ + { + "name": "inline sample", + "command": "curl -i", + "input": "% Total % Received\n 0 0 0 0\nHTTP/1.1 500 Internal Server Error\ncurl: (22) The requested URL returned error: 500\n", + "expected": "HTTP/1.1 500 Internal Server Error\ncurl: (22) The requested URL returned error: 500" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/df.json b/open-sse/services/compression/engines/rtk/filters/df.json new file mode 100644 index 0000000000..5be798a273 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/df.json @@ -0,0 +1,34 @@ +{ + "id": "df", + "label": "df", + "description": "Keep filesystem usage table headers and mounted filesystems.", + "category": "shell", + "priority": 67, + "match": { + "outputTypes": ["shell-df"], + "commands": ["^df\\b"], + "patterns": ["^Filesystem\\s+.*Use%"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^tmpfs\\s", "^udev\\s", "^\\s*$"], + "includePatterns": ["^Filesystem\\s+", "^/dev/\\S+\\s+"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 25, + "tailLines": 25 + }, + "preserve": { + "errorPatterns": ["No such file", "Permission denied"], + "summaryPatterns": ["Use%", "^/dev/"] + }, + "tests": [ + { + "name": "inline sample", + "command": "df -h", + "input": "Filesystem Size Used Avail Use% Mounted on\nudev 2.0G 0 2.0G 0% /dev\n/dev/sda1 50G 45G 5.0G 90% /\n", + "expected": "Filesystem Size Used Avail Use% Mounted on\n/dev/sda1 50G 45G 5.0G 90% /" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/docker-logs.json b/open-sse/services/compression/engines/rtk/filters/docker-logs.json new file mode 100644 index 0000000000..10b5d11f43 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/docker-logs.json @@ -0,0 +1,42 @@ +{ + "id": "docker-logs", + "label": "Docker Logs", + "description": "Keep warnings/errors and recent Docker log context.", + "category": "docker", + "priority": 84, + "match": { + "outputTypes": ["docker-logs"], + "commands": ["^docker\\s+(?:compose\\s+)?logs\\b"], + "patterns": ["\\b(?:ERROR|WARN|INFO)\\b", "^Attaching to "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Attaching to ", "^\\s*$"], + "includePatterns": [ + "ERROR", + "WARN", + "Exception", + "Traceback", + "failed", + "listening", + "started" + ], + "collapsePatterns": ["INFO"], + "deduplicate": true, + "maxLines": 120, + "headLines": 20, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["ERROR", "Exception", "failed"], + "summaryPatterns": ["started", "listening"] + }, + "tests": [ + { + "name": "inline sample", + "command": "docker (?:compose )?logs", + "input": "Attaching to api\nINFO started\nINFO started\nERROR failed to connect\n", + "expected": "INFO started\nERROR failed to connect" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/docker-ps.json b/open-sse/services/compression/engines/rtk/filters/docker-ps.json new file mode 100644 index 0000000000..b7dc4efee0 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/docker-ps.json @@ -0,0 +1,33 @@ +{ + "id": "docker-ps", + "label": "Docker PS", + "description": "Keep docker container table header and rows under a fixed cap.", + "category": "docker", + "priority": 60, + "match": { + "outputTypes": ["docker-ps"], + "commands": ["^docker\\s+ps\\b"], + "patterns": ["^CONTAINER ID\\s+IMAGE\\s+COMMAND"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 30, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^CONTAINER ID"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "docker ps", + "input": "CONTAINER ID IMAGE COMMAND\nabc123 node \"npm start\"\n", + "expected": "CONTAINER ID IMAGE COMMAND\nabc123 node \"npm start\"" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/du.json b/open-sse/services/compression/engines/rtk/filters/du.json new file mode 100644 index 0000000000..54cca64e89 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/du.json @@ -0,0 +1,33 @@ +{ + "id": "du", + "label": "du", + "description": "Summarize disk-usage listings while preserving largest entries.", + "category": "shell", + "priority": 66, + "match": { + "outputTypes": ["shell-du"], + "commands": ["^du\\b"], + "patterns": ["^\\d+(?:\\.\\d+)?[KMGTP]?\\s+\\S+"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["Permission denied", "No such file"], + "summaryPatterns": ["\\s+\\.$", "\\s+total$"] + }, + "tests": [ + { + "name": "inline sample", + "command": "du -sh *", + "input": "4.0K package.json\n120M node_modules\n120M node_modules\n", + "expected": "4.0K package.json\n120M node_modules\n120M node_modules" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json b/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json new file mode 100644 index 0000000000..0d5fd3b8bf --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json @@ -0,0 +1,41 @@ +{ + "id": "error-stacktrace", + "label": "Error Stacktrace", + "description": "Collapse stack traces while preserving error text and top frames.", + "category": "generic", + "priority": 70, + "match": { + "outputTypes": ["error-stacktrace"], + "commands": [], + "patterns": ["Traceback \\(most recent call last\\):", "^\\s+at\\s+", "^panic: "] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "Error:", + "Exception:", + "Traceback", + "^\\s+at ", + "^panic:", + "^thread ", + "^Caused by:" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 80, + "headLines": 25, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["Error:", "Exception:", "^panic:", "Traceback"], + "summaryPatterns": ["Caused by:"] + }, + "tests": [ + { + "name": "inline sample", + "input": "Error: boom\n at fn (/repo/a.ts:1:1)\n at fn (/repo/a.ts:1:1)\nCaused by: bad\n", + "expected": "Error: boom\n at fn (/repo/a.ts:1:1)\nCaused by: bad" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/gcloud.json b/open-sse/services/compression/engines/rtk/filters/gcloud.json new file mode 100644 index 0000000000..19c5dd09a9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/gcloud.json @@ -0,0 +1,34 @@ +{ + "id": "gcloud", + "label": "gcloud", + "description": "Compress Google Cloud CLI output and preserve actionable errors.", + "category": "cloud", + "priority": 75, + "match": { + "outputTypes": ["gcloud"], + "commands": ["^gcloud\\b"], + "patterns": ["^ERROR: \\(gcloud\\.", "^Updated property \\["] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Waiting for operation", "^\\s*$"], + "includePatterns": ["^ERROR: \\(gcloud\\.", "^Updated property \\[", "^Created \\["], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^ERROR: \\(gcloud\\.", "PERMISSION_DENIED", "NOT_FOUND"], + "summaryPatterns": ["^Updated property \\[", "^Created \\["] + }, + "tests": [ + { + "name": "inline sample", + "command": "gcloud run deploy", + "input": "Waiting for operation [deploy]\nUpdated property [core/project].\nERROR: (gcloud.run.deploy) PERMISSION_DENIED: denied\n", + "expected": "Updated property [core/project].\nERROR: (gcloud.run.deploy) PERMISSION_DENIED: denied" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/generic-output.json b/open-sse/services/compression/engines/rtk/filters/generic-output.json new file mode 100644 index 0000000000..da93022888 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/generic-output.json @@ -0,0 +1,32 @@ +{ + "id": "generic-output", + "label": "Generic Output", + "description": "Fallback filter for unknown command output.", + "category": "generic", + "priority": 10, + "match": { + "outputTypes": ["generic-output", "generic-error"], + "commands": [], + "patterns": ["Error:", "Exception:", "Traceback \\(most recent call last\\):"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 35, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["error", "failed", "exception", "traceback"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "input": "\nError: boom\n\nTraceback (most recent call last):\n file.py:1\n", + "expected": "Error: boom\nTraceback (most recent call last):\n file.py:1" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-branch.json b/open-sse/services/compression/engines/rtk/filters/git-branch.json new file mode 100644 index 0000000000..b333932f08 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-branch.json @@ -0,0 +1,45 @@ +{ + "id": "git-branch", + "label": "Git Branch", + "description": "Compact git branch/switch output.", + "category": "git", + "priority": 94, + "match": { + "outputTypes": ["git-branch"], + "commands": ["^git\\s+(?:branch|checkout|switch)\\b"], + "patterns": ["^\\*\\s+\\S+", "Switched to (?:a new )?branch", "Already on"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "Switched to (?:a new )?branch", + "message": "git: switched branch", + "unless": "error|failed" + }, + { + "pattern": "Already on", + "message": "git: already on branch" + } + ], + "includePatterns": ["^\\*\\s+\\S+", "^\\s{2}\\S+", "Switched to", "Already on"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 60, + "headLines": 20, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["Switched to", "Already on"] + }, + "tests": [ + { + "name": "inline sample", + "command": "git (?:branch|checkout|switch)", + "input": " main\n* feature/rtk\n release\n", + "expected": " main\n* feature/rtk\n release" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-diff.json b/open-sse/services/compression/engines/rtk/filters/git-diff.json new file mode 100644 index 0000000000..a97df1fbac --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-diff.json @@ -0,0 +1,33 @@ +{ + "id": "git-diff", + "label": "Git Diff", + "description": "Drop low-signal diff metadata while preserving files, hunks, and changed lines.", + "category": "git", + "priority": 94, + "match": { + "outputTypes": ["git-diff"], + "commands": ["^git\\s+(?:diff|show)\\b"], + "patterns": ["^diff --git ", "^@@\\s+-\\d+,\\d+\\s+\\+\\d+,\\d+\\s+@@"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^index [0-9a-f]+\\.\\.[0-9a-f]+.*$", "^--- a/", "^\\+\\+\\+ b/"], + "collapsePatterns": ["^\\s*$"], + "deduplicate": true, + "maxLines": 180, + "headLines": 40, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["^diff --git ", "^@@ ", "^[+-](?![+-]{2})"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git (?:diff|show)", + "input": "diff --git a/a.ts b/a.ts\n@@ -1,2 +1,2 @@\n-console.log(\"old\")\n+console.log(\"new\")\n", + "expected": "diff --git a/a.ts b/a.ts\n@@ -1,2 +1,2 @@\n-console.log(\"old\")\n+console.log(\"new\")\n" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-log.json b/open-sse/services/compression/engines/rtk/filters/git-log.json new file mode 100644 index 0000000000..4a54160537 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-log.json @@ -0,0 +1,33 @@ +{ + "id": "git-log", + "label": "Git Log", + "description": "Keep commit identifiers and subjects from verbose logs.", + "category": "git", + "priority": 70, + "match": { + "outputTypes": ["git-log"], + "commands": ["^git\\s+log\\b"], + "patterns": ["^commit [0-9a-f]{7,40}", "^Author: "] + }, + "rules": { + "includePatterns": ["^commit [0-9a-f]{7,40}", "^\\s{4}\\S"], + "dropPatterns": ["^Author: ", "^Date: ", "^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 40, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^commit "], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git log", + "input": "commit abcdef1234567890\nAuthor: Dev \nDate: today\n\n feat: add rtk\n", + "expected": "commit abcdef1234567890\n feat: add rtk" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-status.json b/open-sse/services/compression/engines/rtk/filters/git-status.json new file mode 100644 index 0000000000..d2d21a66f5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-status.json @@ -0,0 +1,40 @@ +{ + "id": "git-status", + "label": "Git Status", + "description": "Keep branch, staged/unstaged headings, and changed file lines.", + "category": "git", + "priority": 95, + "match": { + "outputTypes": ["git-status"], + "commands": ["^git\\s+status\\b"], + "patterns": ["^On branch ", "^Changes (?:not staged|to be committed)", "^Untracked files:"] + }, + "rules": { + "includePatterns": [ + "^On branch ", + "^Your branch ", + "^Changes ", + "^Untracked files:", + "^\\s*(modified|new file|deleted|renamed|both modified):", + "^\\s*[MADRCU?!]{1,2}\\s+" + ], + "dropPatterns": ["^\\s*$", "^\\s*\\(use .*\\)$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 12, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^Changes ", "^Untracked files:"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git status", + "input": "On branch main\nChanges not staged for commit:\n (use \"git add\" to update)\n\tmodified: src/app.ts\nnothing added to commit\n", + "expected": "On branch main\nChanges not staged for commit:\n\tmodified: src/app.ts" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/golangci-lint.json b/open-sse/services/compression/engines/rtk/filters/golangci-lint.json new file mode 100644 index 0000000000..8bffbee1af --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/golangci-lint.json @@ -0,0 +1,34 @@ +{ + "id": "golangci-lint", + "label": "golangci-lint", + "description": "Keep golangci-lint findings and summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^golangci-lint\\b"], + "patterns": ["issues:", "level=error", ":\\d+:\\d+:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [":\\d+:\\d+:", "level=error", "issues:", "Error:"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["level=error", "Error:"], + "summaryPatterns": ["issues:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "golangci-lint", + "input": "main.go:1:1: issue text\nlevel=error msg=\"runner failed\"\nissues: 1\n", + "expected": "main.go:1:1: issue text\nlevel=error msg=\"runner failed\"\nissues: 1" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/json-output.json b/open-sse/services/compression/engines/rtk/filters/json-output.json new file mode 100644 index 0000000000..1ab8769266 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/json-output.json @@ -0,0 +1,40 @@ +{ + "id": "json-output", + "label": "JSON Output", + "description": "Keep actionable JSON status, error and message fields while preserving structure.", + "category": "generic", + "priority": 70, + "match": { + "outputTypes": ["json-output"], + "commands": ["^(?:jq\\b|cat\\s+.*\\.json\\b)"], + "patterns": ["^\\s*[\\[{][\\s\\S]*[\\]}]\\s*$"] + }, + "rules": { + "includePatterns": [ + "\"status\"\\s*:", + "\"error\"\\s*:", + "\"errors\"\\s*:", + "\"code\"\\s*:", + "\"message\"\\s*:", + "[{}\\[\\],]" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["\"error\"", "\"errors\"", "\"message\""], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:jq\\b|cat .*\\.json\\b)", + "input": "{\n \"status\": \"ok\",\n \"items\": [1,2,3]\n}\n", + "expected": "{\n \"status\": \"ok\",\n \"items\": [1,2,3]\n}" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/make.json b/open-sse/services/compression/engines/rtk/filters/make.json new file mode 100644 index 0000000000..7b9d3a6b24 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/make.json @@ -0,0 +1,38 @@ +{ + "id": "make", + "label": "Make", + "description": "Strip make directory chatter and empty lines.", + "category": "build", + "priority": 78, + "match": { + "outputTypes": [], + "commands": ["^make\\b"], + "patterns": ["make\\[\\d+\\]: (?:Entering|Leaving) directory"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": [ + "^make\\[\\d+\\]: (?:Entering|Leaving) directory", + "^\\s*$", + "Nothing to be done for" + ], + "collapsePatterns": ["^(?:cc|gcc|clang|g\\+\\+)\\b"], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40, + "onEmpty": "make: ok" + }, + "preserve": { + "errorPatterns": ["error:", "failed", "\\*\\*\\*"], + "summaryPatterns": ["built", "linking"] + }, + "tests": [ + { + "name": "inline sample", + "command": "make", + "input": "make[1]: Entering directory '/repo'\ngcc -O2 foo.c\nmake[1]: Leaving directory '/repo'\n", + "expected": "gcc -O2 foo.c" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/mypy.json b/open-sse/services/compression/engines/rtk/filters/mypy.json new file mode 100644 index 0000000000..861bbe3e9b --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/mypy.json @@ -0,0 +1,34 @@ +{ + "id": "mypy", + "label": "mypy", + "description": "Keep mypy errors grouped by file and final summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^mypy\\b", "^python\\s+-m\\s+mypy\\b"], + "patterns": [": error:", "Found \\d+ errors? in"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [": error:", ": note:", "Found \\d+ errors?", "Success: no issues found"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 160, + "headLines": 40, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": [": error:", "Found \\d+ errors?"], + "summaryPatterns": ["Success:", "Found \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "mypy", + "input": "app.py:1: error: Incompatible types\nFound 1 error in 1 file\n", + "expected": "app.py:1: error: Incompatible types\nFound 1 error in 1 file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/npm-audit.json b/open-sse/services/compression/engines/rtk/filters/npm-audit.json new file mode 100644 index 0000000000..19fa224a38 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/npm-audit.json @@ -0,0 +1,43 @@ +{ + "id": "npm-audit", + "label": "npm audit", + "description": "Keep vulnerability counts and remediation hints.", + "category": "package", + "priority": 84, + "match": { + "outputTypes": ["npm-audit"], + "commands": ["^(?:npm|pnpm|yarn)\\s+audit\\b"], + "patterns": ["found \\d+ vulnerabilities", "\\bcritical\\b"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "vulnerabilities", + "critical", + "high", + "moderate", + "low", + "fix available", + "npm audit fix", + "No vulnerabilities" + ], + "dropPatterns": ["^\\s*$", "^# npm audit report"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["critical", "high"], + "summaryPatterns": ["vulnerabilities", "No vulnerabilities"] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:npm|pnpm|yarn) audit", + "input": "# npm audit report\ncritical vulnerability in foo\nfound 1 vulnerabilities\nnpm audit fix\n", + "expected": "critical vulnerability in foo\nfound 1 vulnerabilities\nnpm audit fix" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/npm-install.json b/open-sse/services/compression/engines/rtk/filters/npm-install.json new file mode 100644 index 0000000000..df001e67b2 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/npm-install.json @@ -0,0 +1,41 @@ +{ + "id": "npm-install", + "label": "Package Install", + "description": "Keep install summaries, warnings and audit results.", + "category": "package", + "priority": 65, + "match": { + "outputTypes": ["npm-install"], + "commands": ["^(?:npm|pnpm|yarn)\\s+(?:install|add|update)\\b"], + "patterns": ["added \\d+ packages", "packages are looking for funding", "audited \\d+ packages"] + }, + "rules": { + "includePatterns": [ + "added \\d+ packages", + "removed \\d+ packages", + "changed \\d+ packages", + "audited \\d+ packages", + "vulnerabilities", + "WARN", + "ERR!" + ], + "dropPatterns": ["^\\s*$", "^\\s*\\|", "^Progress:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["WARN", "ERR", "vulnerabilities"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:npm|pnpm|yarn) (?:install|add|update)", + "input": "added 3 packages\npackages are looking for funding\naudited 3 packages\n", + "expected": "added 3 packages\naudited 3 packages" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/nx.json b/open-sse/services/compression/engines/rtk/filters/nx.json new file mode 100644 index 0000000000..6973433228 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/nx.json @@ -0,0 +1,34 @@ +{ + "id": "nx", + "label": "Nx", + "description": "Compact Nx task graph and output.", + "category": "build", + "priority": 76, + "match": { + "outputTypes": [], + "commands": ["^nx\\b", "^npx\\s+nx\\b"], + "patterns": ["NX ", "Successfully ran target"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^> nx run"], + "includePatterns": ["NX ", "ERROR", "Failed", "Successfully ran", "Running target"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 25, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["ERROR", "Failed"], + "summaryPatterns": ["Successfully ran"] + }, + "tests": [ + { + "name": "inline sample", + "command": "nx", + "input": "> nx run app:build\nNX Failed tasks:\nERROR boom\n", + "expected": "NX Failed tasks:\nERROR boom" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/pip.json b/open-sse/services/compression/engines/rtk/filters/pip.json new file mode 100644 index 0000000000..91edf38ca3 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/pip.json @@ -0,0 +1,41 @@ +{ + "id": "pip", + "label": "pip", + "description": "Strip pip install progress and keep install/failure summary.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^pip\\s+(?:install|sync|download)\\b", "^python\\s+-m\\s+pip\\s+install\\b"], + "patterns": ["Successfully installed", "Requirement already satisfied"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": [ + "^Collecting ", + "^Downloading ", + "^Installing collected packages:", + "^Requirement already satisfied", + "^\\s*$" + ], + "includePatterns": ["Successfully installed", "ERROR:", "WARNING:", "failed", "Could not find"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "pip: ok" + }, + "preserve": { + "errorPatterns": ["ERROR:", "failed", "Could not find"], + "summaryPatterns": ["Successfully installed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "pip (?:install|sync|download)", + "input": "Collecting requests\nDownloading requests.whl\nSuccessfully installed requests-2.0\n", + "expected": "Successfully installed requests-2.0" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/playwright.json b/open-sse/services/compression/engines/rtk/filters/playwright.json new file mode 100644 index 0000000000..1899b6367c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/playwright.json @@ -0,0 +1,42 @@ +{ + "id": "playwright", + "label": "Playwright", + "description": "Keep failed specs, error context, and summary.", + "category": "test", + "priority": 86, + "match": { + "outputTypes": [], + "commands": ["^playwright\\s+test\\b", "^npx\\s+playwright\\s+test\\b"], + "patterns": ["Running \\d+ tests?", "Error Context:", "failed"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^Running \\d+ tests? using"], + "includePatterns": [ + "✘", + "Error:", + "Error Context:", + "failed", + "passed", + "Slow test file", + "^\\s*\\d+\\)" + ], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 160, + "headLines": 35, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["Error:", "✘", "failed"], + "summaryPatterns": ["passed", "failed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "playwright test", + "input": "Running 2 tests using 1 worker\n ✘ 1 login.spec.ts:3:1 › login\nError: expected visible\n 1 failed\n", + "expected": " ✘ 1 login.spec.ts:3:1 › login\nError: expected visible\n 1 failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/poetry-install.json b/open-sse/services/compression/engines/rtk/filters/poetry-install.json new file mode 100644 index 0000000000..f40169365e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/poetry-install.json @@ -0,0 +1,35 @@ +{ + "id": "poetry-install", + "label": "Poetry Install", + "description": "Strip Poetry package operation noise.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^poetry\\s+(?:install|update)\\b"], + "patterns": ["Package operations:", "Installing dependencies"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^ - Installing ", "^ - Updating ", "^Installing dependencies", "^\\s*$"], + "includePatterns": ["Package operations:", "Writing lock file", "Error", "failed"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "poetry: ok" + }, + "preserve": { + "errorPatterns": ["Error", "failed"], + "summaryPatterns": ["Package operations:", "Writing lock file"] + }, + "tests": [ + { + "name": "inline sample", + "command": "poetry (?:install|update)", + "input": "Installing dependencies\nPackage operations: 2 installs, 0 updates\n - Installing foo (1.0)\nWriting lock file\n", + "expected": "Package operations: 2 installs, 0 updates\nWriting lock file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/prettier.json b/open-sse/services/compression/engines/rtk/filters/prettier.json new file mode 100644 index 0000000000..0a1c894fc6 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/prettier.json @@ -0,0 +1,40 @@ +{ + "id": "prettier", + "label": "Prettier", + "description": "Keep files that would be changed and final summary.", + "category": "build", + "priority": 79, + "match": { + "outputTypes": [], + "commands": ["^prettier\\b", "^npx\\s+prettier\\b"], + "patterns": ["\\[warn\\]", "All matched files use Prettier"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "All matched files use Prettier", + "message": "prettier: ok" + } + ], + "dropPatterns": ["^\\s*$"], + "includePatterns": ["\\[warn\\]", "Code style issues", "All matched files"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["\\[error\\]", "Code style issues"], + "summaryPatterns": ["All matched files"] + }, + "tests": [ + { + "name": "inline sample", + "command": "prettier", + "input": "[warn] src/a.ts\n[warn] Code style issues found in the above file. Forgot to run Prettier?\n", + "expected": "[warn] src/a.ts\n[warn] Code style issues found in the above file. Forgot to run Prettier?" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ps.json b/open-sse/services/compression/engines/rtk/filters/ps.json new file mode 100644 index 0000000000..843866d70e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ps.json @@ -0,0 +1,34 @@ +{ + "id": "ps", + "label": "ps", + "description": "Keep process table headers and process rows, dropping empty noise.", + "category": "shell", + "priority": 68, + "match": { + "outputTypes": ["shell-ps"], + "commands": ["^ps\\b"], + "patterns": ["^(?:USER\\s+PID|\\s*PID\\s+)"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^(?:USER\\s+PID|\\s*PID\\s+)", "^\\S+\\s+\\d+\\s+", "^\\s*\\d+\\s+"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 30, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["defunct", "Permission denied"], + "summaryPatterns": ["^(?:USER\\s+PID|\\s*PID\\s+)"] + }, + "tests": [ + { + "name": "inline sample", + "command": "ps aux", + "input": "USER PID %CPU COMMAND\nroot 1 0.0 init\nnode 1234 5.0 node server.js\n", + "expected": "USER PID %CPU COMMAND\nroot 1 0.0 init\nnode 1234 5.0 node server.js" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/rsync.json b/open-sse/services/compression/engines/rtk/filters/rsync.json new file mode 100644 index 0000000000..36ecd135bc --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/rsync.json @@ -0,0 +1,33 @@ +{ + "id": "rsync", + "label": "rsync", + "description": "Drop rsync transfer chatter while preserving copied files and errors.", + "category": "cloud", + "priority": 73, + "match": { + "outputTypes": ["rsync"], + "commands": ["^rsync\\b"], + "patterns": ["^sending incremental file list", "^rsync error:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^sending incremental file list", "^sent \\d", "^total size is", "^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["^rsync error:", "Permission denied", "No such file"], + "summaryPatterns": ["^sent \\d", "^total size is"] + }, + "tests": [ + { + "name": "inline sample", + "command": "rsync -av", + "input": "sending incremental file list\napp.js\nsent 120 bytes received 20 bytes\nrsync error: some files could not be transferred (code 23)\n", + "expected": "app.js\nrsync error: some files could not be transferred (code 23)" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/rubocop.json b/open-sse/services/compression/engines/rtk/filters/rubocop.json new file mode 100644 index 0000000000..da92cc4beb --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/rubocop.json @@ -0,0 +1,34 @@ +{ + "id": "rubocop", + "label": "RuboCop", + "description": "Keep RuboCop offenses and final summary.", + "category": "build", + "priority": 74, + "match": { + "outputTypes": ["rubocop"], + "commands": ["^rubocop\\b", "^bundle\\s+exec\\s+rubocop\\b"], + "patterns": ["^Inspecting \\d+ files", "^\\S+\\.rb:\\d+:\\d+: [A-Z]:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Inspecting \\d+ files", "^\\.+$", "^\\s*$"], + "includePatterns": ["^\\S+\\.rb:\\d+:\\d+: [A-Z]:", "^\\d+ files inspected"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": [" [EF]: ", "Offenses:"], + "summaryPatterns": ["^\\d+ files inspected"] + }, + "tests": [ + { + "name": "inline sample", + "command": "rubocop", + "input": "Inspecting 2 files\n.C\napp/models/user.rb:12:3: C: Style/IfUnlessModifier: Favor modifier if usage.\n2 files inspected, 1 offense detected\n", + "expected": "app/models/user.rb:12:3: C: Style/IfUnlessModifier: Favor modifier if usage.\n2 files inspected, 1 offense detected" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ruff.json b/open-sse/services/compression/engines/rtk/filters/ruff.json new file mode 100644 index 0000000000..0e3d80d6ee --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ruff.json @@ -0,0 +1,40 @@ +{ + "id": "ruff", + "label": "Ruff", + "description": "Keep Ruff diagnostics and summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^ruff\\b", "^uv\\s+run\\s+ruff\\b"], + "patterns": ["Found \\d+ errors?", "All checks passed"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "All checks passed", + "message": "ruff: ok" + } + ], + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^[A-Z]\\d{3}", "Found \\d+", "error:", "warning:", "-->"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^[A-Z]\\d{3}", "error:"], + "summaryPatterns": ["Found \\d+", "All checks passed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "ruff", + "input": "F401 app.py:1:1 unused import\nFound 1 error.\n", + "expected": "F401 app.py:1:1 unused import\nFound 1 error." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-find.json b/open-sse/services/compression/engines/rtk/filters/shell-find.json new file mode 100644 index 0000000000..e95ce818d9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-find.json @@ -0,0 +1,33 @@ +{ + "id": "shell-find", + "label": "find", + "description": "Limit large find output while preserving head and tail.", + "category": "shell", + "priority": 80, + "match": { + "outputTypes": ["shell-find"], + "commands": ["^find\\b"], + "patterns": ["^(?:\\./|/|[A-Za-z0-9_.-]+/)"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 40, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["Permission denied", "No such file"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "find", + "input": "./src/a.ts\n./src/b.ts\n./node_modules/pkg/index.js\n", + "expected": "./src/a.ts\n./src/b.ts\n./node_modules/pkg/index.js" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-grep.json b/open-sse/services/compression/engines/rtk/filters/shell-grep.json new file mode 100644 index 0000000000..6429ff82f9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-grep.json @@ -0,0 +1,37 @@ +{ + "id": "shell-grep", + "label": "grep/rg", + "description": "Deduplicate grep output while preserving matches.", + "category": "shell", + "priority": 82, + "match": { + "outputTypes": ["shell-grep"], + "commands": ["^(?:grep|rg|ag)\\b"], + "patterns": [ + "^[\\w./-]+\\.(?:ts|tsx|js|jsx|py|go|rs|java|rb|md|json|ya?ml|txt):\\d*:", + "^[\\w./-]+/[\\w./-]+:\\d*:" + ] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^[\\w./-]+(?::\\d+)?:"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 160, + "headLines": 40, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["No such file", "Permission denied"], + "summaryPatterns": ["^[^:\\n]+(?::\\d+)?:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:grep|rg|ag)", + "input": "src/a.ts:1:match\nsrc/a.ts:1:match\nsrc/b.ts:2:other\n", + "expected": "src/a.ts:1:match\nsrc/a.ts:1:match\nsrc/b.ts:2:other" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-ls.json b/open-sse/services/compression/engines/rtk/filters/shell-ls.json new file mode 100644 index 0000000000..d8bb51cf8e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-ls.json @@ -0,0 +1,33 @@ +{ + "id": "shell-ls", + "label": "Shell Listing", + "description": "Keep directory listing boundaries without flooding the prompt.", + "category": "shell", + "priority": 50, + "match": { + "outputTypes": ["shell-ls"], + "commands": ["^(?:ls(?:\\s+-[A-Za-z]+)?|find)\\b"], + "patterns": ["^total \\d+", "^\\S+\\s+\\S+\\s+\\d+\\s+\\w+\\s+\\d{1,2}\\s+"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 100, + "headLines": 40, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["^total \\d+"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:ls(?: -[A-Za-z]+)?|find)", + "input": "total 8\ndrwxr-xr-x 2 user user 4096 May 1 src\n-rw-r--r-- 1 user user 10 May 1 package.json\n", + "expected": "total 8\ndrwxr-xr-x 2 user user 4096 May 1 src\n-rw-r--r-- 1 user user 10 May 1 package.json" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ssh.json b/open-sse/services/compression/engines/rtk/filters/ssh.json new file mode 100644 index 0000000000..2b67759776 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ssh.json @@ -0,0 +1,43 @@ +{ + "id": "ssh", + "label": "ssh", + "description": "Compress SSH connection logs while preserving access and host-key failures.", + "category": "cloud", + "priority": 72, + "match": { + "outputTypes": ["ssh"], + "commands": ["^ssh\\b"], + "patterns": [ + "Permission denied \\(", + "Host key verification failed", + "Connection (?:closed|timed out)" + ] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^debug\\d:", "^\\s*$"], + "includePatterns": [ + "Permission denied", + "Host key verification failed", + "Connection (?:closed|timed out)", + "^Welcome " + ], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["Permission denied", "Host key verification failed", "Connection timed out"], + "summaryPatterns": ["^Welcome "] + }, + "tests": [ + { + "name": "inline sample", + "command": "ssh host", + "input": "debug1: Connecting to host\nWelcome to Ubuntu\nPermission denied (publickey).\n", + "expected": "Welcome to Ubuntu\nPermission denied (publickey)." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/systemctl-status.json b/open-sse/services/compression/engines/rtk/filters/systemctl-status.json new file mode 100644 index 0000000000..f75b502bc1 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/systemctl-status.json @@ -0,0 +1,43 @@ +{ + "id": "systemctl-status", + "label": "systemctl status", + "description": "Preserve unit status, active state, and journal errors.", + "category": "infra", + "priority": 77, + "match": { + "outputTypes": [], + "commands": ["^systemctl\\s+status\\b"], + "patterns": ["Loaded:", "Active:", "Main PID:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "Loaded:", + "Active:", + "Main PID:", + "CGroup:", + "error", + "failed", + "Started", + "Stopped" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^\\s+Docs:"], + "deduplicate": true, + "maxLines": 100, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["failed", "error"], + "summaryPatterns": ["Active:", "Loaded:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "systemctl status", + "input": "Loaded: loaded (/etc/systemd/system/app.service)\nActive: failed (Result: exit-code)\nMain PID: 123\nMay 1 app[123]: error boot failed\n", + "expected": "Loaded: loaded (/etc/systemd/system/app.service)\nActive: failed (Result: exit-code)\nMain PID: 123\nMay 1 app[123]: error boot failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/terraform-plan.json b/open-sse/services/compression/engines/rtk/filters/terraform-plan.json new file mode 100644 index 0000000000..3f9d417253 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/terraform-plan.json @@ -0,0 +1,34 @@ +{ + "id": "terraform-plan", + "label": "Terraform Plan", + "description": "Keep plan actions and summary while stripping refresh noise.", + "category": "infra", + "priority": 88, + "match": { + "outputTypes": [], + "commands": ["^terraform\\s+plan\\b"], + "patterns": ["Terraform will perform the following actions", "Plan: \\d+ to add"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["Refreshing state", "Acquiring state lock", "Releasing state lock", "^\\s*$"], + "includePatterns": ["Terraform will perform", "^[ ]*[#~+\\-/]", "Plan: \\d+", "Error:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 160, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["Error:", "failed"], + "summaryPatterns": ["Plan: \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "terraform plan", + "input": "Refreshing state... [id=a]\nTerraform will perform the following actions:\n # aws_instance.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy.\n", + "expected": "Terraform will perform the following actions:\n # aws_instance.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-cargo.json b/open-sse/services/compression/engines/rtk/filters/test-cargo.json new file mode 100644 index 0000000000..e7aecaca5c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-cargo.json @@ -0,0 +1,42 @@ +{ + "id": "test-cargo", + "label": "Cargo Test", + "description": "Keep Cargo test failures and final summary.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-cargo"], + "commands": ["^cargo\\s+(?:test|nextest)\\b"], + "patterns": ["^running \\d+ tests?", "test result:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "^running \\d+ tests?", + "^test .* \\.\\.\\. FAILED", + "^failures:", + "^---- ", + "^thread ", + "^test result:", + "^error:" + ], + "dropPatterns": ["^test .* \\.\\.\\. ok$", "^\\s*$"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 180, + "headLines": 40, + "tailLines": 70 + }, + "preserve": { + "errorPatterns": ["FAILED", "^error:", "^thread "], + "summaryPatterns": ["test result:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "cargo (?:test|nextest)", + "input": "running 2 tests\ntest a ... ok\ntest b ... FAILED\nfailures:\n---- b stdout ----\nthread b panicked at boom\ntest result: FAILED. 1 passed; 1 failed\n", + "expected": "running 2 tests\ntest b ... FAILED\nfailures:\n---- b stdout ----\nthread b panicked at boom\ntest result: FAILED. 1 passed; 1 failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-go.json b/open-sse/services/compression/engines/rtk/filters/test-go.json new file mode 100644 index 0000000000..5e4f05e50d --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-go.json @@ -0,0 +1,41 @@ +{ + "id": "test-go", + "label": "Go Test", + "description": "Keep failing Go tests and package summaries.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-go"], + "commands": ["^go\\s+test\\b"], + "patterns": ["^--- FAIL:", "^(?:ok|FAIL)\\s+"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "^--- FAIL:", + "^FAIL", + "^ok\\s+", + "^panic:", + "^\\s+.*_test\\.go:\\d+", + "^\\s*Error" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^=== RUN"], + "deduplicate": true, + "maxLines": 160, + "headLines": 35, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["^--- FAIL:", "^panic:", "^FAIL"], + "summaryPatterns": ["^ok\\s+", "^FAIL\\s+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "go test", + "input": "=== RUN TestA\n--- FAIL: TestA (0.00s)\n a_test.go:1: boom\nFAIL\t./pkg\t0.1s\n", + "expected": "--- FAIL: TestA (0.00s)\n a_test.go:1: boom\nFAIL\t./pkg\t0.1s" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-jest.json b/open-sse/services/compression/engines/rtk/filters/test-jest.json new file mode 100644 index 0000000000..62f4c0a5d0 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-jest.json @@ -0,0 +1,43 @@ +{ + "id": "test-jest", + "label": "Jest", + "description": "Keep Jest failures and summary while removing passing noise.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-jest"], + "commands": ["^(?:jest|npm\\s+(?:run\\s+)?test)\\b"], + "patterns": ["Test Suites:\\s+\\d+", "Tests:\\s+\\d+", "^PASS\\s+", "^FAIL\\s+"] + }, + "rules": { + "includePatterns": [ + "^FAIL\\s+", + "^\\s*●\\s+", + "Expected:", + "Received:", + "Test Suites:", + "Tests:", + "Snapshots:", + "Time:", + "\\s+at\\s+" + ], + "dropPatterns": ["^PASS\\s+", "^\\s*✓\\s+"], + "collapsePatterns": ["^\\s*at "], + "deduplicate": true, + "maxLines": 140, + "headLines": 24, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["FAIL", "Expected:", "Received:", "Test Suites:"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:jest|npm (?:run )?test)", + "input": "PASS src/a.test.ts\nFAIL src/b.test.ts\nError: boom\nTest Suites: 1 failed, 1 passed\nTests: 1 failed, 1 passed\n", + "expected": "FAIL src/b.test.ts\nTest Suites: 1 failed, 1 passed\nTests: 1 failed, 1 passed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-pytest.json b/open-sse/services/compression/engines/rtk/filters/test-pytest.json new file mode 100644 index 0000000000..4e62bcc34c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-pytest.json @@ -0,0 +1,41 @@ +{ + "id": "test-pytest", + "label": "Pytest", + "description": "Keep pytest failures, tracebacks and terminal summary.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-pytest"], + "commands": ["^(?:pytest|python\\s+-m\\s+pytest)\\b"], + "patterns": ["=+\\s+(?:\\d+\\s+)?(?:passed|failed|errors?)", "^E\\s+", "^FAILED "] + }, + "rules": { + "includePatterns": [ + "^FAILED ", + "^ERROR ", + "^E\\s+", + "Traceback \\(most recent call last\\):", + "=+ short test summary info =+", + "=+ .*failed", + "AssertionError" + ], + "dropPatterns": ["^\\.", "^\\s*$"], + "collapsePatterns": ["^E\\s+"], + "deduplicate": true, + "maxLines": 160, + "headLines": 28, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["FAILED", "ERROR", "Traceback", "AssertionError"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:pytest|python -m pytest)", + "input": "FAILED tests/test_a.py::test_a - AssertionError\nE assert 1 == 2\n================ 1 failed, 2 passed ================\n", + "expected": "FAILED tests/test_a.py::test_a - AssertionError\nE assert 1 == 2\n================ 1 failed, 2 passed ================" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-vitest.json b/open-sse/services/compression/engines/rtk/filters/test-vitest.json new file mode 100644 index 0000000000..bb8a7480de --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-vitest.json @@ -0,0 +1,43 @@ +{ + "id": "test-vitest", + "label": "Vitest", + "description": "Keep failing suites, failed tests, stack headers, and final summary.", + "category": "test", + "priority": 92, + "match": { + "outputTypes": ["test-vitest"], + "commands": ["^(?:vitest|npm\\s+(?:run\\s+)?test:vitest)\\b"], + "patterns": ["\\bvitest\\b", "^ ✓ ", "^ ❯ ", "Test Files\\s+\\d+\\s+(?:passed|failed)"] + }, + "rules": { + "includePatterns": [ + "^\\s*FAIL\\s+", + "^\\s*❯\\s+", + "^\\s*×\\s+", + "Test Files\\s+", + "Tests\\s+", + "Duration\\s+", + "Error:", + "AssertionError", + "\\s+at\\s+" + ], + "dropPatterns": ["^\\s*✓\\s+", "^\\s*stdout \\|", "^\\s*stderr \\|"], + "collapsePatterns": ["^\\s*at "], + "deduplicate": true, + "maxLines": 140, + "headLines": 24, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["FAIL", "Error:", "AssertionError", "Test Files"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:vitest|npm (?:run )?test:vitest)", + "input": " ✓ src/a.test.ts (1)\n ❯ src/b.test.ts (1)\n FAIL src/b.test.ts > fails\nError: boom\nTest Files 1 failed | 1 passed\nTests 1 failed | 1 passed\n", + "expected": " ❯ src/b.test.ts (1)\n FAIL src/b.test.ts > fails\nError: boom\nTest Files 1 failed | 1 passed\nTests 1 failed | 1 passed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/tofu-plan.json b/open-sse/services/compression/engines/rtk/filters/tofu-plan.json new file mode 100644 index 0000000000..d6b8c69ee5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/tofu-plan.json @@ -0,0 +1,34 @@ +{ + "id": "tofu-plan", + "label": "OpenTofu Plan", + "description": "Keep OpenTofu plan actions and summary while stripping refresh noise.", + "category": "infra", + "priority": 87, + "match": { + "outputTypes": [], + "commands": ["^tofu\\s+plan\\b"], + "patterns": ["OpenTofu will perform the following actions", "Plan: \\d+ to add"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["Refreshing state", "Acquiring state lock", "Releasing state lock", "^\\s*$"], + "includePatterns": ["OpenTofu will perform", "^[ ]*[#~+\\-/]", "Plan: \\d+", "Error:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 160, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["Error:", "failed"], + "summaryPatterns": ["Plan: \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "tofu plan", + "input": "Refreshing state... [id=a]\nOpenTofu will perform the following actions:\n # tofu_resource.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy.\n", + "expected": "OpenTofu will perform the following actions:\n # tofu_resource.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/turbo.json b/open-sse/services/compression/engines/rtk/filters/turbo.json new file mode 100644 index 0000000000..0727717b65 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/turbo.json @@ -0,0 +1,34 @@ +{ + "id": "turbo", + "label": "Turborepo", + "description": "Compact turbo task cache and failure output.", + "category": "build", + "priority": 76, + "match": { + "outputTypes": [], + "commands": ["^turbo\\b", "^npx\\s+turbo\\b"], + "patterns": ["Tasks:", "Remote caching"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["cache hit", "cache miss", "^\\s*$"], + "includePatterns": ["failed", "error", "Tasks:", "Cached:", "Time:", "successful"], + "collapsePatterns": ["^• Packages in scope"], + "deduplicate": true, + "maxLines": 120, + "headLines": 25, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["failed", "error"], + "summaryPatterns": ["Tasks:", "Cached:", "Time:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "turbo", + "input": "cache hit app#build\napp#test: failed\nTasks: 1 successful, 1 failed\nTime: 2s\n", + "expected": "app#test: failed\nTasks: 1 successful, 1 failed\nTime: 2s" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/uv-sync.json b/open-sse/services/compression/engines/rtk/filters/uv-sync.json new file mode 100644 index 0000000000..ecac0a0321 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/uv-sync.json @@ -0,0 +1,35 @@ +{ + "id": "uv-sync", + "label": "uv sync", + "description": "Compact uv sync/install output.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^uv\\s+sync\\b", "^uv\\s+pip\\s+install\\b"], + "patterns": ["Resolved \\d+ packages?", "Prepared \\d+ packages?"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Resolved ", "^Prepared ", "^Installed ", "^ \\+", "^\\s*$"], + "includePatterns": ["error", "failed", "Installed", "Uninstalled", "Resolved"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "uv: ok" + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["Installed", "Resolved"] + }, + "tests": [ + { + "name": "inline sample", + "command": "uv sync", + "input": "Resolved 10 packages in 1ms\nInstalled 10 packages in 2ms\n", + "expected": "uv: ok" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/wget.json b/open-sse/services/compression/engines/rtk/filters/wget.json new file mode 100644 index 0000000000..b5de52bb99 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/wget.json @@ -0,0 +1,34 @@ +{ + "id": "wget", + "label": "wget", + "description": "Remove wget progress output and keep final errors or saved-file summaries.", + "category": "cloud", + "priority": 70, + "match": { + "outputTypes": ["wget"], + "commands": ["^wget\\b"], + "patterns": ["^--\\d{4}-\\d{2}-\\d{2}", "^ERROR \\d{3}:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*\\d+K", "^Saving to:", "^\\s*$"], + "includePatterns": ["^ERROR \\d{3}:", "saved \\[\\d", "^HTTP request sent"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^ERROR \\d{3}:", "failed", "unable to resolve"], + "summaryPatterns": ["saved \\[\\d", "^HTTP request sent"] + }, + "tests": [ + { + "name": "inline sample", + "command": "wget", + "input": "--2026-05-02-- https://example.test/file\nSaving to: 'file'\nHTTP request sent, awaiting response... 404 Not Found\nERROR 404: Not Found.\n", + "expected": "HTTP request sent, awaiting response... 404 Not Found\nERROR 404: Not Found." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/index.ts b/open-sse/services/compression/engines/rtk/index.ts new file mode 100644 index 0000000000..61fd9d535a --- /dev/null +++ b/open-sse/services/compression/engines/rtk/index.ts @@ -0,0 +1,404 @@ +import { createCompressionStats, estimateCompressionTokens } from "../../stats.ts"; +import { DEFAULT_RTK_CONFIG, type CompressionResult, type RtkConfig } from "../../types.ts"; +import type { CompressionEngine, EngineConfigField, EngineValidationResult } from "../types.ts"; +import { detectCommandType } from "./commandDetector.ts"; +import { deduplicateRepeatedLines } from "./deduplicator.ts"; +import { matchRtkFilter } from "./filterLoader.ts"; +import { applyLineFilter } from "./lineFilter.ts"; +import { smartTruncate } from "./smartTruncate.ts"; +import { normalizeCodeLanguage, stripCode } from "./codeStripper.ts"; +import { maybePersistRtkRawOutput, type RtkRawOutputPointer } from "./rawOutput.ts"; + +type Message = { + role: string; + content?: string | Array<{ type?: string; text?: string; [key: string]: unknown }>; + [key: string]: unknown; +}; + +const RTK_SCHEMA: EngineConfigField[] = [ + { + key: "intensity", + type: "select", + label: "Intensity", + defaultValue: DEFAULT_RTK_CONFIG.intensity, + options: [ + { value: "minimal", label: "minimal" }, + { value: "standard", label: "standard" }, + { value: "aggressive", label: "aggressive" }, + ], + }, + { + key: "applyToToolResults", + type: "boolean", + label: "Apply to tool results", + defaultValue: DEFAULT_RTK_CONFIG.applyToToolResults, + }, + { + key: "applyToAssistantMessages", + type: "boolean", + label: "Apply to assistant messages", + defaultValue: DEFAULT_RTK_CONFIG.applyToAssistantMessages, + }, + { + key: "applyToCodeBlocks", + type: "boolean", + label: "Apply to code blocks", + defaultValue: DEFAULT_RTK_CONFIG.applyToCodeBlocks, + }, + { + key: "maxLinesPerResult", + type: "number", + label: "Max lines per result", + defaultValue: DEFAULT_RTK_CONFIG.maxLinesPerResult, + min: 0, + max: 5000, + }, + { + key: "maxCharsPerResult", + type: "number", + label: "Max chars per result", + defaultValue: DEFAULT_RTK_CONFIG.maxCharsPerResult, + min: 0, + max: 500000, + }, + { + key: "deduplicateThreshold", + type: "number", + label: "Deduplicate threshold", + defaultValue: DEFAULT_RTK_CONFIG.deduplicateThreshold, + min: 2, + max: 100, + }, + { + key: "rawOutputRetention", + type: "select", + label: "Raw output retention", + defaultValue: DEFAULT_RTK_CONFIG.rawOutputRetention, + options: [ + { value: "never", label: "never" }, + { value: "failures", label: "failures" }, + { value: "always", label: "always" }, + ], + }, +]; + +function validateRtkEngineConfig(config: Record): EngineValidationResult { + const errors: string[] = []; + if ( + config.intensity !== undefined && + config.intensity !== "minimal" && + config.intensity !== "standard" && + config.intensity !== "aggressive" + ) { + errors.push("intensity must be minimal, standard, or aggressive"); + } + for (const key of [ + "enabled", + "applyToToolResults", + "applyToAssistantMessages", + "applyToCodeBlocks", + ]) { + if (config[key] !== undefined && typeof config[key] !== "boolean") { + errors.push(`${key} must be a boolean`); + } + } + for (const key of ["maxLinesPerResult", "maxCharsPerResult", "deduplicateThreshold"]) { + if (config[key] !== undefined && (typeof config[key] !== "number" || config[key] < 0)) { + errors.push(`${key} must be a non-negative number`); + } + } + if (config.enabledFilters !== undefined && !Array.isArray(config.enabledFilters)) { + errors.push("enabledFilters must be an array"); + } + if (config.disabledFilters !== undefined && !Array.isArray(config.disabledFilters)) { + errors.push("disabledFilters must be an array"); + } + if ( + config.rawOutputRetention !== undefined && + config.rawOutputRetention !== "never" && + config.rawOutputRetention !== "failures" && + config.rawOutputRetention !== "always" + ) { + errors.push("rawOutputRetention must be never, failures, or always"); + } + return { valid: errors.length === 0, errors }; +} + +export interface RtkProcessResult { + text: string; + compressed: boolean; + originalTokens: number; + compressedTokens: number; + techniquesUsed: string[]; + rulesApplied: string[]; + rawOutputPointers?: RtkRawOutputPointer[]; +} + +function mergeRtkConfig(base?: Partial, override?: Record): RtkConfig { + const merged = { ...DEFAULT_RTK_CONFIG, ...(base ?? {}), ...(override ?? {}) }; + return { + ...merged, + intensity: + merged.intensity === "minimal" || + merged.intensity === "standard" || + merged.intensity === "aggressive" + ? merged.intensity + : DEFAULT_RTK_CONFIG.intensity, + enabledFilters: Array.isArray(merged.enabledFilters) + ? merged.enabledFilters.filter((id): id is string => typeof id === "string") + : [], + disabledFilters: Array.isArray(merged.disabledFilters) + ? merged.disabledFilters.filter((id): id is string => typeof id === "string") + : [], + maxLinesPerResult: + typeof merged.maxLinesPerResult === "number" && Number.isFinite(merged.maxLinesPerResult) + ? Math.max(0, Math.floor(merged.maxLinesPerResult)) + : DEFAULT_RTK_CONFIG.maxLinesPerResult, + maxCharsPerResult: + typeof merged.maxCharsPerResult === "number" && Number.isFinite(merged.maxCharsPerResult) + ? Math.max(0, Math.floor(merged.maxCharsPerResult)) + : DEFAULT_RTK_CONFIG.maxCharsPerResult, + deduplicateThreshold: + typeof merged.deduplicateThreshold === "number" && + Number.isFinite(merged.deduplicateThreshold) + ? Math.max(2, Math.floor(merged.deduplicateThreshold)) + : DEFAULT_RTK_CONFIG.deduplicateThreshold, + customFiltersEnabled: + typeof merged.customFiltersEnabled === "boolean" + ? merged.customFiltersEnabled + : DEFAULT_RTK_CONFIG.customFiltersEnabled, + trustProjectFilters: + typeof merged.trustProjectFilters === "boolean" + ? merged.trustProjectFilters + : DEFAULT_RTK_CONFIG.trustProjectFilters, + rawOutputRetention: + merged.rawOutputRetention === "never" || + merged.rawOutputRetention === "failures" || + merged.rawOutputRetention === "always" + ? merged.rawOutputRetention + : DEFAULT_RTK_CONFIG.rawOutputRetention, + rawOutputMaxBytes: + typeof merged.rawOutputMaxBytes === "number" && Number.isFinite(merged.rawOutputMaxBytes) + ? Math.max(1024, Math.floor(merged.rawOutputMaxBytes)) + : DEFAULT_RTK_CONFIG.rawOutputMaxBytes, + }; +} + +function shouldCompressMessage(message: Message, config: RtkConfig): boolean { + if (message.role === "tool") return config.applyToToolResults || config.applyToCodeBlocks; + if (message.role === "assistant") + return config.applyToAssistantMessages || config.applyToCodeBlocks; + return false; +} + +function mapStringContent( + content: Message["content"], + transform: (text: string) => string +): Message["content"] { + if (typeof content === "string") return transform(content); + if (!Array.isArray(content)) return content; + return content.map((part) => { + if (part && typeof part === "object" && part.type === "text" && typeof part.text === "string") { + return { ...part, text: transform(part.text) }; + } + return part; + }); +} + +function extractContentText(content: Message["content"]): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map((part) => + part && typeof part === "object" && typeof part.text === "string" ? part.text : "" + ) + .filter(Boolean) + .join("\n"); +} + +export function processRtkText( + text: string, + options: { command?: string | null; config?: Partial } = {} +): RtkProcessResult { + const config = mergeRtkConfig(options.config); + const originalTokens = estimateCompressionTokens(text); + const techniquesUsed: string[] = []; + const rulesApplied: string[] = []; + const rawOutputPointers: RtkRawOutputPointer[] = []; + let result = text; + + const detection = detectCommandType(text, options.command); + const filter = matchRtkFilter(text, detection.command, { + customFiltersEnabled: config.customFiltersEnabled, + trustProjectFilters: config.trustProjectFilters, + }); + if (filter && !config.disabledFilters.includes(filter.id)) { + if (config.enabledFilters.length === 0 || config.enabledFilters.includes(filter.id)) { + const filtered = applyLineFilter(result, { + ...filter, + maxLines: filter.maxLines || config.maxLinesPerResult, + }); + result = filtered.text; + if (filtered.appliedRules.length > 0) { + techniquesUsed.push("rtk-filter"); + rulesApplied.push(...filtered.appliedRules); + } + } + } + + if (config.applyToCodeBlocks) { + let strippedCodeBlocks = 0; + result = result.replace( + /```([A-Za-z0-9_+.-]*)\r?\n([\s\S]*?)```/g, + (match, languageHint: string, code: string) => { + const stripped = stripCode(code, normalizeCodeLanguage(languageHint)); + if (stripped.strippedLines <= 0 && stripped.text === code.trim()) return match; + strippedCodeBlocks++; + const fenceLanguage = languageHint?.trim() || stripped.language; + return `\`\`\`${fenceLanguage}\n${stripped.text}\n\`\`\``; + } + ); + if (strippedCodeBlocks > 0) { + techniquesUsed.push("rtk-code-strip"); + rulesApplied.push("rtk:code-strip"); + } + } + + if (config.intensity !== "minimal") { + const deduped = deduplicateRepeatedLines(result, { threshold: config.deduplicateThreshold }); + if (deduped.collapsed > 0) { + result = deduped.text; + techniquesUsed.push("rtk-dedup"); + rulesApplied.push("rtk:dedup"); + } + } + + const truncated = smartTruncate(result, { + maxLines: config.maxLinesPerResult, + maxChars: config.maxCharsPerResult, + preserveHead: config.intensity === "aggressive" ? 16 : 24, + preserveTail: config.intensity === "aggressive" ? 16 : 24, + priorityPatterns: [/error|failed|exception|traceback|TS\d{4}|FAIL|✖/i], + }); + if (truncated.truncated) { + result = truncated.text; + techniquesUsed.push("rtk-truncate"); + rulesApplied.push("rtk:truncate"); + } + + const compressedTokens = estimateCompressionTokens(result); + if (compressedTokens < originalTokens) { + const pointer = maybePersistRtkRawOutput(text, { + retention: config.rawOutputRetention, + command: detection.command, + maxBytes: config.rawOutputMaxBytes, + }); + if (pointer) { + rawOutputPointers.push(pointer); + techniquesUsed.push("rtk-raw-output-retention"); + rulesApplied.push("rtk:raw-output-retention"); + } + } + return { + text: result, + compressed: compressedTokens < originalTokens, + originalTokens, + compressedTokens, + techniquesUsed: [...new Set(techniquesUsed)], + rulesApplied: [...new Set(rulesApplied)], + ...(rawOutputPointers.length > 0 ? { rawOutputPointers } : {}), + }; +} + +export function applyRtkCompression( + body: Record, + options: { config?: Partial; stepConfig?: Record } = {} +): CompressionResult { + const start = performance.now(); + const config = mergeRtkConfig(options.config, options.stepConfig); + if (!config.enabled) return { body, compressed: false, stats: null }; + + const messages = body.messages as Message[] | undefined; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + + const allTechniques: string[] = []; + const allRules: string[] = []; + const rawOutputPointers: RtkRawOutputPointer[] = []; + const compressedMessages = messages.map((message) => { + if (!shouldCompressMessage(message, config)) return message; + const text = extractContentText(message.content); + if (!text) return message; + const processed = processRtkText(text, { config }); + allTechniques.push(...processed.techniquesUsed); + allRules.push(...processed.rulesApplied); + if (processed.rawOutputPointers) rawOutputPointers.push(...processed.rawOutputPointers); + if (!processed.compressed) return message; + return { + ...message, + content: mapStringContent(message.content, () => processed.text), + }; + }); + + const compressedBody = { ...body, messages: compressedMessages }; + const stats = createCompressionStats( + body, + compressedBody, + "rtk", + [...new Set(allTechniques)], + allRules.length > 0 ? [...new Set(allRules)] : undefined, + Math.round((performance.now() - start) * 100) / 100 + ); + stats.engine = "rtk"; + if (rawOutputPointers.length > 0) { + stats.rtkRawOutputPointers = rawOutputPointers; + } + return { + body: compressedBody, + compressed: stats.compressedTokens < stats.originalTokens, + stats, + }; +} + +export const rtkEngine: CompressionEngine = { + id: "rtk", + name: "RTK", + description: "Command-aware tool output compression with declarative filters.", + icon: "filter_alt", + targets: ["tool_results", "code_blocks"], + stackable: true, + stackPriority: 10, + metadata: { + id: "rtk", + name: "RTK", + description: "Command-aware tool output compression with declarative filters.", + inputScope: "tool-results", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + return applyRtkCompression(body, { + config: options?.config?.rtkConfig, + stepConfig: options?.stepConfig, + }); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return RTK_SCHEMA; + }, + validateConfig(config) { + return validateRtkEngineConfig(config); + }, +}; + +export { + detectCommandFromText, + detectCommandOutput, + detectCommandType, +} from "./commandDetector.ts"; +export { runRtkFilterTests } from "./verify.ts"; +export { maybePersistRtkRawOutput, readRtkRawOutput, redactRtkRawOutput } from "./rawOutput.ts"; diff --git a/open-sse/services/compression/engines/rtk/lineFilter.ts b/open-sse/services/compression/engines/rtk/lineFilter.ts new file mode 100644 index 0000000000..589b618086 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/lineFilter.ts @@ -0,0 +1,155 @@ +import type { RtkFilterDefinition } from "./filterSchema.ts"; +import { smartTruncate } from "./smartTruncate.ts"; + +export interface LineFilterResult { + text: string; + strippedLines: number; + keptByRule: boolean; + appliedRules: string[]; +} + +function compilePatterns(patterns: string[]): RegExp[] { + return patterns.flatMap((pattern) => { + try { + return [new RegExp(pattern, "i")]; + } catch { + return []; + } + }); +} + +function compileGlobalPattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "g"); + } catch { + return null; + } +} + +function compileBlobPattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "im"); + } catch { + return null; + } +} + +function stripAnsi(text: string): string { + return text.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, ""); +} + +function normalizeStderrPrefix(line: string): string { + return line.replace(/^\s*(?:stderr|err)\s*(?:\||:)\s*/i, ""); +} + +function truncateUnicodeSafe(line: string, maxChars: number): string { + if (maxChars <= 0) return line; + const chars = Array.from(line); + if (chars.length <= maxChars) return line; + if (maxChars <= 3) return chars.slice(0, maxChars).join(""); + return `${chars.slice(0, maxChars - 3).join("")}...`; +} + +export function applyLineFilter(text: string, filter: RtkFilterDefinition): LineFilterResult { + const stripPatterns = compilePatterns(filter.stripPatterns); + const keepPatterns = compilePatterns(filter.keepPatterns); + const collapsePatterns = compilePatterns(filter.collapsePatterns); + const priorityPatterns = compilePatterns(filter.priorityPatterns); + const appliedRules: string[] = []; + + let lines = text.split(/\r?\n/); + const originalLineCount = lines.length; + + if (filter.stripAnsi) { + const stripped = lines.map(stripAnsi); + if (stripped.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:strip-ansi`); + } + lines = stripped; + } + + if (filter.filterStderr) { + const normalized = lines.map(normalizeStderrPrefix); + if (normalized.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:filter-stderr`); + } + lines = normalized; + } + + for (const rule of filter.replace) { + const pattern = compileGlobalPattern(rule.pattern); + if (!pattern) continue; + const replaced = lines.map((line) => line.replace(pattern, rule.replacement)); + if (replaced.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:replace`); + } + lines = replaced; + } + + if (filter.matchOutput.length > 0) { + const blob = lines.join("\n"); + for (const rule of filter.matchOutput) { + const pattern = compileBlobPattern(rule.pattern); + if (!pattern?.test(blob)) continue; + const unless = rule.unless ? compileBlobPattern(rule.unless) : null; + if (unless?.test(blob)) continue; + appliedRules.push(`${filter.id}:match-output`); + return { + text: rule.message, + strippedLines: Math.max(0, originalLineCount - 1), + keptByRule: false, + appliedRules, + }; + } + } + + if (stripPatterns.length > 0) { + lines = lines.filter((line) => !stripPatterns.some((pattern) => pattern.test(line))); + if (lines.length !== originalLineCount) appliedRules.push(`${filter.id}:strip`); + } + + if (keepPatterns.length > 0) { + const kept = lines.filter((line) => keepPatterns.some((pattern) => pattern.test(line))); + if (kept.length > 0) { + lines = kept; + appliedRules.push(`${filter.id}:keep`); + } + } + + if (collapsePatterns.length > 0) { + const seen = new Set(); + lines = lines.filter((line) => { + if (!collapsePatterns.some((pattern) => pattern.test(line))) return true; + const key = line.trim(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + appliedRules.push(`${filter.id}:collapse`); + } + + if (filter.truncateLineAt > 0) { + const truncatedLines = lines.map((line) => truncateUnicodeSafe(line, filter.truncateLineAt)); + if (truncatedLines.join("\n") !== lines.join("\n")) { + lines = truncatedLines; + appliedRules.push(`${filter.id}:truncate-line`); + } + } + + const truncated = smartTruncate(lines.join("\n"), { + maxLines: filter.maxLines, + preserveHead: filter.preserveHead, + preserveTail: filter.preserveTail, + priorityPatterns, + }); + if (truncated.truncated) appliedRules.push(`${filter.id}:truncate`); + const output = + truncated.text.trim().length === 0 && filter.onEmpty ? filter.onEmpty : truncated.text; + + return { + text: output, + strippedLines: Math.max(0, originalLineCount - output.split(/\r?\n/).length), + keptByRule: keepPatterns.length > 0, + appliedRules, + }; +} diff --git a/open-sse/services/compression/engines/rtk/rawOutput.ts b/open-sse/services/compression/engines/rtk/rawOutput.ts new file mode 100644 index 0000000000..7e40d6a9cb --- /dev/null +++ b/open-sse/services/compression/engines/rtk/rawOutput.ts @@ -0,0 +1,112 @@ +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import crypto from "node:crypto"; + +export type RtkRawOutputRetention = "never" | "failures" | "always"; + +export interface RtkRawOutputPointer { + id: string; + path: string; + bytes: number; + sha256: string; + redacted: boolean; +} + +const SECRET_PATTERNS: Array<[RegExp, string]> = [ + [/\b(sk-[A-Za-z0-9_-]{16,})\b/g, "[REDACTED_OPENAI_KEY]"], + [/\b(xox[baprs]-[A-Za-z0-9-]{16,})\b/g, "[REDACTED_SLACK_TOKEN]"], + [/\b(AKIA[0-9A-Z]{16})\b/g, "[REDACTED_AWS_KEY]"], + [/((?:api[_-]?key|token|secret|password)\s*[:=]\s*)("[^"]+"|'[^']+'|[^\s]+)/gi, "$1[REDACTED]"], + [/(Authorization:\s*Bearer\s+)[A-Za-z0-9._~+/-]+=*/gi, "$1[REDACTED]"], +]; + +function dataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +function safeId(seed: string): string { + return crypto.createHash("sha256").update(seed).digest("hex").slice(0, 24); +} + +function safeUtf8Slice(value: string, maxBytes: number): string { + if (maxBytes <= 0 || Buffer.byteLength(value, "utf8") <= maxBytes) return value; + let bytes = 0; + let output = ""; + for (const char of value) { + const len = Buffer.byteLength(char, "utf8"); + if (bytes + len > maxBytes) break; + output += char; + bytes += len; + } + return `${output}\n\n--- truncated at ${maxBytes} bytes ---`; +} + +export function redactRtkRawOutput(value: string): { text: string; redacted: boolean } { + let redacted = false; + let text = value; + for (const [pattern, replacement] of SECRET_PATTERNS) { + const next = text.replace(pattern, (...args: string[]) => { + redacted = true; + return typeof replacement === "string" + ? replacement.replace("$1", args[1] ?? "") + : replacement; + }); + text = next; + } + return { text, redacted }; +} + +export function isLikelyFailureOutput(value: string): boolean { + return /\b(error|failed|failure|exception|traceback|panic|fatal|critical|TS\d{4}|FAIL)\b/i.test( + value + ); +} + +export function maybePersistRtkRawOutput( + raw: string, + options: { + retention: RtkRawOutputRetention; + command?: string | null; + maxBytes?: number; + failure?: boolean; + } +): RtkRawOutputPointer | null { + if (options.retention === "never") return null; + const failure = options.failure ?? isLikelyFailureOutput(raw); + if (options.retention === "failures" && !failure) return null; + if (raw.trim().length === 0) return null; + + const maxBytes = Math.max(1024, Math.floor(options.maxBytes ?? 1_048_576)); + const redaction = redactRtkRawOutput(safeUtf8Slice(raw, maxBytes)); + const now = Date.now(); + const commandSlug = (options.command || "tool-output") + .replace(/[^A-Za-z0-9_-]+/g, "_") + .replace(/^_+|_+$/g, "") + .slice(0, 48); + const id = safeId(`${now}:${commandSlug}:${raw.length}:${redaction.text}`); + const dir = path.join(dataDir(), "rtk", "raw-output"); + const filePath = path.join(dir, `${now}-${commandSlug || "tool-output"}-${id}.log`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(filePath, redaction.text); + + return { + id, + path: filePath, + bytes: Buffer.byteLength(redaction.text, "utf8"), + sha256: crypto.createHash("sha256").update(redaction.text).digest("hex"), + redacted: redaction.redacted, + }; +} + +export function readRtkRawOutput(pointerId: string): string | null { + const dir = path.join(dataDir(), "rtk", "raw-output"); + if (!fs.existsSync(dir)) return null; + const entry = fs + .readdirSync(dir) + .find((file) => file.endsWith(".log") && file.includes(pointerId)); + if (!entry) return null; + const fullPath = path.join(dir, entry); + if (!fullPath.startsWith(dir)) return null; + return fs.readFileSync(fullPath, "utf8"); +} diff --git a/open-sse/services/compression/engines/rtk/smartTruncate.ts b/open-sse/services/compression/engines/rtk/smartTruncate.ts new file mode 100644 index 0000000000..8af0e708c2 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/smartTruncate.ts @@ -0,0 +1,67 @@ +export interface SmartTruncateOptions { + maxLines?: number; + maxChars?: number; + preserveHead?: number; + preserveTail?: number; + priorityPatterns?: RegExp[]; +} + +export function smartTruncate( + text: string, + options: SmartTruncateOptions = {} +): { + text: string; + truncated: boolean; + droppedLines: number; +} { + const maxChars = Math.max(0, Math.floor(options.maxChars ?? 0)); + const maxLines = Math.max(0, Math.floor(options.maxLines ?? 0)); + const lines = text.split(/\r?\n/); + const overLineLimit = maxLines > 0 && lines.length > maxLines; + const overCharLimit = maxChars > 0 && text.length > maxChars; + if (!overLineLimit && !overCharLimit) { + return { text, truncated: false, droppedLines: 0 }; + } + + const preserveHead = Math.max(0, Math.floor(options.preserveHead ?? 20)); + const preserveTail = Math.max(0, Math.floor(options.preserveTail ?? 20)); + const priorityPatterns = options.priorityPatterns ?? []; + const priorityLines = priorityPatterns.length + ? lines.filter((line) => priorityPatterns.some((pattern) => pattern.test(line))) + : []; + + const head = lines.slice(0, preserveHead); + const tail = preserveTail > 0 ? lines.slice(-preserveTail) : []; + const selected = [...head]; + for (const line of priorityLines) { + if (!selected.includes(line)) selected.push(line); + } + const tailStart = lines.length - tail.length; + tail.forEach((line, offset) => { + const originalIndex = tailStart + offset; + if (originalIndex >= preserveHead && !selected.includes(line)) selected.push(line); + }); + + const droppedLines = Math.max(0, lines.length - selected.length); + let result = [ + ...selected.slice(0, head.length), + `[rtk:truncated ${droppedLines} lines]`, + ...selected.slice(head.length), + ].join("\n"); + + if (maxChars > 0 && result.length > maxChars) { + const marker = "\n[rtk:truncated by chars]\n"; + const budget = Math.max(0, maxChars - marker.length); + if (budget === 0) { + result = marker.slice(0, maxChars); + return { text: result, truncated: true, droppedLines }; + } + const headChars = Math.ceil(budget * 0.55); + const tailChars = Math.max(0, budget - headChars); + const tailText = tailChars > 0 ? result.slice(-tailChars) : ""; + result = `${result.slice(0, headChars)}${marker}${tailText}`; + if (result.length > maxChars) result = result.slice(0, maxChars); + } + + return { text: result, truncated: true, droppedLines }; +} diff --git a/open-sse/services/compression/engines/rtk/verify.ts b/open-sse/services/compression/engines/rtk/verify.ts new file mode 100644 index 0000000000..c59a5e9102 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/verify.ts @@ -0,0 +1,107 @@ +import { estimateCompressionTokens } from "../../stats.ts"; +import { + getRtkFilterLoadDiagnostics, + loadRtkFilters, + type RtkFilterLoadDiagnostic, +} from "./filterLoader.ts"; +import { applyLineFilter } from "./lineFilter.ts"; + +export interface RtkFilterTestOutcome { + filterId: string; + testName: string; + passed: boolean; + actual: string; + expected: string; +} + +export interface RtkFilterBenchmarkRow { + category: string; + filters: number; + tests: number; + averageSavingsPercent: number; +} + +export interface RtkVerifyResult { + passed: boolean; + outcomes: RtkFilterTestOutcome[]; + filtersWithoutTests: string[]; + benchmark: RtkFilterBenchmarkRow[]; + diagnostics: RtkFilterLoadDiagnostic[]; +} + +function trimComparable(value: string): string { + return value.replace(/\n+$/g, ""); +} + +export function runRtkFilterTests( + options: { + requireAll?: boolean; + customFiltersEnabled?: boolean; + trustProjectFilters?: boolean; + } = {} +): RtkVerifyResult { + const filters = loadRtkFilters({ + refresh: true, + customFiltersEnabled: options.customFiltersEnabled, + trustProjectFilters: options.trustProjectFilters, + }); + const outcomes: RtkFilterTestOutcome[] = []; + const filtersWithoutTests: string[] = []; + const benchmarkByCategory = new Map< + string, + { filters: Set; tests: number; savingsTotal: number } + >(); + + for (const filter of filters) { + const categoryStats = benchmarkByCategory.get(filter.category) ?? { + filters: new Set(), + tests: 0, + savingsTotal: 0, + }; + categoryStats.filters.add(filter.id); + benchmarkByCategory.set(filter.category, categoryStats); + + if (filter.tests.length === 0) { + filtersWithoutTests.push(filter.id); + continue; + } + + for (const test of filter.tests) { + const result = applyLineFilter(test.input, filter).text; + const actual = trimComparable(result); + const expected = trimComparable(test.expected); + const originalTokens = estimateCompressionTokens(test.input); + const compressedTokens = estimateCompressionTokens(result); + const savings = + originalTokens > 0 ? ((originalTokens - compressedTokens) / originalTokens) * 100 : 0; + categoryStats.tests += 1; + categoryStats.savingsTotal += Math.max(0, savings); + outcomes.push({ + filterId: filter.id, + testName: test.name, + passed: actual === expected, + actual, + expected, + }); + } + } + + const benchmark = Array.from(benchmarkByCategory.entries()) + .map(([category, value]) => ({ + category, + filters: value.filters.size, + tests: value.tests, + averageSavingsPercent: + value.tests > 0 ? Math.round((value.savingsTotal / value.tests) * 100) / 100 : 0, + })) + .sort((a, b) => a.category.localeCompare(b.category)); + + const failed = outcomes.some((outcome) => !outcome.passed); + return { + passed: !failed && (!options.requireAll || filtersWithoutTests.length === 0), + outcomes, + filtersWithoutTests, + benchmark, + diagnostics: options.customFiltersEnabled === false ? [] : getRtkFilterLoadDiagnostics(), + }; +} diff --git a/open-sse/services/compression/engines/types.ts b/open-sse/services/compression/engines/types.ts new file mode 100644 index 0000000000..b270d8d1d5 --- /dev/null +++ b/open-sse/services/compression/engines/types.ts @@ -0,0 +1,59 @@ +import type { CompressionConfig, CompressionResult } from "../types.ts"; + +export type CompressionEngineTarget = "messages" | "tool_results" | "code_blocks"; + +export interface EngineConfigField { + key: string; + type: "boolean" | "number" | "string" | "select" | "multiselect"; + label: string; + i18nKey?: string; + description?: string; + defaultValue: unknown; + options?: Array<{ value: string; label: string }>; + min?: number; + max?: number; +} + +export interface EngineValidationResult { + valid: boolean; + errors: string[]; +} + +export interface CompressionEngineMetadata { + id: string; + name: string; + description: string; + inputScope: "messages" | "tool-results" | "mixed"; + targetLatencyMs: number; + supportsPreview: boolean; + stable: boolean; +} + +export interface CompressionEngineApplyOptions { + model?: string; + supportsVision?: boolean | null; + config?: CompressionConfig; + compressionComboId?: string | null; + stepConfig?: Record; +} + +export interface CompressionEngine { + id: string; + name: string; + description: string; + icon: string; + targets: CompressionEngineTarget[]; + stackable: boolean; + stackPriority: number; + metadata: CompressionEngineMetadata; + apply(body: Record, options?: CompressionEngineApplyOptions): CompressionResult; + compress(body: Record, config?: Record): CompressionResult; + getConfigSchema(): EngineConfigField[]; + validateConfig(config: Record): EngineValidationResult; +} + +export interface EngineRegistryEntry { + engine: CompressionEngine; + enabled: boolean; + config: Record; +} diff --git a/open-sse/services/compression/index.ts b/open-sse/services/compression/index.ts index efc5b4ddb6..7f16cd9f00 100644 --- a/open-sse/services/compression/index.ts +++ b/open-sse/services/compression/index.ts @@ -7,6 +7,12 @@ export type { CavemanRule, CavemanIntensity, CavemanOutputModeConfig, + RtkConfig, + RtkIntensity, + RtkRawOutputRetention, + CompressionEngineId, + CompressionLanguageConfig, + CompressionPipelineStep, AggressiveConfig, AgingThresholds, ToolStrategiesConfig, @@ -18,6 +24,8 @@ export { DEFAULT_COMPRESSION_CONFIG, DEFAULT_CAVEMAN_CONFIG, DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG, + DEFAULT_RTK_CONFIG, + DEFAULT_COMPRESSION_LANGUAGE_CONFIG, DEFAULT_AGGRESSIVE_CONFIG, } from "./types.ts"; @@ -32,6 +40,19 @@ export { export { cavemanCompress, applyRulesToText } from "./caveman.ts"; export { getRulesForContext, getCavemanRuleMetadata, CAVEMAN_RULES } from "./cavemanRules.ts"; +export { + getAvailableLanguagePacks, + listCavemanRulePacks, + loadAllRulesForLanguage, + loadCavemanFileRules, + loadRulePack, + validateRulePack, +} from "./ruleLoader.ts"; +export type { RulePackMetadata } from "./ruleLoader.ts"; +export { + detectCompressionLanguage, + listSupportedCompressionLanguages, +} from "./languageDetector.ts"; export { extractPreservedBlocks, restorePreservedBlocks, @@ -62,8 +83,62 @@ export { applyCompression, checkComboOverride, shouldAutoTrigger, + applyStackedCompression, } from "./strategySelector.ts"; +export type { + CompressionEngine, + CompressionEngineApplyOptions, + CompressionEngineMetadata, + CompressionEngineTarget, + EngineConfigField, + EngineRegistryEntry, + EngineValidationResult, +} from "./engines/types.ts"; + +export { + registerEngine, + registerCompressionEngine, + unregisterCompressionEngine, + getEngine, + getEngineEntry, + getCompressionEngine, + listEngines, + listCompressionEngines, + listEnabledEngines, + setEngineEnabled, + updateEngineConfig, + clearCompressionEngineRegistry, +} from "./engines/registry.ts"; +export { registerBuiltinCompressionEngines } from "./engines/index.ts"; + +export { applyRtkCompression, processRtkText, rtkEngine } from "./engines/rtk/index.ts"; +export { + detectCommandFromText, + detectCommandOutput, + detectCommandType, + type CommandDetectionResult, +} from "./engines/rtk/commandDetector.ts"; +export { + loadRtkFilters, + getRtkFilterCatalog, + matchRtkFilter, + getRtkFilterLoadDiagnostics, +} from "./engines/rtk/filterLoader.ts"; +export { runRtkFilterTests } from "./engines/rtk/verify.ts"; +export { + maybePersistRtkRawOutput, + readRtkRawOutput, + redactRtkRawOutput, +} from "./engines/rtk/rawOutput.ts"; +export { + detectCodeLanguage, + normalizeCodeLanguage, + stripCode, + stripCodeComments, +} from "./engines/rtk/codeStripper.ts"; +export type { CodeLanguage, CodeStripperOptions } from "./engines/rtk/codeStripper.ts"; + export { RuleBasedSummarizer, createSummarizer } from "./summarizer.ts"; export { compressToolResult } from "./toolResultCompressor.ts"; diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts new file mode 100644 index 0000000000..3b3aa9b002 --- /dev/null +++ b/open-sse/services/compression/languageDetector.ts @@ -0,0 +1,18 @@ +const LANGUAGE_HINTS: Record = { + "pt-BR": [/\b(?:voce|você|preciso|arquivo|codigo|código|erro|falha|obrigado)\b/i], + es: [/\b(?:necesito|archivo|codigo|código|error|fallo|gracias|puedes)\b/i], + de: [/\b(?:ich|datei|fehler|bitte|kannst|konfiguration|danke)\b/i], + fr: [/\b(?:fichier|erreur|merci|peux|configuration|besoin)\b/i], + ja: [/[\u3040-\u30ff]/], +}; + +export function detectCompressionLanguage(text: string): string { + for (const [language, patterns] of Object.entries(LANGUAGE_HINTS)) { + if (patterns.some((pattern) => pattern.test(text))) return language; + } + return "en"; +} + +export function listSupportedCompressionLanguages(): string[] { + return ["en", ...Object.keys(LANGUAGE_HINTS)]; +} diff --git a/open-sse/services/compression/outputMode.ts b/open-sse/services/compression/outputMode.ts index 478ae863f2..9f4e7bfe45 100644 --- a/open-sse/services/compression/outputMode.ts +++ b/open-sse/services/compression/outputMode.ts @@ -19,11 +19,43 @@ export interface CavemanOutputModeResult { skippedReason?: string; } -const CAVEMAN_INSTRUCTION_BY_INTENSITY = { - lite: "Respond concise. Drop filler, pleasantries, hedging. Keep full sentences, technical terms, code, errors, URLs, and identifiers exact.", - full: "Respond terse like smart caveman. Drop articles, filler, pleasantries, hedging. Fragments OK. Keep all technical substance, code, errors, URLs, identifiers exact.", - ultra: - "Respond ultra terse. Use short technical prose, arrows for causality, common abbreviations like DB/auth/config/req/res/fn. Never abbreviate code symbols, API names, error strings, URLs, or identifiers.", +const CAVEMAN_INSTRUCTION_BY_LANGUAGE = { + en: { + lite: "Respond concise. Drop filler, pleasantries, hedging. Keep full sentences, technical terms, code, errors, URLs, and identifiers exact.", + full: "Respond terse like smart caveman. Drop articles, filler, pleasantries, hedging. Fragments OK. Keep all technical substance, code, errors, URLs, identifiers exact.", + ultra: + "Respond ultra terse. Use short technical prose, arrows for causality, common abbreviations like DB/auth/config/req/res/fn. Never abbreviate code symbols, API names, error strings, URLs, or identifiers.", + }, + "pt-BR": { + lite: "Responda conciso. Remova enrolacao, cortesias e incerteza. Preserve termos tecnicos, codigo, erros, URLs e identificadores exatamente.", + full: "Responda seco e compacto. Frases curtas OK. Preserve todo conteudo tecnico, codigo, erros, URLs e identificadores exatamente.", + ultra: + "Responda ultra compacto. Use prosa tecnica curta e abreviacoes comuns como DB/auth/config/req/res/fn. Nunca abrevie simbolos de codigo, APIs, erros, URLs ou identificadores.", + }, + es: { + lite: "Responde conciso. Quita relleno, cortesias y dudas. Conserva terminos tecnicos, codigo, errores, URLs e identificadores exactos.", + full: "Responde seco y compacto. Fragmentos OK. Conserva todo el contenido tecnico, codigo, errores, URLs e identificadores exactos.", + ultra: + "Responde ultra compacto. Usa prosa tecnica corta y abreviaturas comunes como DB/auth/config/req/res/fn. Nunca abrevies simbolos de codigo, APIs, errores, URLs o identificadores.", + }, + de: { + lite: "Antworte knapp. Entferne Fuellwoerter, Hoeflichkeit und Unsicherheit. Bewahre Fachbegriffe, Code, Fehler, URLs und Bezeichner exakt.", + full: "Antworte sehr knapp. Fragmente OK. Bewahre alle technischen Inhalte, Code, Fehler, URLs und Bezeichner exakt.", + ultra: + "Antworte ultra knapp. Nutze kurze technische Prosa und uebliche Abkuerzungen wie DB/auth/config/req/res/fn. Code-Symbole, APIs, Fehler, URLs und Bezeichner nie abkuerzen.", + }, + fr: { + lite: "Reponds concis. Retire remplissage, politesses et hesitations. Garde termes techniques, code, erreurs, URLs et identifiants exacts.", + full: "Reponds tres compact. Fragments OK. Garde tout le contenu technique, code, erreurs, URLs et identifiants exacts.", + ultra: + "Reponds ultra compact. Utilise une prose technique courte et des abreviations communes comme DB/auth/config/req/res/fn. N'abrege jamais symboles de code, APIs, erreurs, URLs ou identifiants.", + }, + ja: { + lite: "簡潔に回答。冗長表現、挨拶、曖昧表現を削る。技術用語、コード、エラー、URL、識別子は正確に保持。", + full: "短く圧縮して回答。断片文可。技術内容、コード、エラー、URL、識別子は正確に保持。", + ultra: + "超短く回答。DB/auth/config/req/res/fn など一般的な略語は可。コード記号、API名、エラー文字列、URL、識別子は省略しない。", + }, } as const; const CAVEMAN_OUTPUT_MARKER = "[OmniRoute Caveman Output Mode]"; @@ -60,14 +92,21 @@ export function shouldBypassCavemanOutputMode(messages: ChatMessage[]): string | return null; } -export function buildCavemanOutputInstruction(config: CavemanOutputModeConfig): string { +export function buildCavemanOutputInstruction( + config: CavemanOutputModeConfig, + language = "en" +): string { const intensity = config.intensity ?? "full"; - return `${CAVEMAN_OUTPUT_MARKER}\n${CAVEMAN_INSTRUCTION_BY_INTENSITY[intensity]}`; + const instructions = + CAVEMAN_INSTRUCTION_BY_LANGUAGE[language as keyof typeof CAVEMAN_INSTRUCTION_BY_LANGUAGE] ?? + CAVEMAN_INSTRUCTION_BY_LANGUAGE.en; + return `${CAVEMAN_OUTPUT_MARKER}\n${instructions[intensity]}`; } export function applyCavemanOutputMode( body: ChatRequestBody, - options?: Partial + options?: Partial, + language = "en" ): CavemanOutputModeResult { const config: CavemanOutputModeConfig = { ...DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG, @@ -84,7 +123,7 @@ export function applyCavemanOutputMode( return { body: { ...body, - instructions: `${body.instructions.trim()}\n\n${buildCavemanOutputInstruction(config)}`, + instructions: `${body.instructions.trim()}\n\n${buildCavemanOutputInstruction(config, language)}`, }, applied: true, }; @@ -105,7 +144,7 @@ export function applyCavemanOutputMode( ); if (alreadyApplied) return { body, applied: false, skippedReason: "already_applied" }; - const instruction = buildCavemanOutputInstruction(config); + const instruction = buildCavemanOutputInstruction(config, language); const nextMessages = [...messages]; const first = nextMessages[0]; diff --git a/open-sse/services/compression/ruleLoader.ts b/open-sse/services/compression/ruleLoader.ts new file mode 100644 index 0000000000..ba149d8515 --- /dev/null +++ b/open-sse/services/compression/ruleLoader.ts @@ -0,0 +1,206 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CavemanIntensity, CavemanRule } from "./types.ts"; + +type CavemanRuleCategory = NonNullable; +type CavemanRuleContext = CavemanRule["context"]; + +interface FileRule { + name: string; + pattern: string; + replacement: string; + context?: CavemanRuleContext; + category?: CavemanRuleCategory; + minIntensity?: CavemanIntensity; + description?: string; +} + +interface RulePack { + language: string; + category: string; + rules: FileRule[]; +} + +export interface RulePackMetadata { + language: string; + categories: string[]; + ruleCount: number; +} + +const VALID_CONTEXTS = new Set(["all", "user", "system", "assistant"]); +const VALID_CATEGORIES = new Set(["filler", "context", "structural", "dedup", "terse", "ultra"]); +const VALID_INTENSITIES = new Set(["lite", "full", "ultra"]); +const cache = new Map(); + +function getRulesDir(): string { + const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + const candidates = [ + path.join(moduleDir, "rules"), + path.join(moduleDir, "..", "services", "compression", "rules"), + path.join(process.cwd(), "open-sse", "services", "compression", "rules"), + path.join(process.cwd(), "app", "open-sse", "services", "compression", "rules"), + ]; + return ( + candidates.find((candidate, index) => { + return candidates.indexOf(candidate) === index && fs.existsSync(candidate); + }) ?? candidates[0] + ); +} + +function compileRule(rule: FileRule, source: string): CavemanRule { + try { + return { + name: rule.name, + pattern: new RegExp(rule.pattern, "gi"), + replacement: rule.replacement, + context: rule.context ?? "all", + category: rule.category ?? "filler", + minIntensity: rule.minIntensity ?? "lite", + description: rule.description, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid Caveman rule pattern in ${source}:${rule.name}: ${message}`); + } +} + +export function validateRulePack(pack: unknown): { valid: boolean; errors: string[] } { + const errors: string[] = []; + if (!pack || typeof pack !== "object") { + return { valid: false, errors: ["Rule pack must be an object"] }; + } + + const value = pack as Partial; + if (typeof value.language !== "string" || !value.language.trim()) { + errors.push("language must be a non-empty string"); + } + if (typeof value.category !== "string" || !value.category.trim()) { + errors.push("category must be a non-empty string"); + } + if (!Array.isArray(value.rules)) { + errors.push("rules must be an array"); + } else { + value.rules.forEach((rule, index) => { + if (!rule || typeof rule !== "object") { + errors.push(`rules[${index}] must be an object`); + return; + } + const entry = rule as Partial; + if (typeof entry.name !== "string" || !entry.name.trim()) { + errors.push(`rules[${index}].name must be a non-empty string`); + } + if (typeof entry.pattern !== "string" || !entry.pattern.trim()) { + errors.push(`rules[${index}].pattern must be a non-empty string`); + } else { + try { + new RegExp(entry.pattern, "gi"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + errors.push(`rules[${index}].pattern is invalid: ${message}`); + } + } + if (typeof entry.replacement !== "string") { + errors.push(`rules[${index}].replacement must be a string`); + } + if (entry.context !== undefined && !VALID_CONTEXTS.has(entry.context)) { + errors.push(`rules[${index}].context is invalid`); + } + if (entry.category !== undefined && !VALID_CATEGORIES.has(entry.category)) { + errors.push(`rules[${index}].category is invalid`); + } + if (entry.minIntensity !== undefined && !VALID_INTENSITIES.has(entry.minIntensity)) { + errors.push(`rules[${index}].minIntensity is invalid`); + } + }); + } + return { valid: errors.length === 0, errors }; +} + +function readPack(language: string, category: string): RulePack | null { + const filename = path.join(getRulesDir(), language, `${category}.json`); + if (!fs.existsSync(filename)) return null; + const parsed = JSON.parse(fs.readFileSync(filename, "utf8")) as unknown; + const validation = validateRulePack(parsed); + if (!validation.valid) { + throw new Error( + `Invalid Caveman rule pack ${language}/${category}: ${validation.errors.join("; ")}` + ); + } + return parsed as RulePack; +} + +export function loadRulePack( + language: string, + category: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + const key = `${getRulesDir()}:${language}:${category}`; + if (cache.has(key) && !options.refresh) return cache.get(key) ?? []; + + const pack = readPack(language, category); + if (!pack) { + cache.set(key, []); + return []; + } + + const rules = pack.rules.map((rule) => compileRule(rule, `${language}/${category}`)); + cache.set(key, rules); + return rules; +} + +export function loadAllRulesForLanguage( + language: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + const key = `${getRulesDir()}:${language}:*`; + if (cache.has(key) && !options.refresh) return cache.get(key) ?? []; + + const languageDir = path.join(getRulesDir(), language); + if (!fs.existsSync(languageDir)) { + cache.set(key, []); + return []; + } + + const rules = fs + .readdirSync(languageDir) + .filter((entry) => entry.endsWith(".json")) + .sort() + .flatMap((entry) => loadRulePack(language, path.basename(entry, ".json"), options)); + + cache.set(key, rules); + return rules; +} + +export function getAvailableLanguagePacks(): RulePackMetadata[] { + const root = getRulesDir(); + if (!fs.existsSync(root)) return []; + + return fs + .readdirSync(root) + .filter((entry) => fs.statSync(path.join(root, entry)).isDirectory()) + .map((language) => { + const categories = fs + .readdirSync(path.join(root, language)) + .filter((entry) => entry.endsWith(".json")) + .map((entry) => path.basename(entry, ".json")) + .sort(); + const ruleCount = categories.reduce( + (count, category) => count + loadRulePack(language, category).length, + 0 + ); + return { language, categories, ruleCount }; + }) + .sort((a, b) => a.language.localeCompare(b.language)); +} + +export function loadCavemanFileRules( + language: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + return loadAllRulesForLanguage(language, options); +} + +export function listCavemanRulePacks(): RulePackMetadata[] { + return getAvailableLanguagePacks(); +} diff --git a/open-sse/services/compression/rules/_schema.json b/open-sse/services/compression/rules/_schema.json new file mode 100644 index 0000000000..5182229389 --- /dev/null +++ b/open-sse/services/compression/rules/_schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OmniRoute Caveman Rule Pack", + "type": "object", + "required": ["language", "category", "rules"], + "properties": { + "language": { "type": "string" }, + "category": { "type": "string" }, + "rules": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "pattern", "replacement"], + "properties": { + "name": { "type": "string" }, + "pattern": { "type": "string" }, + "replacement": { "type": "string" }, + "context": { "enum": ["all", "user", "system", "assistant"] }, + "category": { "enum": ["filler", "context", "structural", "dedup", "terse", "ultra"] }, + "minIntensity": { "enum": ["lite", "full", "ultra"] }, + "description": { "type": "string" } + } + } + } + } +} diff --git a/open-sse/services/compression/rules/de/context.json b/open-sse/services/compression/rules/de/context.json new file mode 100644 index 0000000000..64173166bf --- /dev/null +++ b/open-sse/services/compression/rules/de/context.json @@ -0,0 +1,38 @@ +{ + "language": "de", + "category": "context", + "rules": [ + { + "name": "de_context_setup", + "pattern": "\\b(?:hier ist der code|unten ist der code|dies ist der code)\\b\\s*[:.]?\\s*", + "replacement": "Code:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_intent", + "pattern": "\\b(?:mein ziel ist|was ich brauche ist|ich versuche|die idee ist)\\b\\s*", + "replacement": "Ziel:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_question_directive", + "pattern": "\\b(?:kannst du erklaeren warum|kannst du erklären warum|kannst du zeigen wie)\\b\\s*", + "replacement": "Erklaere ", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_background", + "pattern": "\\b(?:wie du weisst|wie du weißt|wie bereits erwaehnt|wie bereits erwähnt)\\b[,.]?\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/de/filler.json b/open-sse/services/compression/rules/de/filler.json new file mode 100644 index 0000000000..f4f2e6eb62 --- /dev/null +++ b/open-sse/services/compression/rules/de/filler.json @@ -0,0 +1,70 @@ +{ + "language": "de", + "category": "filler", + "rules": [ + { + "name": "de_polite_framing", + "pattern": "\\b(?:bitte|kannst du|koenntest du|könntest du|ich moechte|ich möchte|ich brauche)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_pleasantries", + "pattern": "\\b(?:hallo|guten morgen|guten tag|danke|vielen dank)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_hedging", + "pattern": "\\b(?:ich denke dass|ich glaube dass|es scheint dass|vielleicht|wahrscheinlich|moeglicherweise|möglicherweise)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_filler_adverbs", + "pattern": "\\b(?:grundsaetzlich|grundsätzlich|eigentlich|buchstaeblich|buchstäblich|einfach)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_self_reference", + "pattern": "^(?:ich versuche|ich will|ich moechte|ich möchte|ich brauche)\\b\\s*", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_verbose_request", + "pattern": "\\b(?:kannst du erklaeren|kannst du erklären|koenntest du erklaeren|könntest du erklären)\\b\\s*", + "replacement": "Erklaere ", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_qualifiers", + "pattern": "\\b(?:ein bisschen|etwas|irgendwie|mehr oder weniger)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_softeners", + "pattern": "\\b(?:wenn moeglich|wenn möglich|wenn du zeit hast|ohne eile)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/de/structural.json b/open-sse/services/compression/rules/de/structural.json new file mode 100644 index 0000000000..7107c977b3 --- /dev/null +++ b/open-sse/services/compression/rules/de/structural.json @@ -0,0 +1,30 @@ +{ + "language": "de", + "category": "structural", + "rules": [ + { + "name": "de_purpose", + "pattern": "\\b(?:um zu|mit dem ziel|damit ich)\\b\\s*", + "replacement": "um ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "de_connectors", + "pattern": "\\b(?:ausserdem|außerdem|zusaetzlich|zusätzlich|darueber hinaus|darüber hinaus)\\b\\s*", + "replacement": "auch ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "de_emphasis", + "pattern": "\\b(?:sehr|wirklich|extrem|ziemlich)\\s+(?=[a-zäöüß])", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/context.json b/open-sse/services/compression/rules/en/context.json new file mode 100644 index 0000000000..9f0964692e --- /dev/null +++ b/open-sse/services/compression/rules/en/context.json @@ -0,0 +1,70 @@ +{ + "language": "en", + "category": "context", + "rules": [ + { + "name": "compound_collapse", + "pattern": "\\band any potential\\b", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "full" + }, + { + "name": "explanatory_prefix", + "pattern": "\\b(?:The function appears to be handling|The code seems to|The class is|This module is)\\b", + "replacement": "Function:", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "question_to_directive", + "pattern": "\\b(?:Can you explain why|Could you show me how|Would you tell me|Can you tell me)\\b\\s*", + "replacement": "Explain why", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "context_setup", + "pattern": "\\b(?:I have the following code|Here is my code|Below is the code)\\b\\s*[:.]?\\s*", + "replacement": "Code:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "intent_clarification", + "pattern": "\\b(?:What I'm trying to do is|My objective is to|What I need is|I'm aiming to)\\b\\s*", + "replacement": "Goal:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "background_removal", + "pattern": "\\b(?:As you may know,?\\s*|As we discussed earlier,?\\s*)", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "meta_commentary", + "pattern": "^(?:Note that|Keep in mind that|Remember that)\\b\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "purpose_statement", + "pattern": "\\b(?:for the purpose of|with the goal of|in an effort to|for every)\\b", + "replacement": "for", + "context": "all", + "category": "context", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/dedup.json b/open-sse/services/compression/rules/en/dedup.json new file mode 100644 index 0000000000..23f0820bfe --- /dev/null +++ b/open-sse/services/compression/rules/en/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "en", + "category": "dedup", + "rules": [ + { + "name": "repeated_context", + "pattern": "\\b(?:As we discussed earlier|As mentioned before|As previously stated|As I said before)\\b[,.]?\\s*", + "replacement": "See above. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "repeated_question", + "pattern": "\\b(?:Same question as before|I asked this earlier|This is the same question)\\b[,.]?\\s*", + "replacement": "[same question] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "reestablished_context", + "pattern": "\\b(?:Going back to the code above|Referring back to|Returning to)\\b\\s*", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "summary_replacement", + "pattern": "\\b(?:To summarize what we've discussed|In summary of our conversation|To recap)\\b[,.]?\\s*", + "replacement": "Summary: ", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/filler.json b/open-sse/services/compression/rules/en/filler.json new file mode 100644 index 0000000000..975d7c324c --- /dev/null +++ b/open-sse/services/compression/rules/en/filler.json @@ -0,0 +1,118 @@ +{ + "language": "en", + "category": "filler", + "rules": [ + { + "name": "pleasantries", + "pattern": "(?, + pipeline?: Array, + options?: { + model?: string; + supportsVision?: boolean | null; + config?: CompressionConfig; + compressionComboId?: string | null; + } +): CompressionResult { + const steps = + pipeline && pipeline.length > 0 + ? pipeline.map(normalizePipelineStep) + : [ + { engine: "rtk" as const, intensity: "standard" as const }, + { engine: "caveman" as const, intensity: "full" as const }, + ]; + registerBuiltinCompressionEngines(); + + let currentBody = body; + let compressed = false; + const techniques = new Set(); + const rules = new Set(); + const breakdown: NonNullable = []; + const rtkRawOutputPointers: NonNullable = []; + const start = performance.now(); + + for (const step of steps) { + const engine = getCompressionEngine(step.engine); + if (!engine) continue; + const result = engine.apply(currentBody, { + ...options, + compressionComboId: options?.compressionComboId ?? options?.config?.compressionComboId, + stepConfig: { + ...(step.config ?? {}), + ...(step.intensity ? { intensity: step.intensity } : {}), + }, + }); + if (result.stats) { + result.stats.techniquesUsed.forEach((technique) => techniques.add(technique)); + result.stats.rulesApplied?.forEach((rule) => rules.add(rule)); + result.stats.rtkRawOutputPointers?.forEach((pointer) => { + rtkRawOutputPointers.push(pointer); + }); + breakdown.push({ + engine: step.engine, + originalTokens: result.stats.originalTokens, + compressedTokens: result.stats.compressedTokens, + savingsPercent: result.stats.savingsPercent, + techniquesUsed: result.stats.techniquesUsed, + ...(result.stats.rulesApplied ? { rulesApplied: result.stats.rulesApplied } : {}), + ...(result.stats.durationMs !== undefined ? { durationMs: result.stats.durationMs } : {}), + }); + } + if (result.compressed) { + currentBody = result.body; + compressed = true; + } + } + + const stats = createCompressionStats( + body, + currentBody, + "stacked", + Array.from(techniques), + rules.size > 0 ? Array.from(rules) : undefined, + Math.round((performance.now() - start) * 100) / 100 + ); + stats.engine = "stacked"; + stats.compressionComboId = + options?.compressionComboId ?? options?.config?.compressionComboId ?? null; + stats.engineBreakdown = breakdown; + if (rtkRawOutputPointers.length > 0) { + const seenPointers = new Set(); + stats.rtkRawOutputPointers = rtkRawOutputPointers.filter((pointer) => { + if (seenPointers.has(pointer.id)) return false; + seenPointers.add(pointer.id); + return true; + }); + } + + return { + body: currentBody, + compressed, + stats, + }; +} diff --git a/open-sse/services/compression/types.ts b/open-sse/services/compression/types.ts index 443cc93fdf..abe842fb35 100644 --- a/open-sse/services/compression/types.ts +++ b/open-sse/services/compression/types.ts @@ -1,15 +1,26 @@ /** - * Compression Pipeline Types — Phase 1 (Lite) + Phase 2 (Standard/Caveman) + Phase 3 (Aggressive) + Phase 4 (Ultra) + * Compression Pipeline Types — Lite, Caveman, Aggressive, Ultra, RTK, and Stacked modes. * * Shared type definitions for the compression pipeline. * Phase 1: 'off' and 'lite' modes. * Phase 2: 'standard' mode (caveman engine). * Phase 3: 'aggressive' mode (summarization + tool compression + aging). * Phase 4: 'ultra' mode (heuristic token pruning + optional SLM tier). + * Phase 5: 'rtk' and 'stacked' modes (tool-output filters + multi-engine pipeline). */ -export type CompressionMode = "off" | "lite" | "standard" | "aggressive" | "ultra"; +export type CompressionMode = + | "off" + | "lite" + | "standard" + | "aggressive" + | "ultra" + | "rtk" + | "stacked"; export type CavemanIntensity = "lite" | "full" | "ultra"; +export type RtkIntensity = "minimal" | "standard" | "aggressive"; +export type RtkRawOutputRetention = "never" | "failures" | "always"; +export type CompressionEngineId = "lite" | "caveman" | "aggressive" | "ultra" | "rtk"; export interface CavemanRule { name: string; @@ -29,6 +40,9 @@ export interface CavemanConfig { minMessageLength: number; preservePatterns: string[]; intensity: CavemanIntensity; + language?: string; + autoDetectLanguage?: boolean; + enabledLanguagePacks?: string[]; } export interface CavemanOutputModeConfig { @@ -37,6 +51,36 @@ export interface CavemanOutputModeConfig { autoClarity: boolean; } +export interface RtkConfig { + enabled: boolean; + intensity: RtkIntensity; + applyToToolResults: boolean; + applyToCodeBlocks: boolean; + applyToAssistantMessages: boolean; + enabledFilters: string[]; + disabledFilters: string[]; + maxLinesPerResult: number; + maxCharsPerResult: number; + deduplicateThreshold: number; + customFiltersEnabled: boolean; + trustProjectFilters: boolean; + rawOutputRetention: RtkRawOutputRetention; + rawOutputMaxBytes: number; +} + +export interface CompressionLanguageConfig { + enabled: boolean; + defaultLanguage: string; + autoDetect: boolean; + enabledPacks: string[]; +} + +export interface CompressionPipelineStep { + engine: CompressionEngineId; + intensity?: CavemanIntensity | RtkIntensity; + config?: Record; +} + export interface CompressionConfig { enabled: boolean; defaultMode: CompressionMode; @@ -46,8 +90,12 @@ export interface CompressionConfig { preserveSystemPrompt: boolean; mcpDescriptionCompressionEnabled?: boolean; comboOverrides: Record; + compressionComboId?: string | null; + stackedPipeline?: CompressionPipelineStep[]; cavemanConfig?: CavemanConfig; cavemanOutputMode?: CavemanOutputModeConfig; + rtkConfig?: RtkConfig; + languageConfig?: CompressionLanguageConfig; aggressive?: AggressiveConfig; ultra?: UltraConfig; } @@ -58,6 +106,8 @@ export interface CompressionStats { savingsPercent: number; techniquesUsed: string[]; mode: CompressionMode; + engine?: string; + compressionComboId?: string | null; timestamp: number; rulesApplied?: string[]; durationMs?: number; @@ -65,6 +115,13 @@ export interface CompressionStats { validationErrors?: string[]; fallbackApplied?: boolean; preservedBlockCount?: number; + rtkRawOutputPointers?: Array<{ + id: string; + path: string; + bytes: number; + sha256: string; + redacted: boolean; + }>; realUsage?: { promptTokens?: number; completionTokens?: number; @@ -76,6 +133,15 @@ export interface CompressionStats { toolResultSavings: number; agingSavings: number; }; + engineBreakdown?: Array<{ + engine: string; + originalTokens: number; + compressedTokens: number; + savingsPercent: number; + techniquesUsed: string[]; + rulesApplied?: string[]; + durationMs?: number; + }>; } export interface CompressionResult { @@ -93,6 +159,11 @@ export const DEFAULT_COMPRESSION_CONFIG: CompressionConfig = { preserveSystemPrompt: true, mcpDescriptionCompressionEnabled: true, comboOverrides: {}, + compressionComboId: null, + stackedPipeline: [ + { engine: "rtk", intensity: "standard" }, + { engine: "caveman", intensity: "full" }, + ], }; export const DEFAULT_CAVEMAN_CONFIG: CavemanConfig = { @@ -110,6 +181,30 @@ export const DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG: CavemanOutputModeConfig = { autoClarity: true, }; +export const DEFAULT_RTK_CONFIG: RtkConfig = { + enabled: true, + intensity: "standard", + applyToToolResults: true, + applyToCodeBlocks: false, + applyToAssistantMessages: false, + enabledFilters: [], + disabledFilters: [], + maxLinesPerResult: 120, + maxCharsPerResult: 12000, + deduplicateThreshold: 3, + customFiltersEnabled: true, + trustProjectFilters: false, + rawOutputRetention: "never", + rawOutputMaxBytes: 1_048_576, +}; + +export const DEFAULT_COMPRESSION_LANGUAGE_CONFIG: CompressionLanguageConfig = { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], +}; + /** Aging thresholds for progressive message degradation (Phase 3) */ export interface AgingThresholds { fullSummary: number; diff --git a/package-lock.json b/package-lock.json index c9be8e0b17..8753eccd38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,6 +79,7 @@ "husky": "^9.1.7", "jsdom": "^29.0.1", "lint-staged": "^16.2.7", + "node-loader": "^2.1.0", "prettier": "^3.8.1", "tailwindcss": "^4", "typescript": "^6.0.2", @@ -5569,6 +5570,16 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", @@ -6773,6 +6784,16 @@ "dev": true, "license": "MIT" }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -10080,6 +10101,21 @@ "dev": true, "license": "MIT" }, + "node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -11260,6 +11296,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/node-loader": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-loader/-/node-loader-2.1.0.tgz", + "integrity": "sha512-OwjPkyh8+7jW8DMd/iq71uU1Sspufr/C2+c3t0p08J3CrM9ApZ4U53xuisNrDXOHyGi5OYHgtfmmh+aK9zJA6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.3" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, "node_modules/node-machine-id": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/node-machine-id/-/node-machine-id-1.1.12.tgz", diff --git a/package.json b/package.json index 9e6f28ddce..24568eb1fb 100644 --- a/package.json +++ b/package.json @@ -174,6 +174,7 @@ "husky": "^9.1.7", "jsdom": "^29.0.1", "lint-staged": "^16.2.7", + "node-loader": "^2.1.0", "prettier": "^3.8.1", "tailwindcss": "^4", "typescript": "^6.0.2", diff --git a/scripts/pack-artifact-policy.ts b/scripts/pack-artifact-policy.ts index b2e55c0232..06793c10c6 100644 --- a/scripts/pack-artifact-policy.ts +++ b/scripts/pack-artifact-policy.ts @@ -42,6 +42,8 @@ export const APP_STAGING_ALLOWED_PATH_PREFIXES: string[] = [ ".next/", "data/", "node_modules/", + "open-sse/services/compression/engines/rtk/filters/", + "open-sse/services/compression/rules/", "public/", "src/lib/db/migrations/", "src/mitm/", @@ -88,6 +90,8 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_PATH_PREFIXES: string[] = [ ]; export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [ + "app/open-sse/services/compression/engines/rtk/filters/generic-output.json", + "app/open-sse/services/compression/rules/en/filler.json", "app/server.js", "app/server-ws.mjs", "app/responses-ws-proxy.mjs", diff --git a/scripts/prepublish.ts b/scripts/prepublish.ts index 199c76c68d..8b54f60067 100644 --- a/scripts/prepublish.ts +++ b/scripts/prepublish.ts @@ -475,6 +475,23 @@ if (existsSync(migrationsSrc)) { cpSync(migrationsSrc, migrationsDest, { recursive: true, force: true }); } +const runtimeAssetDirs = [ + { + source: join(ROOT, "open-sse", "services", "compression", "engines", "rtk", "filters"), + destination: join(APP_DIR, "open-sse", "services", "compression", "engines", "rtk", "filters"), + }, + { + source: join(ROOT, "open-sse", "services", "compression", "rules"), + destination: join(APP_DIR, "open-sse", "services", "compression", "rules"), + }, +]; +for (const assetDir of runtimeAssetDirs) { + if (existsSync(assetDir.source)) { + mkdirSync(dirname(assetDir.destination), { recursive: true }); + cpSync(assetDir.source, assetDir.destination, { recursive: true, force: true }); + } +} + // ── Step 10: Ensure data/ directory exists ────────────────── mkdirSync(join(APP_DIR, "data"), { recursive: true }); diff --git a/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx b/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx index c0d08da196..6c84c6b077 100644 --- a/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx +++ b/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx @@ -358,8 +358,8 @@ export default function CompressionAnalyticsTab() { info Compression analytics: Token savings tracked per mode (off, lite, - standard, aggressive, ultra) and provider. Hover over charts for details. Use the time - selector to view different time periods. + standard, aggressive, ultra, RTK, stacked), engine, compression combo, and provider. Hover + over charts for details. Use the time selector to view different time periods. diff --git a/src/app/(dashboard)/dashboard/compression/page.tsx b/src/app/(dashboard)/dashboard/compression/page.tsx index 7f4c6363b6..a347872010 100644 --- a/src/app/(dashboard)/dashboard/compression/page.tsx +++ b/src/app/(dashboard)/dashboard/compression/page.tsx @@ -1,20 +1,5 @@ -"use client"; - -import CompressionSettingsTab from "@/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab"; +import { redirect } from "next/navigation"; export default function CompressionPage() { - return ( -
-
-

- compress - Compression -

-

- Configure context compression settings to reduce token usage and costs. -

-
- -
- ); + redirect("/dashboard/context/caveman"); } diff --git a/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx b/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx new file mode 100644 index 0000000000..cfac49fa4c --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx @@ -0,0 +1,262 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useTranslations } from "next-intl"; +import CompressionSettingsTab from "@/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab"; + +type AnalyticsSummary = { + totalRequests: number; + totalTokensSaved: number; + avgSavingsPct: number; + avgDurationMs: number; + byEngine?: Record; + byMode?: Record; + last24h?: Array<{ hour: string; count: number; tokensSaved: number }>; +}; + +type LanguageConfig = { + enabled: boolean; + defaultLanguage: string; + autoDetect: boolean; + enabledPacks: string[]; +}; + +type OutputModeConfig = { + enabled: boolean; + intensity: "lite" | "full" | "ultra"; + autoClarity: boolean; +}; + +type CompressionSettings = { + languageConfig?: LanguageConfig; + cavemanOutputMode?: OutputModeConfig; +}; + +type LanguagePack = { language: string; ruleCount: number; categories?: string[] }; + +function formatNumber(value: number | undefined): string { + return new Intl.NumberFormat().format(value ?? 0); +} + +export default function CavemanContextPageClient() { + const t = useTranslations("contextCaveman"); + const [analytics, setAnalytics] = useState(null); + const [settings, setSettings] = useState(null); + const [languagePacks, setLanguagePacks] = useState([]); + const [saving, setSaving] = useState(false); + + const refreshSettings = () => { + fetch("/api/context/caveman/config") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setSettings(data)) + .catch(() => setSettings(null)); + }; + + useEffect(() => { + fetch("/api/context/analytics?since=7d") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setAnalytics(data)) + .catch(() => setAnalytics(null)); + fetch("/api/compression/language-packs") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setLanguagePacks(Array.isArray(data?.packs) ? data.packs : [])) + .catch(() => setLanguagePacks([])); + refreshSettings(); + }, []); + + const languageConfig: LanguageConfig = settings?.languageConfig ?? { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], + }; + const outputMode: OutputModeConfig = settings?.cavemanOutputMode ?? { + enabled: false, + intensity: "full", + autoClarity: true, + }; + + const saveSettings = async (patch: Partial) => { + setSaving(true); + try { + const res = await fetch("/api/context/caveman/config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }); + if (res.ok) setSettings(await res.json()); + } finally { + setSaving(false); + } + }; + + const updateLanguageConfig = (patch: Partial) => { + saveSettings({ languageConfig: { ...languageConfig, ...patch } }); + }; + + const updateOutputMode = (patch: Partial) => { + saveSettings({ cavemanOutputMode: { ...outputMode, ...patch } }); + }; + + const togglePack = (language: string, enabled: boolean) => { + const enabledPacks = enabled + ? [...new Set([...languageConfig.enabledPacks, language])] + : languageConfig.enabledPacks.filter((pack) => pack !== language && pack !== "en"); + updateLanguageConfig({ enabledPacks }); + }; + + const cavemanStats = analytics?.byEngine?.caveman ?? analytics?.byEngine?.standard; + const modeBreakdown = useMemo(() => Object.entries(analytics?.byMode ?? {}), [analytics]); + const statCards = [ + [t("requests"), formatNumber(cavemanStats?.count ?? analytics?.totalRequests)], + [t("tokensSaved"), formatNumber(cavemanStats?.tokensSaved ?? analytics?.totalTokensSaved)], + [t("savingsPercent"), `${cavemanStats?.avgSavingsPct ?? analytics?.avgSavingsPct ?? 0}%`], + [t("avgLatency"), `${analytics?.avgDurationMs ?? 0}ms`], + ]; + const previewPrompt = `[OmniRoute Caveman Output Mode]\n${t(`preview.${outputMode.intensity}`)}`; + + return ( +
+
+
+ compress +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+ {statCards.map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))} +
+ +
+
+

{t("languagePacks")}

+

{t("languagePacksDesc")}

+
+
+ + + +
+
+ {languagePacks.map((pack) => ( + + ))} +
+
+ +
+
+

{t("analyticsTitle")}

+
+ {modeBreakdown.length === 0 ? ( +

{t("noAnalytics")}

+ ) : ( + modeBreakdown.map(([mode, stats]) => ( +
+ {mode} + + {formatNumber(stats.tokensSaved)} / {stats.avgSavingsPct}% + +
+ )) + )} +
+
+ +
+

{t("outputModeTitle")}

+

{t("outputModeDesc")}

+
+ + + +
+ 
+            {previewPrompt}
+
+

+ {t("bypassConditions")}: security, irreversible, clarification, order-sensitive +

+
+
+ + +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/context/caveman/page.tsx b/src/app/(dashboard)/dashboard/context/caveman/page.tsx new file mode 100644 index 0000000000..bfeae4c1cb --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/caveman/page.tsx @@ -0,0 +1,5 @@ +import CavemanContextPageClient from "./CavemanContextPageClient"; + +export default function CavemanContextPage() { + return ; +} diff --git a/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx b/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx new file mode 100644 index 0000000000..58149a9aa6 --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx @@ -0,0 +1,391 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useTranslations } from "next-intl"; + +type PipelineStep = { engine: string; intensity?: string }; +type CompressionCombo = { + id: string; + name: string; + description: string; + pipeline: PipelineStep[]; + languagePacks: string[]; + outputMode: boolean; + outputModeIntensity: string; + isDefault: boolean; +}; +type RoutingCombo = { id?: string; name?: string }; +type LanguagePack = { language: string; ruleCount: number }; + +const EMPTY_PIPELINE: PipelineStep[] = [ + { engine: "rtk", intensity: "standard" }, + { engine: "caveman", intensity: "full" }, +]; + +const ENGINE_INTENSITIES: Record = { + rtk: ["minimal", "standard", "aggressive"], + caveman: ["lite", "full", "ultra"], + lite: ["lite"], + aggressive: ["standard"], + ultra: ["ultra"], +}; + +export default function CompressionCombosPageClient() { + const t = useTranslations("contextCombos"); + const [combos, setCombos] = useState([]); + const [routingCombos, setRoutingCombos] = useState([]); + const [languagePacks, setLanguagePacks] = useState([]); + const [editingId, setEditingId] = useState(null); + const [name, setName] = useState(""); + const [description, setDescription] = useState(""); + const [pipeline, setPipeline] = useState(EMPTY_PIPELINE); + const [selectedPacks, setSelectedPacks] = useState(["en"]); + const [outputMode, setOutputMode] = useState(false); + const [outputModeIntensity, setOutputModeIntensity] = useState("full"); + const [assignmentIds, setAssignmentIds] = useState([]); + const [saving, setSaving] = useState(false); + + const refresh = () => { + fetch("/api/context/combos") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setCombos(Array.isArray(data?.combos) ? data.combos : [])) + .catch(() => {}); + }; + + useEffect(() => { + refresh(); + fetch("/api/combos") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setRoutingCombos(Array.isArray(data?.combos) ? data.combos : [])) + .catch(() => {}); + fetch("/api/compression/language-packs") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setLanguagePacks(Array.isArray(data?.packs) ? data.packs : [])) + .catch(() => {}); + }, []); + + const resetForm = () => { + setEditingId(null); + setName(""); + setDescription(""); + setPipeline(EMPTY_PIPELINE); + setSelectedPacks(["en"]); + setOutputMode(false); + setOutputModeIntensity("full"); + setAssignmentIds([]); + }; + + const loadAssignments = async (id: string) => { + const res = await fetch(`/api/context/combos/${id}/assignments`); + if (!res.ok) return []; + const data = await res.json(); + return Array.isArray(data?.assignments) + ? data.assignments.map((item: { routingComboId: string }) => item.routingComboId) + : []; + }; + + const editCombo = async (combo: CompressionCombo) => { + setEditingId(combo.id); + setName(combo.name); + setDescription(combo.description ?? ""); + setPipeline(combo.pipeline.length > 0 ? combo.pipeline : EMPTY_PIPELINE); + setSelectedPacks(combo.languagePacks?.length ? combo.languagePacks : ["en"]); + setOutputMode(Boolean(combo.outputMode)); + setOutputModeIntensity(combo.outputModeIntensity ?? "full"); + setAssignmentIds(await loadAssignments(combo.id)); + }; + + const saveCombo = async () => { + const trimmed = name.trim(); + if (!trimmed) return; + setSaving(true); + try { + const payload = { + name: trimmed, + description, + pipeline, + languagePacks: selectedPacks, + outputMode, + outputModeIntensity, + }; + const res = await fetch( + editingId ? `/api/context/combos/${editingId}` : "/api/context/combos", + { + method: editingId ? "PUT" : "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + } + ); + if (!res.ok) return; + const combo = await res.json(); + await fetch(`/api/context/combos/${combo.id}/assignments`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ routingComboIds: assignmentIds }), + }); + resetForm(); + refresh(); + } finally { + setSaving(false); + } + }; + + const deleteCombo = async (combo: CompressionCombo) => { + if (!confirm(t("deleteConfirm"))) return; + const res = await fetch(`/api/context/combos/${combo.id}`, { method: "DELETE" }); + if (res.ok) refresh(); + }; + + const setDefault = async (id: string) => { + const res = await fetch(`/api/context/combos/${id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ isDefault: true }), + }); + if (res.ok) refresh(); + }; + + const updateStep = (index: number, patch: Partial) => { + setPipeline((current) => + current.map((step, stepIndex) => { + if (stepIndex !== index) return step; + const next = { ...step, ...patch }; + const allowed = ENGINE_INTENSITIES[next.engine] ?? ["standard"]; + return { + ...next, + intensity: allowed.includes(next.intensity ?? "") ? next.intensity : allowed[0], + }; + }) + ); + }; + + const togglePack = (language: string, enabled: boolean) => { + setSelectedPacks((current) => + enabled + ? [...new Set([...current, language])] + : current.filter((item) => item !== language && item !== "en") + ); + }; + + const toggleAssignment = (id: string, enabled: boolean) => { + setAssignmentIds((current) => + enabled ? [...new Set([...current, id])] : current.filter((item) => item !== id) + ); + }; + + return ( +
+
+
+ hub +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+
+ setName(event.target.value)} + placeholder={t("name")} + className="rounded-lg border border-border bg-bg px-3 py-2 text-sm text-text-main" + /> + setDescription(event.target.value)} + placeholder={t("descriptionField")} + className="rounded-lg border border-border bg-bg px-3 py-2 text-sm text-text-main" + /> +
+ +
+
+

{t("pipeline")}

+ +
+ {pipeline.map((step, index) => ( +
+ + + +
+ ))} +
+ +
+
+

{t("languagePacks")}

+
+ {languagePacks.map((pack) => ( + + ))} +
+
+
+

{t("outputMode")}

+ + +
+
+

{t("assignToRouting")}

+
+ {routingCombos.length === 0 ? ( +

{t("noAssignments")}

+ ) : ( + routingCombos.map((combo) => { + const id = combo.id ?? combo.name ?? ""; + if (!id) return null; + return ( + + ); + }) + )} +
+
+
+ +
+ + {editingId && ( + + )} +
+
+ +
+ {combos.map((combo) => ( +
+
+
+

{combo.name}

+

{combo.description}

+
+ {combo.isDefault && ( + + {t("default")} + + )} +
+
+ {combo.pipeline.map((step, index) => ( + + {index + 1}. {step.engine} + {step.intensity ? `:${step.intensity}` : ""} + + ))} +
+

+ {t("languagePacks")}: {combo.languagePacks.join(", ")} +

+
+ + {!combo.isDefault && ( + + )} + {!combo.isDefault && ( + + )} +
+
+ ))} +
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/context/combos/page.tsx b/src/app/(dashboard)/dashboard/context/combos/page.tsx new file mode 100644 index 0000000000..934ab7d451 --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/combos/page.tsx @@ -0,0 +1,5 @@ +import CompressionCombosPageClient from "./CompressionCombosPageClient"; + +export default function CompressionCombosPage() { + return ; +} diff --git a/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx new file mode 100644 index 0000000000..a7b148be8e --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx @@ -0,0 +1,299 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useTranslations } from "next-intl"; + +type RtkFilter = { + id: string; + name: string; + description: string; + commandTypes: string[]; + category: string; + priority: number; +}; + +type RtkConfig = { + enabled: boolean; + intensity: "minimal" | "standard" | "aggressive"; + applyToToolResults: boolean; + applyToAssistantMessages: boolean; + applyToCodeBlocks: boolean; + enabledFilters: string[]; + disabledFilters: string[]; + maxLinesPerResult: number; + maxCharsPerResult: number; + deduplicateThreshold: number; +}; + +type AnalyticsSummary = { + totalRequests: number; + totalTokensSaved: number; + avgSavingsPct: number; + byEngine?: Record; +}; + +type PreviewResult = { + text?: string; + compressed?: boolean; + originalTokens?: number; + compressedTokens?: number; + techniquesUsed?: string[]; + detection?: { type: string; confidence: number; category: string }; + error?: string; +}; + +const SAMPLE_OUTPUT = `$ npm run typecheck +src/lib/example.ts:10:15 - error TS2322: Type 'string' is not assignable to type 'number'. + +10 const value: number = "bad"; + ~~~~~ + +Found 1 error in src/lib/example.ts:10`; + +function formatNumber(value: number | undefined): string { + return new Intl.NumberFormat().format(value ?? 0); +} + +export default function RtkContextPageClient() { + const t = useTranslations("contextRtk"); + const [filters, setFilters] = useState([]); + const [config, setConfig] = useState(null); + const [analytics, setAnalytics] = useState(null); + const [sample, setSample] = useState(SAMPLE_OUTPUT); + const [preview, setPreview] = useState(null); + const [saving, setSaving] = useState(false); + + useEffect(() => { + fetch("/api/context/rtk/filters") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setFilters(Array.isArray(data?.filters) ? data.filters : [])) + .catch(() => {}); + fetch("/api/context/rtk/config") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setConfig(data)) + .catch(() => {}); + fetch("/api/context/analytics?since=7d") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setAnalytics(data)) + .catch(() => {}); + }, []); + + const groupedFilters = useMemo(() => { + return filters.reduce>((groups, filter) => { + groups[filter.category] = [...(groups[filter.category] ?? []), filter]; + return groups; + }, {}); + }, [filters]); + + const activeFilterCount = useMemo(() => { + if (!config) return 0; + if (config.enabledFilters.length > 0) return config.enabledFilters.length; + return filters.filter((filter) => !config.disabledFilters.includes(filter.id)).length; + }, [config, filters]); + + const saveConfig = async (patch: Partial) => { + if (!config) return; + const nextConfig = { ...config, ...patch }; + setConfig(nextConfig); + setSaving(true); + try { + const res = await fetch("/api/context/rtk/config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }); + if (res.ok) setConfig(await res.json()); + } finally { + setSaving(false); + } + }; + + const toggleFilter = (filterId: string, enabled: boolean) => { + if (!config) return; + const disabledFilters = enabled + ? config.disabledFilters.filter((id) => id !== filterId) + : [...new Set([...config.disabledFilters, filterId])]; + saveConfig({ disabledFilters }); + }; + + const runPreview = async () => { + const res = await fetch("/api/context/rtk/test", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: sample, config: config ?? undefined }), + }); + setPreview(res.ok ? await res.json() : { error: await res.text() }); + }; + + const rtkStats = analytics?.byEngine?.rtk; + const statCards = [ + [t("tokensFiltered"), formatNumber(rtkStats?.tokensSaved ?? analytics?.totalTokensSaved)], + [t("filtersActive"), formatNumber(activeFilterCount)], + [t("requests"), formatNumber(rtkStats?.count ?? analytics?.totalRequests)], + [t("avgSavings"), `${rtkStats?.avgSavingsPct ?? analytics?.avgSavingsPct ?? 0}%`], + ]; + + return ( +
+
+
+ filter_alt +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+ {statCards.map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))} +
+ + {config && ( +
+
+ + + + +
+
+ {[ + ["applyToToolResults", t("toolResults")], + ["applyToAssistantMessages", t("assistantMessages")], + ["applyToCodeBlocks", t("codeBlocks")], + ].map(([key, label]) => ( + + ))} +
+
+ )} + +
+
+
+

{t("filterTesting")}

+ +
+