From 743be2985242283d61e3e03f842d67dcc5c2f76c Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sun, 3 May 2026 00:37:08 -0300 Subject: [PATCH] feat(compression): RTK compression roadmap (#1889) * chore(rtk): initialize compression roadmap branch * feat(compression): add RTK engine and compression combos Introduce RTK command-aware tool-output compression alongside stacked RTK -> Caveman pipelines for mixed prompt contexts. Add engine registration, declarative RTK filter packs, language-aware Caveman rule loading, compression combo persistence and assignments, analytics grouped by engine/combo, and new MCP/API endpoints for configuration, previews, filters, and combo management. Expose the new capabilities in the dashboard with dedicated Context & Cache pages for Caveman, RTK, and compression combos, and update docs, i18n strings, migrations, and tests to cover the expanded compression surface. * feat(compression): expand RTK DSL, filter catalog, and recovery APIs Add RTK parity features across the compression pipeline, dashboard, and management APIs. This expands the built-in filter catalog, adds trust-gated custom filter loading, inline filter verification, code stripping, smarter detection, and optional redacted raw-output retention for authenticated recovery. Also extend Caveman with file-based multilingual rule packs, localized output-mode instructions, stricter preview/config schemas, engine registry metadata, analytics fields, and broad unit test coverage for RTK, rule loading, and stacked compression behavior. * fix(auth): protect oauth routes and health reset operations Require authenticated dashboard access for OAuth endpoints that can create or import provider connections when login enforcement is enabled. Move `/api/monitoring/health` to the readonly public route list so safe methods remain public while DELETE now returns 401 for anonymous requests. Also update Next.js native `.node` handling to avoid webpack parse failures from external packages such as ngrok and keytar, and add coverage for the new auth behavior. * build(compression): ship RTK rule and filter assets with app bundles Include compression JSON assets in Next output tracing, prepublish copies, and pack artifact policy checks so standalone and packaged builds can load RTK filters and caveman rule packs at runtime. Also harden compression runtime behavior by resolving alternate asset directories, scoping rule cache entries by source path, carrying RTK raw output pointers through stacked runs, degrading oversized preview diffs, and applying combo language/output mode defaults during chat routing. Add coverage for packaging rules, provider-scoped model parsing, smart truncate edge cases, raw output retention, and combo-driven compression behavior. * docs(workflows): update local repo paths to OmniRoute Replace outdated `/home/diegosouzapw/dev/proxys/9router` references with the current `OmniRoute` directory across deploy, release, and version bump workflow guides so local command examples match the renamed repository layout * feat(compression): complete RTK parity coverage * test(build): align next config assertions --------- Co-authored-by: diegosouzapw --- .agents/workflows/deploy-vps-akamai.md | 2 +- .agents/workflows/deploy-vps-both.md | 2 +- .agents/workflows/deploy-vps-local.md | 2 +- .agents/workflows/generate-release.md | 2 +- .agents/workflows/version-bump.md | 18 +- AGENTS.md | 34 +- CHANGELOG.md | 9 + README.md | 50 +- docs/API_REFERENCE.md | 45 +- docs/ARCHITECTURE.md | 18 +- docs/COMPRESSION_GUIDE.md | 81 ++- docs/ENVIRONMENT.md | 6 + docs/FEATURES.md | 14 + docs/MCP-SERVER.md | 66 ++- docs/SETUP_GUIDE.md | 2 +- docs/compression-engines.md | 127 +++++ docs/compression-language-packs.md | 96 ++++ docs/compression-rules-format.md | 177 +++++++ docs/openapi.yaml | 198 ++++++++ docs/rtk-compression.md | 223 +++++++++ next.config.mjs | 101 +--- open-sse/config/embeddingRegistry.ts | 9 + open-sse/config/rerankRegistry.ts | 9 + open-sse/handlers/chatCore.ts | 129 ++++- open-sse/mcp-server/README.md | 21 +- open-sse/mcp-server/schemas/tools.ts | 77 ++- open-sse/mcp-server/tools/compressionTools.ts | 85 +++- open-sse/services/AGENTS.md | 7 +- open-sse/services/compression/caveman.ts | 116 +++-- open-sse/services/compression/cavemanRules.ts | 14 +- open-sse/services/compression/diffHelper.ts | 35 +- .../compression/engines/cavemanAdapter.ts | 254 ++++++++++ .../services/compression/engines/index.ts | 14 + .../services/compression/engines/registry.ts | 87 ++++ .../compression/engines/rtk/codeStripper.ts | 128 +++++ .../engines/rtk/commandDetector.ts | 465 ++++++++++++++++++ .../compression/engines/rtk/deduplicator.ts | 37 ++ .../compression/engines/rtk/filterLoader.ts | 220 +++++++++ .../compression/engines/rtk/filterSchema.ts | 182 +++++++ .../compression/engines/rtk/filters/aws.json | 34 ++ .../engines/rtk/filters/biome.json | 35 ++ .../engines/rtk/filters/build-eslint.json | 37 ++ .../engines/rtk/filters/build-typescript.json | 33 ++ .../engines/rtk/filters/build-vite.json | 41 ++ .../engines/rtk/filters/build-webpack.json | 34 ++ .../engines/rtk/filters/bundle-install.json | 35 ++ .../compression/engines/rtk/filters/curl.json | 34 ++ .../compression/engines/rtk/filters/df.json | 34 ++ .../engines/rtk/filters/docker-logs.json | 42 ++ .../engines/rtk/filters/docker-ps.json | 33 ++ .../compression/engines/rtk/filters/du.json | 33 ++ .../engines/rtk/filters/error-stacktrace.json | 41 ++ .../engines/rtk/filters/gcloud.json | 34 ++ .../engines/rtk/filters/generic-output.json | 32 ++ .../engines/rtk/filters/git-branch.json | 45 ++ .../engines/rtk/filters/git-diff.json | 33 ++ .../engines/rtk/filters/git-log.json | 33 ++ .../engines/rtk/filters/git-status.json | 40 ++ .../engines/rtk/filters/golangci-lint.json | 34 ++ .../engines/rtk/filters/json-output.json | 40 ++ .../compression/engines/rtk/filters/make.json | 38 ++ .../compression/engines/rtk/filters/mypy.json | 34 ++ .../engines/rtk/filters/npm-audit.json | 43 ++ .../engines/rtk/filters/npm-install.json | 41 ++ .../compression/engines/rtk/filters/nx.json | 34 ++ .../compression/engines/rtk/filters/pip.json | 41 ++ .../engines/rtk/filters/playwright.json | 42 ++ .../engines/rtk/filters/poetry-install.json | 35 ++ .../engines/rtk/filters/prettier.json | 40 ++ .../compression/engines/rtk/filters/ps.json | 34 ++ .../engines/rtk/filters/rsync.json | 33 ++ .../engines/rtk/filters/rubocop.json | 34 ++ .../compression/engines/rtk/filters/ruff.json | 40 ++ .../engines/rtk/filters/shell-find.json | 33 ++ .../engines/rtk/filters/shell-grep.json | 37 ++ .../engines/rtk/filters/shell-ls.json | 33 ++ .../compression/engines/rtk/filters/ssh.json | 43 ++ .../engines/rtk/filters/systemctl-status.json | 43 ++ .../engines/rtk/filters/terraform-plan.json | 34 ++ .../engines/rtk/filters/test-cargo.json | 42 ++ .../engines/rtk/filters/test-go.json | 41 ++ .../engines/rtk/filters/test-jest.json | 43 ++ .../engines/rtk/filters/test-pytest.json | 41 ++ .../engines/rtk/filters/test-vitest.json | 43 ++ .../engines/rtk/filters/tofu-plan.json | 34 ++ .../engines/rtk/filters/turbo.json | 34 ++ .../engines/rtk/filters/uv-sync.json | 35 ++ .../compression/engines/rtk/filters/wget.json | 34 ++ .../services/compression/engines/rtk/index.ts | 404 +++++++++++++++ .../compression/engines/rtk/lineFilter.ts | 155 ++++++ .../compression/engines/rtk/rawOutput.ts | 112 +++++ .../compression/engines/rtk/smartTruncate.ts | 67 +++ .../compression/engines/rtk/verify.ts | 107 ++++ .../services/compression/engines/types.ts | 59 +++ open-sse/services/compression/index.ts | 75 +++ .../services/compression/languageDetector.ts | 18 + open-sse/services/compression/outputMode.ts | 59 ++- open-sse/services/compression/ruleLoader.ts | 206 ++++++++ .../services/compression/rules/_schema.json | 26 + .../compression/rules/de/context.json | 38 ++ .../services/compression/rules/de/filler.json | 70 +++ .../compression/rules/de/structural.json | 30 ++ .../compression/rules/en/context.json | 70 +++ .../services/compression/rules/en/dedup.json | 38 ++ .../services/compression/rules/en/filler.json | 118 +++++ .../compression/rules/en/structural.json | 94 ++++ .../services/compression/rules/en/ultra.json | 102 ++++ .../compression/rules/es/context.json | 38 ++ .../services/compression/rules/es/filler.json | 70 +++ .../compression/rules/es/structural.json | 30 ++ .../compression/rules/fr/context.json | 38 ++ .../services/compression/rules/fr/filler.json | 70 +++ .../compression/rules/fr/structural.json | 30 ++ .../compression/rules/ja/context.json | 38 ++ .../services/compression/rules/ja/filler.json | 70 +++ .../compression/rules/ja/structural.json | 30 ++ .../compression/rules/pt-BR/context.json | 38 ++ .../compression/rules/pt-BR/filler.json | 70 +++ .../compression/rules/pt-BR/structural.json | 30 ++ open-sse/services/compression/stats.ts | 4 + .../services/compression/strategySelector.ts | 122 ++++- open-sse/services/compression/types.ts | 99 +++- package-lock.json | 56 +++ package.json | 1 + scripts/pack-artifact-policy.ts | 4 + scripts/prepublish.ts | 17 + .../analytics/CompressionAnalyticsTab.tsx | 4 +- .../dashboard/compression/page.tsx | 19 +- .../caveman/CavemanContextPageClient.tsx | 262 ++++++++++ .../dashboard/context/caveman/page.tsx | 5 + .../combos/CompressionCombosPageClient.tsx | 391 +++++++++++++++ .../dashboard/context/combos/page.tsx | 5 + .../context/rtk/RtkContextPageClient.tsx | 299 +++++++++++ .../dashboard/context/rtk/page.tsx | 5 + .../components/CompressionSettingsTab.tsx | 16 +- .../(dashboard)/dashboard/settings/page.tsx | 22 +- .../api/compression/language-packs/route.ts | 16 + src/app/api/compression/preview/route.ts | 65 ++- src/app/api/context/analytics/route.ts | 1 + src/app/api/context/caveman/config/route.ts | 1 + .../context/combos/[id]/assignments/route.ts | 47 ++ src/app/api/context/combos/[id]/route.ts | 81 +++ src/app/api/context/combos/route.ts | 52 ++ src/app/api/context/rtk/config/route.ts | 51 ++ src/app/api/context/rtk/filters/route.ts | 15 + .../api/context/rtk/raw-output/[id]/route.ts | 26 + src/app/api/context/rtk/test/route.ts | 42 ++ src/app/api/monitoring/health/route.ts | 7 +- .../api/oauth/[provider]/[action]/route.ts | 13 + src/app/api/oauth/cursor/import/route.ts | 17 +- src/app/api/oauth/kiro/import/route.ts | 12 +- .../api/oauth/kiro/social-authorize/route.ts | 5 + .../api/oauth/kiro/social-exchange/route.ts | 5 + src/app/api/settings/compression/route.ts | 51 +- src/i18n/messages/ar.json | 82 ++- src/i18n/messages/bg.json | 82 ++- src/i18n/messages/bn.json | 82 ++- src/i18n/messages/cs.json | 82 ++- src/i18n/messages/da.json | 82 ++- src/i18n/messages/de.json | 82 ++- src/i18n/messages/en.json | 82 ++- src/i18n/messages/es.json | 82 ++- src/i18n/messages/fa.json | 82 ++- src/i18n/messages/fi.json | 82 ++- src/i18n/messages/fr.json | 82 ++- src/i18n/messages/gu.json | 82 ++- src/i18n/messages/he.json | 82 ++- src/i18n/messages/hi.json | 82 ++- src/i18n/messages/hu.json | 82 ++- src/i18n/messages/id.json | 82 ++- src/i18n/messages/in.json | 82 ++- src/i18n/messages/it.json | 82 ++- src/i18n/messages/ja.json | 82 ++- src/i18n/messages/ko.json | 82 ++- src/i18n/messages/mr.json | 82 ++- src/i18n/messages/ms.json | 82 ++- src/i18n/messages/nl.json | 82 ++- src/i18n/messages/no.json | 82 ++- src/i18n/messages/phi.json | 82 ++- src/i18n/messages/pl.json | 82 ++- src/i18n/messages/pt-BR.json | 82 ++- src/i18n/messages/pt.json | 82 ++- src/i18n/messages/ro.json | 82 ++- src/i18n/messages/ru.json | 82 ++- src/i18n/messages/sk.json | 82 ++- src/i18n/messages/sv.json | 82 ++- src/i18n/messages/sw.json | 82 ++- src/i18n/messages/ta.json | 82 ++- src/i18n/messages/te.json | 82 ++- src/i18n/messages/th.json | 82 ++- src/i18n/messages/tr.json | 82 ++- src/i18n/messages/uk-UA.json | 82 ++- src/i18n/messages/ur.json | 82 ++- src/i18n/messages/vi.json | 82 ++- src/i18n/messages/zh-CN.json | 82 ++- src/lib/db/compression.ts | 151 ++++++ src/lib/db/compressionAnalytics.ts | 77 ++- src/lib/db/compressionCombos.ts | 368 ++++++++++++++ src/lib/db/migrationRunner.ts | 30 ++ .../db/migrations/042_compression_combos.sql | 44 ++ src/lib/localDb.ts | 1 + src/shared/constants/mcpScopes.ts | 3 + src/shared/constants/publicApiRoutes.ts | 6 +- src/shared/constants/sidebarVisibility.ts | 32 +- src/shared/validation/schemas.ts | 10 +- .../chatcore-compression-integration.test.ts | 110 ++++- tests/integration/security-hardening.test.ts | 18 + tests/unit/api-auth.test.ts | 14 + .../api/compression/compression-api.test.ts | 12 +- .../compression/compression-combos-db.test.ts | 80 +++ .../compression-preview-api.test.ts | 53 ++ .../compression/compressionAnalytics.test.ts | 2 + .../compression/compressionMcpTools.test.ts | 49 +- .../context-compression-api.test.ts | 42 ++ tests/unit/compression/diffHelper.test.ts | 9 + .../unit/compression/engine-registry.test.ts | 73 +++ .../compression/fixtures/rtk/aws-sample.txt | 4 + .../compression/fixtures/rtk/biome-sample.txt | 4 + .../fixtures/rtk/build-vite-sample.txt | 3 + .../fixtures/rtk/build-webpack-sample.txt | 3 + .../fixtures/rtk/bundle-install-sample.txt | 4 + .../compression/fixtures/rtk/curl-sample.txt | 4 + .../compression/fixtures/rtk/df-sample.txt | 4 + .../fixtures/rtk/docker-logs-sample.txt | 4 + .../compression/fixtures/rtk/du-sample.txt | 4 + .../fixtures/rtk/error-stacktrace-sample.txt | 4 + .../fixtures/rtk/eslint-output-sample.txt | 5 + .../fixtures/rtk/gcloud-sample.txt | 3 + .../fixtures/rtk/git-branch-sample.txt | 3 + .../fixtures/rtk/git-diff-sample.txt | 8 + .../fixtures/rtk/git-status-sample.txt | 5 + .../fixtures/rtk/golangci-lint-sample.txt | 5 + .../fixtures/rtk/json-output-sample.txt | 7 + .../compression/fixtures/rtk/make-sample.txt | 6 + .../compression/fixtures/rtk/mypy-sample.txt | 2 + .../fixtures/rtk/npm-audit-sample.txt | 2 + .../compression/fixtures/rtk/nx-sample.txt | 6 + .../compression/fixtures/rtk/pip-sample.txt | 4 + .../fixtures/rtk/playwright-sample.txt | 6 + .../fixtures/rtk/poetry-install-sample.txt | 5 + .../fixtures/rtk/prettier-sample.txt | 4 + .../compression/fixtures/rtk/ps-sample.txt | 4 + .../compression/fixtures/rtk/rsync-sample.txt | 5 + .../fixtures/rtk/rubocop-sample.txt | 4 + .../compression/fixtures/rtk/ruff-sample.txt | 3 + .../fixtures/rtk/shell-find-sample.txt | 3 + .../fixtures/rtk/shell-grep-sample.txt | 3 + .../compression/fixtures/rtk/ssh-sample.txt | 3 + .../fixtures/rtk/systemctl-status-sample.txt | 4 + .../fixtures/rtk/terraform-plan-sample.txt | 8 + .../fixtures/rtk/test-cargo-sample.txt | 8 + .../fixtures/rtk/test-go-sample.txt | 4 + .../fixtures/rtk/tofu-plan-sample.txt | 8 + .../compression/fixtures/rtk/turbo-sample.txt | 5 + .../fixtures/rtk/typescript-errors-sample.txt | 2 + .../fixtures/rtk/uv-sync-sample.txt | 4 + .../fixtures/rtk/vitest-output-sample.txt | 7 + .../compression/fixtures/rtk/wget-sample.txt | 4 + tests/unit/compression/language-packs.test.ts | 60 +++ .../compression/pipeline-integration.test.ts | 71 +++ .../compression/rtk-code-stripper.test.ts | 72 +++ .../compression/rtk-command-detector.test.ts | 107 ++++ .../compression/rtk-custom-filters.test.ts | 104 ++++ .../unit/compression/rtk-deduplicator.test.ts | 21 + .../unit/compression/rtk-dsl-pipeline.test.ts | 94 ++++ tests/unit/compression/rtk-engine.test.ts | 89 ++++ .../compression/rtk-filter-catalog.test.ts | 94 ++++ .../unit/compression/rtk-line-filter.test.ts | 39 ++ .../compression/rtk-raw-output-route.test.ts | 138 ++++++ tests/unit/compression/rtk-raw-output.test.ts | 131 +++++ .../compression/rtk-smart-truncate.test.ts | 43 ++ tests/unit/compression/rtk-verify.test.ts | 78 +++ tests/unit/compression/rule-loader.test.ts | 58 +++ .../unit/compression/strategySelector.test.ts | 35 ++ tests/unit/compression/types.test.ts | 12 +- ...embedding-rerank-provider-registry.test.ts | 18 + tests/unit/next-config.test.ts | 96 ++-- tests/unit/pack-artifact-policy.test.ts | 19 +- tests/unit/public-api-routes.test.ts | 5 + tests/unit/sidebar-visibility.test.ts | 24 + 280 files changed, 14881 insertions(+), 477 deletions(-) create mode 100644 docs/compression-engines.md create mode 100644 docs/compression-language-packs.md create mode 100644 docs/compression-rules-format.md create mode 100644 docs/rtk-compression.md create mode 100644 open-sse/services/compression/engines/cavemanAdapter.ts create mode 100644 open-sse/services/compression/engines/index.ts create mode 100644 open-sse/services/compression/engines/registry.ts create mode 100644 open-sse/services/compression/engines/rtk/codeStripper.ts create mode 100644 open-sse/services/compression/engines/rtk/commandDetector.ts create mode 100644 open-sse/services/compression/engines/rtk/deduplicator.ts create mode 100644 open-sse/services/compression/engines/rtk/filterLoader.ts create mode 100644 open-sse/services/compression/engines/rtk/filterSchema.ts create mode 100644 open-sse/services/compression/engines/rtk/filters/aws.json create mode 100644 open-sse/services/compression/engines/rtk/filters/biome.json create mode 100644 open-sse/services/compression/engines/rtk/filters/build-eslint.json create mode 100644 open-sse/services/compression/engines/rtk/filters/build-typescript.json create mode 100644 open-sse/services/compression/engines/rtk/filters/build-vite.json create mode 100644 open-sse/services/compression/engines/rtk/filters/build-webpack.json create mode 100644 open-sse/services/compression/engines/rtk/filters/bundle-install.json create mode 100644 open-sse/services/compression/engines/rtk/filters/curl.json create mode 100644 open-sse/services/compression/engines/rtk/filters/df.json create mode 100644 open-sse/services/compression/engines/rtk/filters/docker-logs.json create mode 100644 open-sse/services/compression/engines/rtk/filters/docker-ps.json create mode 100644 open-sse/services/compression/engines/rtk/filters/du.json create mode 100644 open-sse/services/compression/engines/rtk/filters/error-stacktrace.json create mode 100644 open-sse/services/compression/engines/rtk/filters/gcloud.json create mode 100644 open-sse/services/compression/engines/rtk/filters/generic-output.json create mode 100644 open-sse/services/compression/engines/rtk/filters/git-branch.json create mode 100644 open-sse/services/compression/engines/rtk/filters/git-diff.json create mode 100644 open-sse/services/compression/engines/rtk/filters/git-log.json create mode 100644 open-sse/services/compression/engines/rtk/filters/git-status.json create mode 100644 open-sse/services/compression/engines/rtk/filters/golangci-lint.json create mode 100644 open-sse/services/compression/engines/rtk/filters/json-output.json create mode 100644 open-sse/services/compression/engines/rtk/filters/make.json create mode 100644 open-sse/services/compression/engines/rtk/filters/mypy.json create mode 100644 open-sse/services/compression/engines/rtk/filters/npm-audit.json create mode 100644 open-sse/services/compression/engines/rtk/filters/npm-install.json create mode 100644 open-sse/services/compression/engines/rtk/filters/nx.json create mode 100644 open-sse/services/compression/engines/rtk/filters/pip.json create mode 100644 open-sse/services/compression/engines/rtk/filters/playwright.json create mode 100644 open-sse/services/compression/engines/rtk/filters/poetry-install.json create mode 100644 open-sse/services/compression/engines/rtk/filters/prettier.json create mode 100644 open-sse/services/compression/engines/rtk/filters/ps.json create mode 100644 open-sse/services/compression/engines/rtk/filters/rsync.json create mode 100644 open-sse/services/compression/engines/rtk/filters/rubocop.json create mode 100644 open-sse/services/compression/engines/rtk/filters/ruff.json create mode 100644 open-sse/services/compression/engines/rtk/filters/shell-find.json create mode 100644 open-sse/services/compression/engines/rtk/filters/shell-grep.json create mode 100644 open-sse/services/compression/engines/rtk/filters/shell-ls.json create mode 100644 open-sse/services/compression/engines/rtk/filters/ssh.json create mode 100644 open-sse/services/compression/engines/rtk/filters/systemctl-status.json create mode 100644 open-sse/services/compression/engines/rtk/filters/terraform-plan.json create mode 100644 open-sse/services/compression/engines/rtk/filters/test-cargo.json create mode 100644 open-sse/services/compression/engines/rtk/filters/test-go.json create mode 100644 open-sse/services/compression/engines/rtk/filters/test-jest.json create mode 100644 open-sse/services/compression/engines/rtk/filters/test-pytest.json create mode 100644 open-sse/services/compression/engines/rtk/filters/test-vitest.json create mode 100644 open-sse/services/compression/engines/rtk/filters/tofu-plan.json create mode 100644 open-sse/services/compression/engines/rtk/filters/turbo.json create mode 100644 open-sse/services/compression/engines/rtk/filters/uv-sync.json create mode 100644 open-sse/services/compression/engines/rtk/filters/wget.json create mode 100644 open-sse/services/compression/engines/rtk/index.ts create mode 100644 open-sse/services/compression/engines/rtk/lineFilter.ts create mode 100644 open-sse/services/compression/engines/rtk/rawOutput.ts create mode 100644 open-sse/services/compression/engines/rtk/smartTruncate.ts create mode 100644 open-sse/services/compression/engines/rtk/verify.ts create mode 100644 open-sse/services/compression/engines/types.ts create mode 100644 open-sse/services/compression/languageDetector.ts create mode 100644 open-sse/services/compression/ruleLoader.ts create mode 100644 open-sse/services/compression/rules/_schema.json create mode 100644 open-sse/services/compression/rules/de/context.json create mode 100644 open-sse/services/compression/rules/de/filler.json create mode 100644 open-sse/services/compression/rules/de/structural.json create mode 100644 open-sse/services/compression/rules/en/context.json create mode 100644 open-sse/services/compression/rules/en/dedup.json create mode 100644 open-sse/services/compression/rules/en/filler.json create mode 100644 open-sse/services/compression/rules/en/structural.json create mode 100644 open-sse/services/compression/rules/en/ultra.json create mode 100644 open-sse/services/compression/rules/es/context.json create mode 100644 open-sse/services/compression/rules/es/filler.json create mode 100644 open-sse/services/compression/rules/es/structural.json create mode 100644 open-sse/services/compression/rules/fr/context.json create mode 100644 open-sse/services/compression/rules/fr/filler.json create mode 100644 open-sse/services/compression/rules/fr/structural.json create mode 100644 open-sse/services/compression/rules/ja/context.json create mode 100644 open-sse/services/compression/rules/ja/filler.json create mode 100644 open-sse/services/compression/rules/ja/structural.json create mode 100644 open-sse/services/compression/rules/pt-BR/context.json create mode 100644 open-sse/services/compression/rules/pt-BR/filler.json create mode 100644 open-sse/services/compression/rules/pt-BR/structural.json create mode 100644 src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx create mode 100644 src/app/(dashboard)/dashboard/context/caveman/page.tsx create mode 100644 src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx create mode 100644 src/app/(dashboard)/dashboard/context/combos/page.tsx create mode 100644 src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx create mode 100644 src/app/(dashboard)/dashboard/context/rtk/page.tsx create mode 100644 src/app/api/compression/language-packs/route.ts create mode 100644 src/app/api/context/analytics/route.ts create mode 100644 src/app/api/context/caveman/config/route.ts create mode 100644 src/app/api/context/combos/[id]/assignments/route.ts create mode 100644 src/app/api/context/combos/[id]/route.ts create mode 100644 src/app/api/context/combos/route.ts create mode 100644 src/app/api/context/rtk/config/route.ts create mode 100644 src/app/api/context/rtk/filters/route.ts create mode 100644 src/app/api/context/rtk/raw-output/[id]/route.ts create mode 100644 src/app/api/context/rtk/test/route.ts create mode 100644 src/lib/db/compressionCombos.ts create mode 100644 src/lib/db/migrations/042_compression_combos.sql create mode 100644 tests/unit/compression/compression-combos-db.test.ts create mode 100644 tests/unit/compression/compression-preview-api.test.ts create mode 100644 tests/unit/compression/context-compression-api.test.ts create mode 100644 tests/unit/compression/engine-registry.test.ts create mode 100644 tests/unit/compression/fixtures/rtk/aws-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/biome-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/build-vite-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/build-webpack-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/bundle-install-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/curl-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/df-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/docker-logs-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/du-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/error-stacktrace-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/eslint-output-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/gcloud-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/git-branch-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/git-diff-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/git-status-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/golangci-lint-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/json-output-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/make-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/mypy-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/npm-audit-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/nx-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/pip-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/playwright-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/poetry-install-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/prettier-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/ps-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/rsync-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/rubocop-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/ruff-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/shell-find-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/shell-grep-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/ssh-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/systemctl-status-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/terraform-plan-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/test-cargo-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/test-go-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/tofu-plan-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/turbo-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/typescript-errors-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/uv-sync-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/vitest-output-sample.txt create mode 100644 tests/unit/compression/fixtures/rtk/wget-sample.txt create mode 100644 tests/unit/compression/language-packs.test.ts create mode 100644 tests/unit/compression/pipeline-integration.test.ts create mode 100644 tests/unit/compression/rtk-code-stripper.test.ts create mode 100644 tests/unit/compression/rtk-command-detector.test.ts create mode 100644 tests/unit/compression/rtk-custom-filters.test.ts create mode 100644 tests/unit/compression/rtk-deduplicator.test.ts create mode 100644 tests/unit/compression/rtk-dsl-pipeline.test.ts create mode 100644 tests/unit/compression/rtk-engine.test.ts create mode 100644 tests/unit/compression/rtk-filter-catalog.test.ts create mode 100644 tests/unit/compression/rtk-line-filter.test.ts create mode 100644 tests/unit/compression/rtk-raw-output-route.test.ts create mode 100644 tests/unit/compression/rtk-raw-output.test.ts create mode 100644 tests/unit/compression/rtk-smart-truncate.test.ts create mode 100644 tests/unit/compression/rtk-verify.test.ts create mode 100644 tests/unit/compression/rule-loader.test.ts diff --git a/.agents/workflows/deploy-vps-akamai.md b/.agents/workflows/deploy-vps-akamai.md index 50f92cab5f..b25ae3cd99 100644 --- a/.agents/workflows/deploy-vps-akamai.md +++ b/.agents/workflows/deploy-vps-akamai.md @@ -17,7 +17,7 @@ Deploy OmniRoute to the Akamai VPS using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to Akamai VPS and install diff --git a/.agents/workflows/deploy-vps-both.md b/.agents/workflows/deploy-vps-both.md index 161d197bd4..e1aa4d1def 100644 --- a/.agents/workflows/deploy-vps-both.md +++ b/.agents/workflows/deploy-vps-both.md @@ -22,7 +22,7 @@ Deploy OmniRoute to the production VPSs using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to both VPS and install diff --git a/.agents/workflows/deploy-vps-local.md b/.agents/workflows/deploy-vps-local.md index f348f63da5..549b1f0b2a 100644 --- a/.agents/workflows/deploy-vps-local.md +++ b/.agents/workflows/deploy-vps-local.md @@ -17,7 +17,7 @@ Deploy OmniRoute to the Local VPS using `npm pack + scp` + PM2. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts ``` ### 2. Copy to Local VPS and install diff --git a/.agents/workflows/generate-release.md b/.agents/workflows/generate-release.md index e127c832d4..399e12d214 100644 --- a/.agents/workflows/generate-release.md +++ b/.agents/workflows/generate-release.md @@ -226,7 +226,7 @@ git checkout main git pull origin main # Build and pack locally -cd /home/diegosouzapw/dev/proxys/9router && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts +cd /home/diegosouzapw/dev/proxys/OmniRoute && rm -f omniroute-*.tgz && rm -rf .next/cache app/.next/cache && npm run build:cli && rm -rf app/logs app/coverage app/.git app/.app-build-backup* && npm pack --ignore-scripts # Deploy to LOCAL VPS (192.168.0.15) scp omniroute-*.tgz root@192.168.0.15:/tmp/ diff --git a/.agents/workflows/version-bump.md b/.agents/workflows/version-bump.md index 4b3b77a921..0667e0eb1d 100644 --- a/.agents/workflows/version-bump.md +++ b/.agents/workflows/version-bump.md @@ -20,7 +20,7 @@ Automatically bump the project version, generate CHANGELOG entries from git hist // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute CURRENT_VERSION=$(node -p "require('./package.json').version") LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") CURRENT_BRANCH=$(git branch --show-current) @@ -64,7 +64,7 @@ npm version "$VERSION" --no-git-tag-version // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null) echo "=== Commits since $LAST_TAG ===" git log "$LAST_TAG"..HEAD --pretty=format:"%h %s" --no-merges | head -100 @@ -133,7 +133,7 @@ The date must be today's date in `YYYY-MM-DD` format. // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") # Update docs/openapi.yaml version @@ -156,7 +156,7 @@ echo "✓ All workspace packages synced to $VERSION" // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") OLD_VERSION_PATTERN='[0-9]\+\.[0-9]\+\.[0-9]\+' @@ -174,7 +174,7 @@ echo "✓ llm.txt → $VERSION" // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm install echo "✓ Lock file regenerated" ``` @@ -236,7 +236,7 @@ For each file in `docs/` (excluding `docs/i18n/`), review if CHANGELOG changes a // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm run lint ``` @@ -245,7 +245,7 @@ npm run lint // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute npm test ``` @@ -254,7 +254,7 @@ npm test // turbo ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute VERSION=$(node -p "require('./package.json').version") echo "Expected version: $VERSION" echo "" @@ -299,7 +299,7 @@ grep "^## \[" CHANGELOG.md | head -2 // turbo-all ```bash -cd /home/diegosouzapw/dev/proxys/9router +cd /home/diegosouzapw/dev/proxys/OmniRoute git add -A VERSION=$(node -p "require('./package.json').version") git commit -m "chore(release): bump to v$VERSION — changelog, docs, version sync" diff --git a/AGENTS.md b/AGENTS.md index 3b05a3dd65..aadaf98b2e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Unified AI proxy/router — route any LLM through one endpoint. Multi-provider s with **160+ providers** (OpenAI, Anthropic, Gemini, DeepSeek, Groq, xAI, Mistral, Fireworks, Cohere, NVIDIA, Cerebras, Pollinations, Puter, Cloudflare AI, HuggingFace, DeepInfra, SambaNova, Meta Llama API, Moonshot AI, AI21 Labs, Databricks, Snowflake, and many more) -with **MCP Server** (29 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. +with **MCP Server** (37 tools), **A2A v0.3 Protocol**, and **Electron desktop app**. ## Stack @@ -292,17 +292,28 @@ compression pipeline), and more. Modular prompt compression that runs proactively before the existing reactive context manager. -- **`strategySelector.ts`**: Selects compression mode based on config, combo overrides, auto-trigger - thresholds. Priority: combo override > auto-trigger > default mode > off. +- **`strategySelector.ts`**: Selects compression mode based on config, compression combo assignments, + combo overrides, auto-trigger thresholds, and defaults. Priority: assigned compression combo > + combo override > auto-trigger > default mode > off. - **`lite.ts`**: 5 lite-mode techniques: `collapseWhitespace`, `dedupSystemPrompt`, `compressToolResults`, `removeRedundantContent`, `replaceImageUrls`. Target: 10-15% savings at <1ms latency. +- **`caveman.ts` / `cavemanRules.ts`**: Caveman-style semantic condensation backed by built-in + rules plus file-loaded language packs under `compression/rules/`. +- **`engines/rtk/`**: Rule-based terminal/tool-output compression inspired by RTK patterns. Detects + command output classes, applies JSON filter packs, deduplicates repeated lines, strips ANSI/code + noise, and preserves errors/actionable context. The RTK JSON DSL supports replace, + match-output short-circuit, strip/keep, per-line truncation, head/tail/max-line truncation, + inline tests, trust-gated project/global custom filters, and optional redacted raw-output + retention for authenticated recovery. +- **`engines/registry.ts`**: Registers engines (`caveman`, `rtk`) and powers stacked pipelines. - **`stats.ts`**: Per-request compression stats tracking (original tokens, compressed tokens, - savings %, techniques used). -- **`types.ts`**: `CompressionMode` (off/lite/standard/aggressive/ultra), `CompressionConfig`, - `CompressionStats`, `CompressionResult`. -- DB settings in `src/lib/db/compression.ts`, API route at `src/app/api/settings/compression/`. -- Phase 1 implements lite mode only; standard/aggressive/ultra are placeholders for Phase 2. + savings %, techniques used, engine breakdown, compression combo id). +- **`types.ts`**: `CompressionMode` (off/lite/standard/aggressive/ultra/rtk/stacked), + `CompressionConfig`, `CompressionStats`, `CompressionResult`. +- DB settings in `src/lib/db/compression.ts`, compression combos in + `src/lib/db/compressionCombos.ts`, API routes under `src/app/api/settings/compression/`, + `src/app/api/context/*`, and preview/language-pack routes under `src/app/api/compression/*`. #### Combo Routing Engine (`combo.ts`) @@ -323,7 +334,7 @@ Policy engine modules: `policyEngine.ts`, `comboResolver.ts`, `costRules.ts`, ### MCP Server (`open-sse/mcp-server/`) -29 tools, 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (10 scopes), Zod schemas. +37 tools, 3 transports (stdio / SSE / Streamable HTTP). Scoped auth (10 scopes), Zod schemas. **Core tools** (20): get_health, list_combos, get_combo_metrics, switch_combo, check_quota, route_request, cost_report, list_models_catalog, web_search, simulate_route, set_budget_guard, @@ -332,6 +343,11 @@ best_combo_for_task, explain_route, get_session_snapshot, db_health_check, sync_ **Cache tools** (2): cache_stats, cache_flush. +**Compression tools** (5): compression_status, compression_configure, set_compression_engine, +list_compression_combos, compression_combo_stats. + +**1proxy tools** (3): oneproxy_fetch, oneproxy_rotate, oneproxy_stats. + **Memory tools** (3): memory_search, memory_add, memory_clear. **Skill tools** (4): skills_list, skills_enable, skills_execute, skills_executions. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4707bf82ee..4576402473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased] +### ✨ New Features + +- **feat(compression):** add RTK tool-output compression, stacked Caveman + RTK pipelines, + compression combo assignments, dashboard context pages, MCP management tools, and + language-aware Caveman rule packs. +- **feat(compression):** expand RTK parity with a 39-filter catalog, RTK-style JSON DSL stages, + inline verify/benchmark coverage, trust-gated custom filters, expanded command detection, and + redacted raw-output recovery. + --- ## [3.7.9] — 2026-05-02 diff --git a/README.md b/README.md index 49c38eb156..c80fde019e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### Never stop coding. Save 15-75% tokens with prompt compression + auto-fallback to **FREE & low-cost AI models**. -_The most complete open-source AI proxy — **one endpoint**, **160+ providers**, **13 routing strategies**, zero downtime. Multi-platform: **Web**, **Desktop (Electron)**, **Mobile (PWA + Termux)**. Fully extensible via **MCP Server (29 tools)**, **A2A Protocol**, and **Memory/Skills** systems. Available in **40+ languages**._ +_The most complete open-source AI proxy — **one endpoint**, **160+ providers**, **13 routing strategies**, zero downtime. Multi-platform: **Web**, **Desktop (Electron)**, **Mobile (PWA + Termux)**. Fully extensible via **MCP Server (37 tools)**, **A2A Protocol**, and **Memory/Skills** systems. Available in **40+ languages**._ **Chat Completions • Responses API • Embeddings • Image Generation • Video • Music • Audio Speech/Transcription • Reranking • Moderations • Web Search • MCP Server • A2A Protocol • 4,600+ Tests • 100% TypeScript** @@ -440,16 +440,18 @@ Every request passes through the compression pipeline **transparently** — no c ``` ┌──────────────────┐ ┌─────────────────────────────┐ ┌──────────────┐ │ Client sends │────▶│ OmniRoute Compression │────▶│ Provider │ -│ full prompt │ │ Pipeline (5 modes) │ │ receives │ +│ full prompt │ │ Pipeline (7 options) │ │ receives │ │ (10,000 tok) │ │ │ │ compressed │ │ │ │ 🪶 Lite ........... ~15% │ │ (2,500 tok) │ │ │ │ 🪨 Standard ....... ~30% │ │ │ │ │ │ ⚡ Aggressive ..... ~50% │ │ 💰 75% saved │ │ │ │ 🔥 Ultra .......... ~75% │ │ │ +│ │ │ 🧰 RTK ............ 20-70% │ │ │ +│ │ │ 🔗 Stacked ........ 30-80% │ │ │ └──────────────────┘ └─────────────────────────────┘ └──────────────┘ ``` -### 5 Compression Modes +### 7 Compression Options | Mode | Savings | Technique | Best For | | ------------------------- | ------- | ----------------------------------------------------------------------------------------------- | -------------------------------------- | @@ -458,6 +460,8 @@ Every request passes through the compression pipeline **transparently** — no c | **🪨 Standard (Caveman)** | ~30% | 30+ regex rules: filler removal, context condensation, structural compression, multi-turn dedup | Daily coding with Claude/Codex | | **⚡ Aggressive** | ~50% | All standard + progressive message aging + tool result summarization + LLM-based compression | Long sessions with many tool calls | | **🔥 Ultra** | ~75% | All aggressive + heuristic token pruning + stopword removal + score-based filtering | Maximum savings when tokens are scarce | +| **🧰 RTK** | 20-70% | 39 command-aware filters, RTK-style JSON DSL, verify gate, trust-gated custom filters | Shell/test/build/git output in agents | +| **🔗 Stacked** | 30-80% | Ordered engine pipeline, usually RTK first then Caveman | Mixed prompts with tool logs + prose | ### Before & After (Standard/Caveman Mode) @@ -481,6 +485,8 @@ Request Body ├─ lite.ts ─────────────── Whitespace, dedup, image URLs, redundant content ├─ caveman.ts ──────────── 30+ regex rules via cavemanRules.ts │ └─ preservation.ts ─── Protects code blocks, URLs, JSON from compression + ├─ engines/rtk/ ────────── Command detection + JSON DSL filters + raw-output recovery + ├─ engines/registry.ts ─── Shared engine registry for caveman, RTK, and stacked ├─ aggressive.ts ───────── Summarizer + tool result compressor + progressive aging │ ├─ summarizer.ts ───── Rule-based message summarization │ ├─ toolResultCompressor.ts ── file/grep/shell/JSON/error compression @@ -492,7 +498,7 @@ Request Body ### Configuration ``` -Dashboard → Settings → Compression → Pick your mode +Dashboard → Context & Cache → Caveman / RTK / Compression Combos ``` Or per-combo override: @@ -508,9 +514,11 @@ Or per-combo override: Auto-trigger: set `autoTriggerTokens` to automatically enable compression when a request exceeds a token threshold. -> 🪨 **Fun fact:** The standard/caveman mode is inspired by [Caveman](https://github.com/JuliusBrussee/caveman) — the viral project that proved "caveman speak" cuts 65% of tokens while keeping 100% technical accuracy. OmniRoute takes this further with a **5-mode pipeline** that goes from gentle whitespace cleanup all the way to aggressive heuristic pruning. +Compression combos can also assign a named compression pipeline to routing combos, so a coding combo can use RTK + Caveman while a paid subscription combo stays on lite mode. -📖 **Full compression documentation:** [`docs/COMPRESSION_GUIDE.md`](docs/COMPRESSION_GUIDE.md) +> 🪨 **Fun fact:** The standard/caveman mode is inspired by [Caveman](https://github.com/JuliusBrussee/caveman) — the viral project that proved "caveman speak" cuts 65% of tokens while keeping 100% technical accuracy. OmniRoute takes this further with a **7-option pipeline** that goes from gentle whitespace cleanup through RTK tool-output filters and stacked multi-engine compression. + +📖 **Full compression documentation:** [`docs/COMPRESSION_GUIDE.md`](docs/COMPRESSION_GUIDE.md) • [`docs/rtk-compression.md`](docs/rtk-compression.md) • [`docs/compression-engines.md`](docs/compression-engines.md) • [`docs/compression-rules-format.md`](docs/compression-rules-format.md) • [`docs/compression-language-packs.md`](docs/compression-language-packs.md) --- @@ -933,8 +941,8 @@ Then in `/dashboard/media` → **Transcription** tab: upload any audio or video | ---------------------------------------------------------------------------------------------------- | -------------------------------- | | 🧠 **Smart 4-Tier Fallback** — Subscription → API → Cheap → Free | Never stop coding, zero downtime | | 🔄 **Format Translation** — OpenAI ↔ Claude ↔ Gemini ↔ Responses API | Works with ANY CLI tool | -| 🗜️ **Prompt Compression** — 5-mode pipeline (off → lite → standard → aggressive → ultra) | Save 15-75% tokens automatically | -| 🤖 **MCP Server** — 29 tools, 3 transports (stdio/SSE/HTTP), 10 scopes | IDE/agent tool integration | +| 🗜️ **Prompt Compression** — 7 options including Caveman, RTK, and stacked pipelines | Save 15-75% tokens automatically | +| 🤖 **MCP Server** — 37 tools, 3 transports (stdio/SSE/HTTP), 10 scopes | IDE/agent tool integration | | 🛡️ **Resilience Engine** — circuit breakers, cooldowns, TLS spoofing, anti-thundering herd | Auto-recovery from any failure | | 🎵 **10 Multi-Modal APIs** — chat, embed, images, video, music, TTS, STT, moderation, rerank, search | One endpoint for everything | | 🌍 **3-Level Proxy** — global, per-provider, per-key + 1proxy free marketplace | Access AI from any country | @@ -1142,7 +1150,7 @@ curl http://localhost:20128/.well-known/agent.json - [User Guide](docs/USER_GUIDE.md) — Providers, combos, CLI integration - [API Reference](docs/API_REFERENCE.md) — All endpoints with examples -- [MCP Server](open-sse/mcp-server/README.md) — 29 tools, IDE configs +- [MCP Server](open-sse/mcp-server/README.md) — 37 tools, IDE configs - [A2A Server](src/lib/a2a/README.md) — JSON-RPC, skills, streaming - [Environment Config](docs/ENVIRONMENT.md) — Complete `.env` reference - [VM Deployment](docs/VM_DEPLOYMENT_GUIDE.md) — VM + nginx + Cloudflare @@ -1325,16 +1333,20 @@ See the [Proxy Guide](docs/PROXY_GUIDE.md) for setup instructions. ### 🧠 Features & Architecture -| Document | Description | -| -------------------------------------------------------- | ---------------------------------------------------------------------- | -| [Architecture](docs/ARCHITECTURE.md) | System architecture, data flow, and internals | -| [Compression Guide](docs/COMPRESSION_GUIDE.md) | 5-mode pipeline: off / lite / standard / aggressive / ultra | -| [Resilience Guide](docs/RESILIENCE_GUIDE.md) | Circuit breakers, cooldowns, queue, anti-thundering herd, TLS spoofing | -| [Auto-Combo Engine](docs/AUTO-COMBO.md) | 6-factor scoring, mode packs, self-healing | -| [Proxy Guide](docs/PROXY_GUIDE.md) | 3-level proxy system, 1proxy marketplace, registry CRUD | -| [Free Tiers](docs/FREE_TIERS.md) | 25+ free API providers consolidated directory | -| [Features Gallery](docs/FEATURES.md) | Visual dashboard tour with screenshots | -| [Codebase Documentation](docs/CODEBASE_DOCUMENTATION.md) | Beginner-friendly codebase walkthrough | +| Document | Description | +| ---------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| [Architecture](docs/ARCHITECTURE.md) | System architecture, data flow, and internals | +| [Compression Guide](docs/COMPRESSION_GUIDE.md) | 7-option pipeline: off / lite / standard / aggressive / ultra / RTK / stacked | +| [RTK Compression](docs/rtk-compression.md) | Command-output compression, filters, trust, verify, raw-output recovery | +| [Compression Engines](docs/compression-engines.md) | Caveman, RTK, stacked pipelines, dashboard/API/MCP surfaces | +| [Compression Rules Format](docs/compression-rules-format.md) | JSON rule-pack schemas for Caveman and RTK filters | +| [Compression Language Packs](docs/compression-language-packs.md) | Language detection and Caveman rule-pack authoring | +| [Resilience Guide](docs/RESILIENCE_GUIDE.md) | Circuit breakers, cooldowns, queue, anti-thundering herd, TLS spoofing | +| [Auto-Combo Engine](docs/AUTO-COMBO.md) | 6-factor scoring, mode packs, self-healing | +| [Proxy Guide](docs/PROXY_GUIDE.md) | 3-level proxy system, 1proxy marketplace, registry CRUD | +| [Free Tiers](docs/FREE_TIERS.md) | 25+ free API providers consolidated directory | +| [Features Gallery](docs/FEATURES.md) | Visual dashboard tour with screenshots | +| [Codebase Documentation](docs/CODEBASE_DOCUMENTATION.md) | Beginner-friendly codebase walkthrough | ### 🤖 Protocols & APIs diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 3b4c38ec29..dfec7d1198 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -216,14 +216,32 @@ Response example: ### Settings -| Endpoint | Method | Description | -| ------------------------------- | ------------- | ---------------------- | -| `/api/settings` | GET/PUT/PATCH | General settings | -| `/api/settings/proxy` | GET/PUT | Network proxy config | -| `/api/settings/proxy/test` | POST | Test proxy connection | -| `/api/settings/ip-filter` | GET/PUT | IP allowlist/blocklist | -| `/api/settings/thinking-budget` | GET/PUT | Reasoning token budget | -| `/api/settings/system-prompt` | GET/PUT | Global system prompt | +| Endpoint | Method | Description | +| ------------------------------- | ------------- | ------------------------- | +| `/api/settings` | GET/PUT/PATCH | General settings | +| `/api/settings/proxy` | GET/PUT | Network proxy config | +| `/api/settings/proxy/test` | POST | Test proxy connection | +| `/api/settings/ip-filter` | GET/PUT | IP allowlist/blocklist | +| `/api/settings/thinking-budget` | GET/PUT | Reasoning token budget | +| `/api/settings/system-prompt` | GET/PUT | Global system prompt | +| `/api/settings/compression` | GET/PUT | Global compression config | + +### Context & Compression + +| Endpoint | Method | Description | +| -------------------------------------- | -------------- | ------------------------------------------------------------------------ | +| `/api/compression/preview` | POST | Preview off/lite/standard/aggressive/ultra/RTK/stacked compression | +| `/api/compression/language-packs` | GET | List available Caveman language packs | +| `/api/compression/rules` | GET | List Caveman rule metadata | +| `/api/context/caveman/config` | GET/PUT | Caveman-specific settings alias | +| `/api/context/rtk/config` | GET/PUT | RTK-specific settings, including custom filters and raw-output retention | +| `/api/context/rtk/filters` | GET | RTK filter catalog and custom-filter diagnostics | +| `/api/context/rtk/test` | POST | Run RTK preview/test against a text payload | +| `/api/context/rtk/raw-output/[id]` | GET | Read retained redacted raw output by pointer id | +| `/api/context/combos` | GET/POST | Compression combo list/create | +| `/api/context/combos/[id]` | GET/PUT/DELETE | Compression combo detail/update/delete | +| `/api/context/combos/[id]/assignments` | GET/PUT | Assign compression combos to routing combos | +| `/api/context/analytics` | GET | Compression analytics alias | ### Monitoring @@ -451,11 +469,12 @@ Content-Type: application/json 2. Route handler calls `handleChat`, `handleEmbedding`, `handleAudioTranscription`, or `handleImageGeneration` 3. Model is resolved (direct provider/model or alias/combo) 4. Credentials selected from local DB with account availability filtering -5. For chat: `handleChatCore` — format detection, translation, cache check, idempotency check -6. Provider executor sends upstream request -7. Response translated back to client format (chat) or returned as-is (embeddings/images/audio) -8. Usage/logging recorded -9. Fallback applies on errors according to combo rules +5. For chat: `handleChatCore` checks semantic/signature cache and resolves combo compression settings +6. Proactive compression runs before provider translation when enabled (`lite`, Caveman, RTK, or stacked) +7. Provider executor sends upstream request +8. Response translated back to client format (chat) or returned as-is (embeddings/images/audio) +9. Usage, compression analytics, and request logs are recorded +10. Fallback applies on errors according to combo rules Full architecture reference: [`ARCHITECTURE.md`](ARCHITECTURE.md) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 948004b42a..cd08c672d7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -2,7 +2,7 @@ 🌐 **Languages:** 🇺🇸 [English](ARCHITECTURE.md) | 🇧🇷 [Português (Brasil)](i18n/pt-BR/ARCHITECTURE.md) | 🇪🇸 [Español](i18n/es/ARCHITECTURE.md) | 🇫🇷 [Français](i18n/fr/ARCHITECTURE.md) | 🇮🇹 [Italiano](i18n/it/ARCHITECTURE.md) | 🇷🇺 [Русский](i18n/ru/ARCHITECTURE.md) | 🇨🇳 [中文 (简体)](i18n/zh-CN/ARCHITECTURE.md) | 🇩🇪 [Deutsch](i18n/de/ARCHITECTURE.md) | 🇮🇳 [हिन्दी](i18n/in/ARCHITECTURE.md) | 🇹🇭 [ไทย](i18n/th/ARCHITECTURE.md) | 🇺🇦 [Українська](i18n/uk-UA/ARCHITECTURE.md) | 🇸🇦 [العربية](i18n/ar/ARCHITECTURE.md) | 🇯🇵 [日本語](i18n/ja/ARCHITECTURE.md) | 🇻🇳 [Tiếng Việt](i18n/vi/ARCHITECTURE.md) | 🇧🇬 [Български](i18n/bg/ARCHITECTURE.md) | 🇩🇰 [Dansk](i18n/da/ARCHITECTURE.md) | 🇫🇮 [Suomi](i18n/fi/ARCHITECTURE.md) | 🇮🇱 [עברית](i18n/he/ARCHITECTURE.md) | 🇭🇺 [Magyar](i18n/hu/ARCHITECTURE.md) | 🇮🇩 [Bahasa Indonesia](i18n/id/ARCHITECTURE.md) | 🇰🇷 [한국어](i18n/ko/ARCHITECTURE.md) | 🇲🇾 [Bahasa Melayu](i18n/ms/ARCHITECTURE.md) | 🇳🇱 [Nederlands](i18n/nl/ARCHITECTURE.md) | 🇳🇴 [Norsk](i18n/no/ARCHITECTURE.md) | 🇵🇹 [Português (Portugal)](i18n/pt/ARCHITECTURE.md) | 🇷🇴 [Română](i18n/ro/ARCHITECTURE.md) | 🇵🇱 [Polski](i18n/pl/ARCHITECTURE.md) | 🇸🇰 [Slovenčina](i18n/sk/ARCHITECTURE.md) | 🇸🇪 [Svenska](i18n/sv/ARCHITECTURE.md) | 🇵🇭 [Filipino](i18n/phi/ARCHITECTURE.md) | 🇨🇿 [Čeština](i18n/cs/ARCHITECTURE.md) -_Last updated: 2026-04-15_ +_Last updated: 2026-05-02_ ## Executive Summary @@ -52,12 +52,13 @@ Core capabilities: - Compliance audit logging with opt-out per API key - Eval framework for LLM quality assurance - Health dashboard with real-time provider circuit breaker status -- MCP Server (29 tools) with 3 transports (stdio/SSE/Streamable HTTP) +- MCP Server (37 tools) with 3 transports (stdio/SSE/Streamable HTTP) - A2A Server (JSON-RPC 2.0 + SSE) with skills and task lifecycle - Memory system (extraction, injection, retrieval, summarization) - Skills system (registry, executor, sandbox, built-in skills) - MITM proxy with certificate management and DNS handling - Prompt injection guard middleware +- Prompt compression pipeline with Caveman, RTK, stacked pipelines, compression combos, language packs, and analytics - ACP (Agent Communication Protocol) registry - Modular OAuth providers (13 individual modules under `src/lib/oauth/providers/`) - Uninstall/full-uninstall scripts @@ -112,6 +113,9 @@ Main pages under `src/app/(dashboard)/dashboard/`: - `/dashboard/health` — uptime, circuit breakers, rate limits, quota-monitored sessions - `/dashboard/logs` — request/proxy/audit/console logs - `/dashboard/settings` — system settings tabs (general, routing, combo defaults, etc.) +- `/dashboard/context/caveman` — Caveman compression rules, language packs, preview, and output mode +- `/dashboard/context/rtk` — RTK command-output filters, preview, and runtime safety settings +- `/dashboard/context/combos` — named compression pipelines assigned to routing combos - `/dashboard/api-manager` — API key lifecycle and model permissions ## High-Level System Context @@ -203,6 +207,8 @@ Management domains: - IP filter: `src/app/api/settings/ip-filter` (GET/PUT) - Thinking budget: `src/app/api/settings/thinking-budget` (GET/PUT) - System prompt: `src/app/api/settings/system-prompt` (GET/PUT) +- Compression: `src/app/api/settings/compression`, `src/app/api/compression/*`, and + `src/app/api/context/*` - Sessions: `src/app/api/sessions` (GET) - Rate limits: `src/app/api/rate-limits` (GET) - Resilience: `src/app/api/resilience` (GET/PATCH) — request queue, connection cooldown, provider breaker, wait-for-cooldown config @@ -252,6 +258,8 @@ Services (business logic): - Rate limit management: `open-sse/services/rateLimitManager.ts` - Circuit breaker: `open-sse/services/circuitBreaker.ts` - Context handoff: `open-sse/services/contextHandoff.ts` — handoff summary generation and injection for context-relay strategy +- Compression: `open-sse/services/compression/*` — proactive compression before provider translation; + includes Caveman rules, RTK filters, stacked pipelines, compression combos, stats, and validation - Codex quota fetcher: `open-sse/services/codexQuotaFetcher.ts` — fetches Codex quota for context-relay handoff decisions - Cooldown-aware retry: `src/sse/services/cooldownAwareRetry.ts` — per-model cooldown retries with configurable `requestRetry` / `maxRetryIntervalSec` - Safe outbound fetch: `src/shared/network/safeOutboundFetch.ts` — guarded provider/model fetch with SSRF guard, private-URL blocking, retry, and timeout @@ -630,6 +638,12 @@ flowchart LR - `src/app/api/settings/ip-filter`: IP allowlist/blocklist (GET/PUT) - `src/app/api/settings/thinking-budget`: thinking token budget config (GET/PUT) - `src/app/api/settings/system-prompt`: global system prompt (GET/PUT) +- `src/app/api/settings/compression`: global compression settings (GET/PUT) +- `src/app/api/compression/*`: compression preview, rule metadata, and language packs +- `src/app/api/context/caveman/config`: Caveman settings alias (GET/PUT) +- `src/app/api/context/rtk/*`: RTK config, filter catalog, test endpoint, and raw-output recovery +- `src/app/api/context/combos*`: compression combo CRUD and routing-combo assignments +- `src/app/api/context/analytics`: compression analytics alias - `src/app/api/sessions`: active session listing (GET) - `src/app/api/rate-limits`: per-account rate limit status (GET) - `src/app/api/sync/tokens`: sync token CRUD (GET/POST) diff --git a/docs/COMPRESSION_GUIDE.md b/docs/COMPRESSION_GUIDE.md index a6db971396..f3fd118fa0 100644 --- a/docs/COMPRESSION_GUIDE.md +++ b/docs/COMPRESSION_GUIDE.md @@ -19,6 +19,8 @@ Client Request → Standard: Caveman-speak filler removal (~30%) → Aggressive: History aging + summarization (~50%) → Ultra: Heuristic pruning + code-block thinning (~75%) + → RTK: Command-aware terminal/tool-output filtering (20-70%) + → Stacked: Ordered multi-engine pipeline, usually RTK then Caveman (30-80%) → Compressed Request → Provider ``` @@ -77,6 +79,35 @@ Maximum compression for token-critical scenarios: **Best for:** When you're hitting context limits repeatedly. +### RTK Mode (20-70% savings) + +RTK mode is optimized for verbose tool outputs that appear in coding-agent sessions: + +- Detects command/output classes such as `git status`, `git diff`, `git log`, test runners, + TypeScript/Vite/Webpack builds, ESLint/Biome/Prettier, npm audit/installs, Docker logs, infra + output, and generic shell output +- Applies JSON filter packs from `open-sse/services/compression/engines/rtk/filters/` +- Ships 39 built-in filters with inline verify samples +- Removes ANSI control sequences, progress bars, repeated lines, and non-actionable noise +- Preserves failures, errors, warnings, changed files, summaries, and the tail of long output +- Supports trust-gated project filters, global filters, and optional redacted raw-output recovery + +**Best for:** Agent sessions with shell, build, test, git, grep, and file-output transcripts. + +### Stacked Mode (30-80% savings) + +Stacked mode runs multiple compression engines in a deterministic order. The default pipeline is: + +```txt +RTK -> Caveman +``` + +That order keeps terminal/tool output compact first, then applies Caveman semantic condensation to +the remaining natural-language prompt. Stacked pipelines can be configured globally or through +compression combos assigned to routing combos. + +**Best for:** Mixed context with large tool logs plus human instructions or assistant summaries. + --- ## Token Savings Visualization @@ -87,6 +118,8 @@ With Lite: 40K tokens sent (15% saved — safe, always-on) With Standard: 33K tokens sent (30% saved — caveman-speak rules) With Aggressive: 24K tokens sent (50% saved — aging + summarization) With Ultra: 12K tokens sent (75% saved — heuristic pruning) +With RTK: varies by output (20-70% saved — command-aware filters) +With Stacked: varies by pipeline (30-80% saved — multi-engine execution) ``` --- @@ -95,25 +128,29 @@ With Ultra: 12K tokens sent (75% saved — heuristic pruning) ### Dashboard -Navigate to `Dashboard → Settings → Compression`: +Navigate to `Dashboard → Context & Cache`: -- **Default Mode** — sets the system-wide compression mode +- **Caveman** — mode selection, language packs, preview, and global defaults +- **RTK** — command-filter preview, RTK safety settings, and filter catalog +- **Compression Combos** — named engine pipelines assigned to routing combos - **Auto-Trigger Threshold** — automatically engage compression when token count exceeds threshold -- **Per-Combo Override** — each combo can have its own compression mode ### Per-Combo Override -In `Dashboard → Combos → [Your Combo] → Advanced`, set compression mode per combo: +In `Dashboard → Context & Cache → Compression Combos`, assign a compression combo to a routing +combo: ```txt Combo: "free-forever" - Mode: Standard + Compression Combo: "coding-agent-stack" + Pipeline: RTK -> Caveman Targets: 1. gc/gemini-3-flash 2. if/kimi-k2-thinking ``` -This lets you use aggressive compression on free providers while keeping lite mode on paid subscriptions. +This lets you use stacked compression on free/coding providers while keeping lite mode on paid +subscriptions. ### API @@ -124,7 +161,20 @@ curl http://localhost:20128/api/settings/compression # Update compression settings curl -X PUT http://localhost:20128/api/settings/compression \ -H "Content-Type: application/json" \ - -d '{"defaultMode":"lite","autoTriggerThreshold":32000}' + -d '{"defaultMode":"stacked","autoTriggerMode":"stacked","autoTriggerTokens":32000}' + +# Preview a specific RTK/stacked payload +curl -X POST http://localhost:20128/api/compression/preview \ + -H "Content-Type: application/json" \ + -d '{"mode":"rtk","messages":[{"role":"tool","content":"npm test output here"}]}' + +# List RTK filter packs +curl http://localhost:20128/api/context/rtk/filters + +# Test RTK directly with optional command metadata +curl -X POST http://localhost:20128/api/context/rtk/test \ + -H "Content-Type: application/json" \ + -d '{"command":"npm test","text":"FAIL tests/example.test.ts\nError: boom"}' ``` --- @@ -136,11 +186,14 @@ The compression engine **always preserves:** - ✅ Code blocks (fenced and inline) - ✅ URLs and file paths - ✅ JSON structures and structured data -- ✅ API keys, tokens, and identifiers +- ✅ Identifiers and protected technical tokens - ✅ Mathematical expressions - ✅ Tool/function call definitions - ✅ System prompts (in lite mode) +RTK raw-output recovery redacts common API keys, bearer tokens, Slack tokens, AWS access keys, +passwords, tokens, and secrets before anything is persisted. + --- ## Compression Stats @@ -154,7 +207,10 @@ Every compressed request includes stats in the server logs: "savingsPercent": 15.0, "techniquesUsed": ["collapseWhitespace", "dedupSystemPrompt"], "mode": "lite", - "latencyMs": 0.8 + "engine": "caveman", + "compressionComboId": "coding-agent-stack", + "durationMs": 0.8, + "rtkRawOutputPointers": [] } ``` @@ -166,7 +222,8 @@ Every compressed request includes stats in the server logs: | ------- | ------------------------------------ | ---------- | | Phase 1 | Off, Lite | ✅ Shipped | | Phase 2 | Standard, Aggressive, Ultra | ✅ Shipped | -| Phase 3 | Per-model adaptive, ML-based pruning | 🗓️ Planned | +| Phase 3 | RTK, Stacked, Compression Combos | ✅ Shipped | +| Phase 4 | Per-model adaptive, ML-based pruning | 🗓️ Planned | --- @@ -181,3 +238,7 @@ Standard mode compression rules are inspired by **[Caveman](https://github.com/J - [Environment Config](ENVIRONMENT.md) — Compression environment variables - [Architecture Guide](ARCHITECTURE.md) — Compression pipeline internals - [User Guide](USER_GUIDE.md) — Getting started with compression +- [RTK Compression](rtk-compression.md) — RTK filters, trust model, verify gate, raw-output recovery +- [Compression Engines](compression-engines.md) — Caveman, RTK, stacked, APIs, MCP, dashboard +- [Compression Rules Format](compression-rules-format.md) — JSON rule-pack format +- [Compression Language Packs](compression-language-packs.md) — Language-specific Caveman rules diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 02e7c2843b..066d800705 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -497,6 +497,12 @@ The logging system writes to both stdout and rotated log files. All configuratio | `CONTEXT_LENGTH_DEFAULT` | `128000` | Global fallback max context length for models without explicit config. | | `USAGE_TOKEN_BUFFER` | `100` | Extra token headroom reserved when tracking usage quotas. | +### Compression + +| Variable | Default | Description | +| ------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------- | +| `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS` | unset | Trust project `.rtk/filters.json` without a `.rtk/trust.json` hash. Use only in controlled local development. | + ### Low-RAM Docker Example ```bash diff --git a/docs/FEATURES.md b/docs/FEATURES.md index d68c6a6db9..ff2ec2e3c8 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -113,6 +113,20 @@ Currently supports Codex account rotation. See [Context Relay documentation](fea --- +## 🗜️ Prompt Compression _(v3.7.9+)_ + +Context & Cache now exposes dedicated pages for Caveman, RTK, and Compression Combos: + +- **Caveman** — language-aware rule packs, preview, output-mode controls, and analytics +- **RTK** — command-aware compression for shell, git, test, build, package, Docker, infra, JSON, and stack-trace output +- **Compression Combos** — named pipelines such as `rtk -> caveman` assigned to routing combos +- **Raw-output recovery** — optional redacted RTK raw-output pointers for debugging compressed failures + +See [Compression Guide](COMPRESSION_GUIDE.md), [RTK Compression](rtk-compression.md), and +[Compression Engines](compression-engines.md). + +--- + ## 🛡️ Proxy Hardening _(v3.5.5+)_ Comprehensive proxy configuration enforcement across the entire request pipeline: diff --git a/docs/MCP-SERVER.md b/docs/MCP-SERVER.md index 2f8e6bab3f..0483ef19f8 100644 --- a/docs/MCP-SERVER.md +++ b/docs/MCP-SERVER.md @@ -1,6 +1,6 @@ # OmniRoute MCP Server Documentation -> Model Context Protocol server with 16 intelligent tools +> Model Context Protocol server with 37 tools across routing, cache, compression, memory, skills, and proxy operations ## Installation @@ -49,20 +49,50 @@ See [IDE Configs](integrations/ide-configs.md) for Antigravity, Cursor, Copilot, | `omniroute_explain_route` | Explain a past routing decision | | `omniroute_get_session_snapshot` | Full session state: costs, tokens, errors | +## Cache Tools (2) + +| Tool | Description | +| :---------------------- | :-------------------------------------------------- | +| `omniroute_cache_stats` | Semantic cache, prompt-cache, and idempotency stats | +| `omniroute_cache_flush` | Flush cache globally or by signature/model | + +## Compression Tools (5) + +| Tool | Description | +| :---------------------------------- | :------------------------------------------------------------- | +| `omniroute_compression_status` | Compression settings, analytics summary, and cache-aware stats | +| `omniroute_compression_configure` | Configure compression mode, threshold, and runtime options | +| `omniroute_set_compression_engine` | Set Caveman, RTK, or stacked compression mode and pipeline | +| `omniroute_list_compression_combos` | List named compression combos and routing assignments | +| `omniroute_compression_combo_stats` | Analytics grouped by compression combo and engine | + +See [Compression Engines](compression-engines.md) and [RTK Compression](rtk-compression.md) for +the runtime compression model behind these tools. + +## Other Tool Groups + +The remaining MCP surface includes 1proxy tools, memory tools, and skill tools. The live source of +truth is `open-sse/mcp-server/tools/` and `open-sse/mcp-server/schemas/tools.ts`. + ## Authentication MCP tools are authenticated via API key scopes. Each tool requires specific scopes: -| Scope | Tools | -| :------------- | :----------------------------------------------- | -| `read:health` | get_health, get_provider_metrics | -| `read:combos` | list_combos, get_combo_metrics | -| `write:combos` | switch_combo | -| `read:quota` | check_quota | -| `write:route` | route_request, simulate_route, test_combo | -| `read:usage` | cost_report, get_session_snapshot, explain_route | -| `write:config` | set_budget_guard, set_resilience_profile | -| `read:models` | list_models_catalog, best_combo_for_task | +| Scope | Tools | +| :-------------------- | :------------------------------------------------------------------- | +| `read:health` | get_health, get_provider_metrics | +| `read:combos` | list_combos, get_combo_metrics | +| `write:combos` | switch_combo | +| `read:quota` | check_quota | +| `write:route` | route_request, simulate_route, test_combo | +| `read:usage` | cost_report, get_session_snapshot, explain_route | +| `write:config` | set_budget_guard, set_resilience_profile | +| `read:models` | list_models_catalog, best_combo_for_task | +| `read:cache` | cache_stats | +| `write:cache` | cache_flush | +| `read:compression` | compression_status, list_compression_combos, compression_combo_stats | +| `write:compression` | compression_configure, set_compression_engine | +| `execute:completions` | route_request, test_combo | ## Audit Logging @@ -74,10 +104,10 @@ Every tool call is logged to `mcp_tool_audit` with: ## Files -| File | Purpose | -| :------------------------------------------- | :------------------------------------------ | -| `open-sse/mcp-server/server.ts` | MCP server creation + 16 tool registrations | -| `open-sse/mcp-server/transport.ts` | Stdio + HTTP transport | -| `open-sse/mcp-server/auth.ts` | API key + scope validation | -| `open-sse/mcp-server/audit.ts` | Tool call audit logging | -| `open-sse/mcp-server/tools/advancedTools.ts` | 8 advanced tool handlers | +| File | Purpose | +| :------------------------------------------- | :------------------------------------------------ | +| `open-sse/mcp-server/server.ts` | MCP server creation and scoped tool registrations | +| `open-sse/mcp-server/transport.ts` | Stdio + HTTP transport | +| `open-sse/mcp-server/auth.ts` | API key + scope validation | +| `open-sse/mcp-server/audit.ts` | Tool call audit logging | +| `open-sse/mcp-server/tools/advancedTools.ts` | 8 advanced tool handlers | diff --git a/docs/SETUP_GUIDE.md b/docs/SETUP_GUIDE.md index 9e78e203e9..26c332a5ab 100644 --- a/docs/SETUP_GUIDE.md +++ b/docs/SETUP_GUIDE.md @@ -138,7 +138,7 @@ Add to your MCP settings: } ``` -**Full MCP documentation:** [MCP Server README](../open-sse/mcp-server/README.md) — 29 tools, IDE configs, Python/TS/Go clients. +**Full MCP documentation:** [MCP Server README](../open-sse/mcp-server/README.md) — 37 tools, IDE configs, Python/TS/Go clients. ### A2A Setup (Agent-to-Agent Protocol) diff --git a/docs/compression-engines.md b/docs/compression-engines.md new file mode 100644 index 0000000000..12e18d60c6 --- /dev/null +++ b/docs/compression-engines.md @@ -0,0 +1,127 @@ +# Compression Engines + +OmniRoute compression is built around engine contracts. A mode can run one engine directly +(`caveman` or `rtk`) or a deterministic stacked pipeline that executes multiple engines in order. + +## Modes + +| Mode | Engine path | Intended input | +| ------------ | ---------------------------------- | -------------------------------------------- | +| `off` | none | Exact prompt preservation | +| `lite` | Caveman lite helpers | Low-risk always-on cleanup | +| `standard` | Caveman | Natural-language prompt condensation | +| `aggressive` | Caveman + history/tool summarizers | Long chat sessions | +| `ultra` | Caveman + pruning helpers | Context-limit recovery | +| `rtk` | RTK | Terminal, shell, build, test, and git output | +| `stacked` | Pipeline, default `rtk -> caveman` | Mixed tool logs and prose | + +## Engine Registry + +The registry lives in `open-sse/services/compression/engines/registry.ts`. Engines expose a shared +contract: + +- `id`: stable engine id such as `caveman` or `rtk` +- `label`: dashboard-readable name +- `supports(mode)`: whether the engine can execute a compression mode +- `compress(input)`: transforms text/messages and returns stats + +`strategySelector.ts` registers the built-in engines before compression runs. This lets preview, +runtime compression, stacked mode, tests, and future engines use the same execution path. + +## Caveman + +Caveman mode focuses on semantic condensation of normal prose: + +- preserves code blocks, URLs, JSON, paths, and structured data +- removes filler, hedging, repeated context, and verbose connective phrasing +- supports language-aware file rule packs in `open-sse/services/compression/rules/` +- remains available through the legacy `standard`, `aggressive`, and `ultra` modes + +The dashboard surface is `Dashboard -> Context & Cache -> Caveman`. + +## RTK + +RTK mode focuses on command and tool output: + +- detects output classes such as `git status`, `git branch`, `git diff`, Vitest/Jest/Pytest, + Cargo/Go tests, TypeScript/Vite/Webpack builds, ESLint, npm audit/installs, Docker logs, + shell `find`/`grep`, stack traces, and generic logs +- applies 39 JSON filters from `open-sse/services/compression/engines/rtk/filters/` +- supports the RTK-style declarative pipeline: ANSI stripping, replace, match-output short-circuit, + strip/keep lines, per-line truncation, head/tail/max-line truncation, and on-empty fallback +- supports trust-gated project filters in `.rtk/filters.json` and global filters in + `DATA_DIR/rtk/filters.json` +- strips ANSI sequences, progress noise, repeated lines, and unhelpful boilerplate +- preserves actionable failures, warnings, summaries, changed files, and tail context +- can optionally retain redacted raw output for recovery/debugging through authenticated management + routes + +The dashboard surface is `Dashboard -> Context & Cache -> RTK`. + +Operational details for custom filters, trust, verify, and raw-output recovery live in +[`rtk-compression.md`](rtk-compression.md). + +## Stacked Pipelines + +Stacked mode runs pipeline steps in order. The default is: + +```txt +rtk -> caveman +``` + +Use this for coding-agent sessions where a prompt combines command output with human or assistant +prose. RTK reduces noisy tool logs first, then Caveman compresses remaining natural language. + +Pipeline steps are configured with `stackedPipeline` in compression settings or through compression +combos. + +## Compression Combos + +Compression combos are named compression profiles that can be assigned to routing combos: + +- `compression_combos`: stores mode, pipeline, RTK config, language config, and default marker +- `compression_combo_assignments`: maps a compression combo to a routing combo +- runtime integration resolves an assigned compression combo before generic combo overrides +- analytics include `compression_combo_id` and `engine` + +Dashboard surface: `Dashboard -> Context & Cache -> Compression Combos`. + +## API Surface + +| Route | Purpose | +| -------------------------------------- | ------------------------------------------ | +| `/api/settings/compression` | Global compression settings | +| `/api/compression/preview` | Preview any compression mode | +| `/api/compression/language-packs` | List available Caveman language packs | +| `/api/context/caveman/config` | Caveman settings alias | +| `/api/context/rtk/config` | RTK defaults and settings | +| `/api/context/rtk/filters` | RTK filter catalog | +| `/api/context/rtk/test` | RTK preview/test endpoint | +| `/api/context/rtk/raw-output/[id]` | Authenticated redacted raw-output recovery | +| `/api/context/combos` | Compression combo CRUD | +| `/api/context/combos/[id]/assignments` | Routing-combo assignment CRUD | +| `/api/context/analytics` | Compression analytics alias | + +Management routes require management authentication or API-key policy checks. + +## MCP Tools + +Compression exposes five MCP tools: + +| Tool | Scope | Purpose | +| ----------------------------------- | ------------------- | -------------------------------- | +| `omniroute_compression_status` | `read:compression` | Settings, analytics, cache stats | +| `omniroute_compression_configure` | `write:compression` | Update global settings | +| `omniroute_set_compression_engine` | `write:compression` | Set mode and optional pipeline | +| `omniroute_list_compression_combos` | `read:compression` | List compression combos | +| `omniroute_compression_combo_stats` | `read:compression` | Read combo/engine analytics | + +## Validation + +The focused gates for this area are: + +```bash +node --import tsx/esm --test tests/unit/compression/rtk-*.test.ts tests/unit/compression/pipeline-integration.test.ts tests/unit/compression/context-compression-api.test.ts +node --import tsx/esm --test tests/unit/compression/*.test.ts tests/golden-set/*.test.ts tests/integration/compression-pipeline.test.ts tests/unit/api/compression/compression-api.test.ts +npm run typecheck:core +``` diff --git a/docs/compression-language-packs.md b/docs/compression-language-packs.md new file mode 100644 index 0000000000..f797865697 --- /dev/null +++ b/docs/compression-language-packs.md @@ -0,0 +1,96 @@ +# Compression Language Packs + +Caveman compression can load language-specific rule packs in addition to the built-in English rules. +This keeps the core engine stable while allowing Portuguese, Spanish, German, French, Japanese, and +future language packs to evolve independently. + +## Location + +Language packs live under: + +```txt +open-sse/services/compression/rules// +``` + +Current shipped packs include: + +| Language | Directory | +| ------------------- | -------------- | +| English | `rules/en/` | +| Portuguese (Brazil) | `rules/pt-BR/` | +| Spanish | `rules/es/` | +| German | `rules/de/` | +| French | `rules/fr/` | +| Japanese | `rules/ja/` | + +## Language Detection + +`languageDetector.ts` uses lightweight heuristics to infer the language from prompt text. The +configured default language is still respected, and detection can be disabled by config when exact +control is required. + +Detection output is used only to choose rule packs. It does not change provider routing, locale +selection, or UI language. + +## Config Shape + +Compression settings can include: + +```json +{ + "languageConfig": { + "enabled": true, + "defaultLanguage": "en", + "autoDetect": true, + "enabledPacks": ["en", "pt-BR", "es", "de", "fr", "ja"] + }, + "cavemanConfig": { + "language": "en", + "autoDetectLanguage": true, + "enabledLanguagePacks": ["en", "pt-BR", "es", "de", "fr", "ja"] + } +} +``` + +`languageConfig` controls dashboard/preview defaults. `cavemanConfig` is the runtime engine config +used when Caveman compresses message text. + +## Adding a Language Pack + +1. Create `open-sse/services/compression/rules//.json`. +2. Use the Caveman rule format from `docs/compression-rules-format.md`. +3. Keep replacements conservative and avoid changing code, identifiers, URLs, or JSON. +4. Add or update tests for language selection and replacement behavior. +5. Expose new dashboard/i18n labels if the language appears in UI selectors. + +## API + +Available packs can be queried with: + +```bash +curl http://localhost:20128/api/compression/language-packs +``` + +The preview endpoint accepts language config overrides: + +```bash +curl -X POST http://localhost:20128/api/compression/preview \ + -H "Content-Type: application/json" \ + -d '{ + "mode": "standard", + "text": "Por favor, eu gostaria que voce basicamente resumisse isso.", + "config": { + "languageConfig": { + "defaultLanguage": "pt-BR", + "autoDetect": true + } + } + }' +``` + +## Operational Notes + +- English built-in rules remain the fallback when a language pack is missing. +- Invalid built-in JSON packs fail validation so release assets do not silently degrade. +- Rule packs are data-only and should not import code or run arbitrary logic. +- The compression analytics layer records the selected mode and engine, not full prompt text. diff --git a/docs/compression-rules-format.md b/docs/compression-rules-format.md new file mode 100644 index 0000000000..a3187b9c8e --- /dev/null +++ b/docs/compression-rules-format.md @@ -0,0 +1,177 @@ +# Compression Rules Format + +Compression rules are JSON files loaded at runtime. They are intentionally data-only so new +language packs and RTK command filters can be reviewed without changing engine code. + +## Caveman Rule Packs + +Caveman rule packs live under: + +```txt +open-sse/services/compression/rules//.json +``` + +Each pack contains replacements that apply to normal prose after protected regions are isolated. + +```json +{ + "language": "en", + "category": "filler", + "rules": [ + { + "name": "remove_basically", + "pattern": "\\bbasically\\b", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite", + "description": "Remove filler adverb." + } + ] +} +``` + +### Caveman Fields + +| Field | Required | Description | +| ---------------------- | -------- | ---------------------------------------------------------------- | +| `language` | yes | BCP-47-like language key such as `en`, `pt-BR`, `es` | +| `category` | yes | Pack category filename/category, for example `filler` or `dedup` | +| `rules` | yes | Array of regex replacement rules | +| `rules[].name` | yes | Stable rule name | +| `rules[].pattern` | yes | JavaScript regex source, compiled with `gi` | +| `rules[].replacement` | yes | Replacement string | +| `rules[].context` | no | `all`, `user`, `assistant`, or `system`; default `all` | +| `rules[].category` | no | `filler`, `context`, `structural`, `dedup`, `terse`, or `ultra` | +| `rules[].minIntensity` | no | `lite`, `full`, or `ultra`; default `lite` | +| `rules[].description` | no | Human-readable rule summary | + +Caveman file packs are data-only. Replacement functions are reserved for built-in TypeScript rules; +JSON packs use string replacements only. + +## RTK Filter Packs + +RTK filters live under: + +```txt +open-sse/services/compression/engines/rtk/filters/.json +``` + +Each filter describes how to recognize and compress a command-output family. + +```json +{ + "id": "test-vitest", + "label": "Vitest output", + "category": "test", + "priority": 92, + "match": { + "outputTypes": ["test-vitest"], + "commands": ["vitest", "npm test", "npm run test"], + "patterns": ["\\bFAIL\\b", "\\bPASS\\b", "\\bTest Files\\b"] + }, + "rules": { + "stripAnsi": true, + "replace": [{ "pattern": "\\s+\\[[0-9]+ms\\]", "replacement": "" }], + "matchOutput": [ + { "pattern": "All tests passed", "message": "vitest: ok", "unless": "FAIL|Error:" } + ], + "includePatterns": ["FAIL", "Error:", "Test Files", "Tests"], + "dropPatterns": ["^\\s*$", "Duration\\s+\\d+"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "truncateLineAt": 240, + "maxLines": 160, + "headLines": 24, + "tailLines": 40, + "onEmpty": "vitest: ok", + "filterStderr": false + }, + "preserve": { + "errorPatterns": ["FAIL", "Error:", "AssertionError"], + "summaryPatterns": ["Test Files", "Tests", "Snapshots"] + }, + "tests": [ + { + "name": "keeps failing tests", + "command": "vitest", + "input": "FAIL test/a.test.ts\\nError: boom\\nTest Files 1 failed", + "expected": "FAIL test/a.test.ts\\nError: boom\\nTest Files 1 failed" + } + ] +} +``` + +### RTK Fields + +| Field | Required | Description | +| -------------------------- | -------- | ------------------------------------------------------------------------------ | +| `id` | yes | Stable filter id | +| `label` | yes | Dashboard-readable name | +| `category` | yes | Filter family: git, test, build, shell, docker, package, infra, cloud, generic | +| `priority` | no | Higher priority wins when multiple filters match | +| `match.outputTypes` | no | Detector output ids that select this filter | +| `match.commands` | no | Command tokens that select this filter | +| `match.patterns` | no | Regex patterns that select this filter from output text | +| `rules.stripAnsi` | no | Remove ANSI escape sequences before regex stages | +| `rules.replace` | no | Ordered regex substitutions applied line by line | +| `rules.matchOutput` | no | Short-circuit output rules with optional `unless` guard | +| `rules.includePatterns` | no | Lines to prefer preserving | +| `rules.dropPatterns` | no | Lines to remove as noise | +| `rules.collapsePatterns` | no | Repeated matching lines that can be collapsed | +| `rules.deduplicate` | no | Collapse duplicate normalized lines | +| `rules.truncateLineAt` | no | Unicode-safe per-line character limit | +| `rules.maxLines` | no | Maximum retained lines before tail preservation | +| `rules.headLines` | no | Head lines retained during truncation | +| `rules.tailLines` | no | Tail lines retained for recent context | +| `rules.onEmpty` | no | Fallback message when filtering removes all content | +| `rules.filterStderr` | no | Normalize common stderr prefixes before later filtering stages | +| `preserve.errorPatterns` | no | Error lines that should survive truncation | +| `preserve.summaryPatterns` | no | Summary lines that should survive truncation | +| `tests[]` | no | Inline verification samples used by the RTK verify gate | + +RTK applies declarative stages in this order: `stripAnsi`, `filterStderr`, `replace`, +`matchOutput`, `dropPatterns`/`includePatterns`, `truncateLineAt`, `headLines`/`tailLines`, +`maxLines`, and `onEmpty`. + +Custom filters can be loaded from: + +1. Project `.rtk/filters.json` files only after a matching `.rtk/trust.json` hash is present or + `trustProjectFilters` is enabled. +2. Global `DATA_DIR/rtk/filters.json`. +3. Built-in filters. + +Project/global custom files may contain one filter object or an array of filter objects. Invalid +custom filters are skipped with diagnostics; invalid built-in filters fail validation. + +Project trust file: + +```json +{ + "filtersSha256": "0123456789abcdef..." +} +``` + +The environment override `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=1` trusts project filters without a +hash and should be limited to controlled local development. + +## Safety Rules + +- Keep rules idempotent: running the same filter twice should not corrupt output. +- Preserve exact error text, file paths, line numbers, and command summaries where possible. +- Avoid rules that modify code blocks, JSON payloads, URLs, or secrets. +- Add unit coverage for new command families in detector/filter tests. +- Add `tests[]` samples to every built-in filter and to shared custom filters. + +## Validation + +Rule packs are validated before use. Built-in Caveman packs and built-in RTK filters fail fast +during validation so broken release assets are caught before shipment. Custom RTK filters are +skipped with diagnostics when parsing or trust validation fails. + +Focused validation: + +```bash +node --import tsx/esm --test tests/unit/compression/rule-loader.test.ts tests/unit/compression/language-packs.test.ts +node --import tsx/esm --test tests/unit/compression/rtk-verify.test.ts tests/unit/compression/rtk-dsl-pipeline.test.ts +``` diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e4277d69a5..bea18f734e 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -51,6 +51,8 @@ tags: description: Routing combo management - name: Settings description: Application settings + - name: Compression + description: Prompt compression, RTK filters, Caveman rules, and compression combos - name: Usage description: Usage analytics and logs - name: Translator @@ -734,6 +736,202 @@ paths: "200": description: Updated settings + /api/settings/compression: + get: + tags: [Compression] + summary: Get global compression settings + security: + - ManagementSessionAuth: [] + responses: + "200": + description: Current compression settings + put: + tags: [Compression] + summary: Update global compression settings + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + defaultMode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + autoTriggerMode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + autoTriggerTokens: + type: integer + minimum: 0 + rtkConfig: + type: object + additionalProperties: true + stackedPipeline: + type: array + items: + type: object + responses: + "200": + description: Updated compression settings + + /api/compression/preview: + post: + tags: [Compression] + summary: Preview compression for a message payload + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [messages, mode] + properties: + mode: + type: string + enum: [off, lite, standard, aggressive, ultra, rtk, stacked] + messages: + type: array + items: + type: object + required: [role, content] + properties: + role: + type: string + content: + oneOf: + - type: string + - type: array + items: {} + config: + type: object + additionalProperties: true + responses: + "200": + description: Compression preview with diff, validation, and stats + + /api/compression/language-packs: + get: + tags: [Compression] + summary: List Caveman compression language packs + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + responses: + "200": + description: Available languages and rule-pack metadata + + /api/compression/rules: + get: + tags: [Compression] + summary: List Caveman compression rule metadata + security: + - BearerAuth: [] + - ManagementSessionAuth: [] + responses: + "200": + description: Caveman rule metadata + + /api/context/rtk/config: + get: + tags: [Compression] + summary: Get RTK compression settings + security: + - ManagementSessionAuth: [] + responses: + "200": + description: Current RTK config + put: + tags: [Compression] + summary: Update RTK compression settings + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + intensity: + type: string + enum: [minimal, standard, aggressive] + customFiltersEnabled: + type: boolean + trustProjectFilters: + type: boolean + rawOutputRetention: + type: string + enum: [never, failures, always] + rawOutputMaxBytes: + type: integer + responses: + "200": + description: Updated RTK config + + /api/context/rtk/filters: + get: + tags: [Compression] + summary: List RTK filters and load diagnostics + security: + - ManagementSessionAuth: [] + responses: + "200": + description: RTK filter catalog and diagnostics + + /api/context/rtk/test: + post: + tags: [Compression] + summary: Run RTK compression preview for text + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [text] + properties: + text: + type: string + command: + type: string + config: + type: object + additionalProperties: true + responses: + "200": + description: Detection and RTK compression result + + /api/context/rtk/raw-output/{id}: + get: + tags: [Compression] + summary: Read retained redacted RTK raw output + security: + - ManagementSessionAuth: [] + parameters: + - in: path + name: id + required: true + schema: + type: string + pattern: "^[a-f0-9]{24}$" + responses: + "200": + description: Raw output text + "404": + description: Raw output not found + /api/settings/payload-rules: get: tags: [Settings] diff --git a/docs/rtk-compression.md b/docs/rtk-compression.md new file mode 100644 index 0000000000..20feebef65 --- /dev/null +++ b/docs/rtk-compression.md @@ -0,0 +1,223 @@ +# RTK Compression + +RTK compression is OmniRoute's command-aware compression engine for terminal and tool output. It is +designed for coding-agent sessions where most context growth comes from test logs, build output, +package manager noise, shell transcripts, Docker output, git output, and stack traces. + +RTK can run directly with `defaultMode: "rtk"` or as the first step in a stacked pipeline, usually: + +```txt +rtk -> caveman +``` + +That order compresses noisy machine output first, then lets Caveman condense remaining prose. + +## What It Compresses + +The built-in catalog currently ships 39 filters across these categories: + +| Category | Examples | +| --------- | ------------------------------------------------------------- | +| `git` | `git status`, `git branch`, `git diff`, `git log` | +| `test` | Vitest, Jest, Pytest, Playwright, Go tests, Cargo tests | +| `build` | TypeScript, ESLint, Biome, Prettier, Vite, Webpack, Turbo, Nx | +| `package` | `npm install`, `npm audit`, `pip`, `uv sync`, Poetry, Bundler | +| `shell` | `ls`, `find`, `grep`, generic shell logs | +| `docker` | `docker ps`, Docker logs | +| `infra` | Terraform, OpenTofu, `systemctl status` | +| `generic` | JSON output, stack traces, generic output fallback | + +The detector in `open-sse/services/compression/engines/rtk/commandDetector.ts` classifies output +before filter selection. Filters can also match by command pattern or output regex when a command +class is not enough. + +## Filter Resolution + +RTK loads filters in this order: + +1. Project filters from `.rtk/filters.json`, only when trusted. +2. Global filters from `DATA_DIR/rtk/filters.json`. +3. Built-in filters from `open-sse/services/compression/engines/rtk/filters/`. + +Project filters are intentionally trust-gated because regex filters can change how tool output is +shown to agents. A project filter file is accepted when one of these is true: + +- `rtkConfig.trustProjectFilters` is `true`. +- `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=1` is set. +- `.rtk/trust.json` contains the SHA-256 hash of `.rtk/filters.json`. + +Trust file example: + +```json +{ + "filtersSha256": "0123456789abcdef..." +} +``` + +Custom filters can be one filter object or an array of filter objects. Invalid custom filters are +skipped and reported by `/api/context/rtk/filters` diagnostics. Invalid built-in filters fail fast. + +## Filter DSL + +Filters use the JSON schema described in [Compression Rules Format](compression-rules-format.md). +The runtime applies these stages in order: + +```txt +stripAnsi -> filterStderr -> replace -> matchOutput -> drop/include lines + -> truncateLineAt -> head/tail/maxLines -> onEmpty +``` + +Important fields: + +| Field | Purpose | +| ---------------------------- | -------------------------------------------------------------- | +| `rules.stripAnsi` | Remove terminal color/control sequences before matching | +| `rules.filterStderr` | Normalize common stderr prefixes before matching/filtering | +| `rules.replace` | Apply ordered regex replacements | +| `rules.matchOutput` | Return a compact summary when output matches a known condition | +| `rules.matchOutput[].unless` | Skip the shortcut when an error/failure pattern is present | +| `rules.dropPatterns` | Remove noisy lines | +| `rules.includePatterns` | Prefer actionable lines | +| `rules.collapsePatterns` | Collapse repeated matching lines | +| `rules.truncateLineAt` | Unicode-safe per-line truncation | +| `rules.onEmpty` | Fallback message if all lines are filtered out | +| `tests[]` | Inline samples used by the verify gate | + +Built-in filters are expected to include inline `tests[]` samples. Custom filters should include +them too, especially when they are shared across projects. + +## Configuration + +Global settings are available through `/api/settings/compression`. RTK-specific settings are also +available through `/api/context/rtk/config`. + +```json +{ + "defaultMode": "stacked", + "autoTriggerMode": "stacked", + "autoTriggerTokens": 32000, + "stackedPipeline": [ + { "engine": "rtk", "intensity": "standard" }, + { "engine": "caveman", "intensity": "full" } + ], + "rtkConfig": { + "enabled": true, + "intensity": "standard", + "applyToToolResults": true, + "applyToCodeBlocks": false, + "applyToAssistantMessages": false, + "enabledFilters": [], + "disabledFilters": [], + "maxLinesPerResult": 120, + "maxCharsPerResult": 12000, + "deduplicateThreshold": 3, + "customFiltersEnabled": true, + "trustProjectFilters": false, + "rawOutputRetention": "never", + "rawOutputMaxBytes": 1048576 + } +} +``` + +`enabledFilters` and `disabledFilters` use filter ids, for example `test-vitest` or `git-diff`. + +## API + +| Route | Method | Purpose | +| ---------------------------------- | ------ | -------------------------------------------- | +| `/api/context/rtk/config` | GET | Read RTK config | +| `/api/context/rtk/config` | PUT | Update RTK config | +| `/api/context/rtk/filters` | GET | List filter catalog and load diagnostics | +| `/api/context/rtk/test` | POST | Preview RTK compression for one text payload | +| `/api/context/rtk/raw-output/[id]` | GET | Read retained redacted raw output | +| `/api/compression/preview` | POST | Preview any compression mode | + +RTK test payload: + +```json +{ + "command": "npm test", + "text": "FAIL tests/example.test.ts\nAssertionError: expected true\nTest Files 1 failed", + "config": { + "intensity": "standard" + } +} +``` + +Compression preview payload: + +```json +{ + "mode": "stacked", + "messages": [ + { + "role": "tool", + "content": "FAIL tests/example.test.ts\nAssertionError: expected true\nTest Files 1 failed" + } + ], + "config": { + "rtkConfig": { + "rawOutputRetention": "failures" + } + } +} +``` + +Management routes require dashboard management auth or the matching API-key policy. + +## Raw Output Recovery + +RTK normally returns only compressed text. For debugging, `rawOutputRetention` can retain redacted +raw output: + +| Value | Behavior | +| ---------- | ------------------------------------------------------- | +| `never` | Do not retain raw output | +| `failures` | Retain only likely failure output | +| `always` | Retain every compressed RTK raw output, after redaction | + +Retained files are written under: + +```txt +DATA_DIR/rtk/raw-output/ +``` + +Secrets are redacted before persistence, including common bearer tokens, API keys, Slack tokens, +AWS access keys, and assignment-style `token=...`, `secret=...`, `password=...` values. Analytics +stores only the pointer id, size, and hash metadata. + +## Verify Gate + +The focused verify gate runs built-in inline filter tests without shelling out to external commands: + +```bash +node --import tsx/esm --test tests/unit/compression/rtk-verify.test.ts +``` + +The broader RTK gate is: + +```bash +node --import tsx/esm --test \ + tests/unit/compression/rtk-*.test.ts \ + tests/unit/compression/pipeline-integration.test.ts \ + tests/unit/compression/context-compression-api.test.ts +``` + +Run the broad compression gate before release: + +```bash +node --import tsx/esm --test \ + tests/unit/compression/*.test.ts \ + tests/golden-set/*.test.ts \ + tests/integration/compression-pipeline.test.ts \ + tests/unit/api/compression/compression-api.test.ts +``` + +## Extending RTK + +1. Add or update a filter JSON file. +2. Include at least one `tests[]` sample that proves the important behavior. +3. Add a fixture under `tests/unit/compression/fixtures/rtk/` for new command families. +4. Add command detection coverage when introducing a new output class. +5. Run the verify and broad RTK gates. +6. If the filter is project-local, commit `.rtk/filters.json` and refresh `.rtk/trust.json` only after review. diff --git a/next.config.mjs b/next.config.mjs index be2b7dd40c..38acb2cc93 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -79,9 +79,13 @@ const nextConfig = { }, outputFileTracingRoot: projectRoot, outputFileTracingIncludes: { - // Migration SQL files are read via fs.readFileSync at runtime and are NOT - // auto-traced by webpack/turbopack — include them explicitly. - "/*": ["./src/lib/db/migrations/**/*"], + // Migration SQL and compression rule/filter JSON files are read via fs at + // runtime and are NOT always auto-traced by webpack/turbopack. + "/*": [ + "./src/lib/db/migrations/**/*", + "./open-sse/services/compression/engines/rtk/filters/**/*.json", + "./open-sse/services/compression/rules/**/*.json", + ], }, outputFileTracingExcludes: { // Planning/task docs are not runtime assets and can break standalone copies @@ -136,97 +140,6 @@ const nextConfig = { images: { unoptimized: true, }, - webpack: (config, { isServer, webpack }) => { - if (isServer) { - // Webpack IgnorePlugin: skip thread-stream test files that contain - // intentionally broken syntax/imports (they cause Turbopack build errors) - config.plugins.push( - new webpack.IgnorePlugin({ - resourceRegExp: /\/test\//, - contextRegExp: /thread-stream/, - }) - ); - - // ── Turbopack / Next.js 16 module-hash patch (#394, #396, #398) ──────── - // - // Next.js 16 (with or without Turbopack) compiles the instrumentation hook - // into a separate chunk and emits hashed require() calls such as: - // require('better-sqlite3-90e2652d1716b047') - // require('zod-dcb22c6336e0bc69') - // require('pino-28069d5257187539') - // - // These hashed names don't exist in node_modules and cause a 500 at - // startup on all npm global installs (issues #394, #396, #398). - // - // We use two strategies: - // 1. Exact-name externals for all known server-side packages. - // 2. Hash-strip catch-all: any require('-<16hexchars>[/subpath]') - // strips the hash suffix and falls through to the real package name. - // - const HASH_PATTERN = /^(.+)-[0-9a-f]{16}(\/.*)?$/; - - const KNOWN_EXTERNALS = new Set([ - "better-sqlite3", - "keytar", - "@ngrok/ngrok", - "wreq-js", - "zod", - "pino", - "pino-pretty", - "pino-abstract-transport", - "child_process", - "fs", - "path", - "os", - "crypto", - "net", - "tls", - "http", - "https", - "stream", - "buffer", - "util", - "process", - ]); - - const prev = config.externals ?? []; - const prevArr = Array.isArray(prev) ? prev : [prev]; - config.externals = [ - ...prevArr, - ({ request }, callback) => { - // Case 1: Exact known package — treat as external - if (KNOWN_EXTERNALS.has(request)) { - return callback(null, `commonjs ${request}`); - } - // Case 2: Hash-suffixed name — strip hash, preserve subpath - // e.g. "better-sqlite3-90e2652d1716b047" → "better-sqlite3" - // "zod-dcb22c6336e0bc69" → "zod" - // "zod-dcb22c6336e0bc69/v3" → "zod/v3" - // "zod-dcb22c6336e0bc69/v4-mini" → "zod/v4-mini" - const hashMatch = request?.match?.(HASH_PATTERN); - if (hashMatch) { - const resolved = hashMatch[2] ? `${hashMatch[1]}${hashMatch[2]}` : hashMatch[1]; - return callback(null, `commonjs ${resolved}`); - } - callback(); - }, - ]; - } else { - // Ignore native Node.js modules in browser bundle - config.resolve.fallback = { - ...config.resolve.fallback, - fs: false, - path: false, - child_process: false, - net: false, - tls: false, - crypto: false, - process: false, - os: false, - }; - } - return config; - }, async headers() { return [ diff --git a/open-sse/config/embeddingRegistry.ts b/open-sse/config/embeddingRegistry.ts index 178db272cf..c2526af4a3 100644 --- a/open-sse/config/embeddingRegistry.ts +++ b/open-sse/config/embeddingRegistry.ts @@ -205,6 +205,15 @@ function resolveEmbeddingProviderId(providerId: string): string { } function normalizeProviderScopedModelId(providerId: string, modelId: string): string { + const resolvedProvider = resolveEmbeddingProviderId(providerId); + const provider = EMBEDDING_PROVIDERS[resolvedProvider]; + if (provider?.models.some((model) => model.id === modelId)) return modelId; + + const providerScopedModelId = `${resolvedProvider}/${modelId}`; + if (provider?.models.some((model) => model.id === providerScopedModelId)) { + return providerScopedModelId; + } + return modelId.startsWith(`${providerId}/`) ? modelId.slice(providerId.length + 1) : modelId; } diff --git a/open-sse/config/rerankRegistry.ts b/open-sse/config/rerankRegistry.ts index e47366c4e3..b075bf4357 100644 --- a/open-sse/config/rerankRegistry.ts +++ b/open-sse/config/rerankRegistry.ts @@ -83,6 +83,15 @@ function resolveRerankProviderId(providerId) { } function normalizeProviderScopedModelId(providerId, modelId) { + const resolvedProvider = resolveRerankProviderId(providerId); + const provider = RERANK_PROVIDERS[resolvedProvider]; + if (provider?.models.some((model) => model.id === modelId)) return modelId; + + const providerScopedModelId = `${resolvedProvider}/${modelId}`; + if (provider?.models.some((model) => model.id === providerScopedModelId)) { + return providerScopedModelId; + } + return modelId.startsWith(`${providerId}/`) ? modelId.slice(providerId.length + 1) : modelId; } diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 9759e1e967..99564b41f7 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1578,30 +1578,6 @@ export async function handleChatCore({ ); } - if (compressionSettings?.cavemanOutputMode?.enabled) { - try { - const { applyCavemanOutputMode } = await import("../services/compression/outputMode.ts"); - const outputMode = applyCavemanOutputMode( - body as Parameters[0], - compressionSettings.cavemanOutputMode - ); - if (outputMode.applied) { - body = outputMode.body as typeof body; - cavemanOutputModeApplied = true; - cavemanOutputModeIntensity = compressionSettings.cavemanOutputMode.intensity; - estimatedTokens = estimateTokens(JSON.stringify(body?.messages ?? body?.input ?? [])); - log?.debug?.("COMPRESSION", "Caveman output mode instruction applied"); - } else if (outputMode.skippedReason && outputMode.skippedReason !== "disabled") { - log?.debug?.("COMPRESSION", `Caveman output mode skipped: ${outputMode.skippedReason}`); - } - } catch (err) { - log?.debug?.( - "COMPRESSION", - "Caveman output mode skipped: " + (err instanceof Error ? err.message : String(err)) - ); - } - } - // --- Modular Compression Pipeline (Phase 1 Lite + Phase 2 Standard/Caveman + Phase 3 Aggressive) --- // Runs BEFORE the existing reactive compressContext() to proactively reduce tokens. try { @@ -1642,7 +1618,9 @@ export async function handleChatCore({ comboMode === "lite" || comboMode === "standard" || comboMode === "aggressive" || - comboMode === "ultra" + comboMode === "ultra" || + comboMode === "rtk" || + comboMode === "stacked" ) { config = { ...config, @@ -1654,6 +1632,74 @@ export async function handleChatCore({ }; compressionComboKey = comboName; } + const routingComboIds = [ + comboConfig?.id, + comboName, + comboName.startsWith("combo/") ? comboName.substring(6) : null, + ].filter((id): id is string => typeof id === "string" && id.length > 0); + if (routingComboIds.length > 0) { + const { getCompressionComboForRoutingCombo } = + await import("../../src/lib/db/compressionCombos.ts"); + const assignedCompressionCombo = + routingComboIds + .map((id) => getCompressionComboForRoutingCombo(id)) + .find((combo) => combo !== null) ?? null; + if (assignedCompressionCombo && assignedCompressionCombo.pipeline.length > 0) { + const comboLanguagePacks = [ + ...new Set( + assignedCompressionCombo.languagePacks + .map((pack) => pack.trim()) + .filter((pack) => pack.length > 0) + ), + ]; + const comboOutputIntensity = ( + ["lite", "full", "ultra"].includes(assignedCompressionCombo.outputModeIntensity) + ? assignedCompressionCombo.outputModeIntensity + : (config.cavemanOutputMode?.intensity ?? "full") + ) as "lite" | "full" | "ultra"; + const comboDefaultLanguage = + comboLanguagePacks.find( + (pack) => pack === config.languageConfig?.defaultLanguage + ) ?? + comboLanguagePacks[0] ?? + config.languageConfig?.defaultLanguage ?? + "en"; + config = { + ...config, + compressionComboId: assignedCompressionCombo.id, + stackedPipeline: assignedCompressionCombo.pipeline, + languageConfig: { + ...(config.languageConfig ?? { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], + }), + enabled: true, + defaultLanguage: comboDefaultLanguage, + enabledPacks: + comboLanguagePacks.length > 0 + ? comboLanguagePacks + : (config.languageConfig?.enabledPacks ?? ["en"]), + }, + cavemanOutputMode: { + ...(config.cavemanOutputMode ?? { + enabled: false, + intensity: "full", + autoClarity: true, + }), + enabled: assignedCompressionCombo.outputMode, + intensity: comboOutputIntensity, + }, + comboOverrides: { + ...(config.comboOverrides ?? {}), + ...(comboName ? { [comboName]: "stacked" as const } : {}), + ...(comboConfig?.id ? { [comboConfig.id]: "stacked" as const } : {}), + }, + }; + compressionComboKey = comboName; + } + } } catch (err) { log?.debug?.( "COMPRESSION", @@ -1662,6 +1708,32 @@ export async function handleChatCore({ ); } } + if (config.cavemanOutputMode?.enabled) { + try { + const { applyCavemanOutputMode } = await import("../services/compression/outputMode.ts"); + const outputModeLanguage = + config.languageConfig?.enabled === true ? config.languageConfig.defaultLanguage : "en"; + const outputMode = applyCavemanOutputMode( + body as Parameters[0], + config.cavemanOutputMode, + outputModeLanguage + ); + if (outputMode.applied) { + body = outputMode.body as typeof body; + cavemanOutputModeApplied = true; + cavemanOutputModeIntensity = config.cavemanOutputMode.intensity; + estimatedTokens = estimateTokens(JSON.stringify(body?.messages ?? body?.input ?? [])); + log?.debug?.("COMPRESSION", "Caveman output mode instruction applied"); + } else if (outputMode.skippedReason && outputMode.skippedReason !== "disabled") { + log?.debug?.("COMPRESSION", `Caveman output mode skipped: ${outputMode.skippedReason}`); + } + } catch (err) { + log?.debug?.( + "COMPRESSION", + "Caveman output mode skipped: " + (err instanceof Error ? err.message : String(err)) + ); + } + } const compressionInputBody = body as Record; const mode = selectCompressionStrategy( config, @@ -1706,6 +1778,9 @@ export async function handleChatCore({ combo_id: comboName ?? null, provider: provider ?? null, mode, + engine: result.stats.engine ?? mode, + compression_combo_id: + result.stats.compressionComboId ?? config.compressionComboId ?? null, original_tokens: result.stats.originalTokens, compressed_tokens: result.stats.compressedTokens, tokens_saved: tokensSaved, @@ -1714,6 +1789,8 @@ export async function handleChatCore({ estimated_usd_saved: estimatedUsdSaved || null, validation_fallback: result.stats.fallbackApplied ? 1 : 0, output_mode: cavemanOutputModeApplied ? cavemanOutputModeIntensity : null, + rtk_raw_output_pointer: result.stats.rtkRawOutputPointers?.[0]?.id ?? null, + rtk_raw_output_bytes: result.stats.rtkRawOutputPointers?.[0]?.bytes ?? null, }); } catch (err) { log?.debug?.( @@ -1776,6 +1853,8 @@ export async function handleChatCore({ combo_id: comboName ?? null, provider: provider ?? null, mode: "output-caveman", + engine: "caveman-output", + compression_combo_id: config.compressionComboId ?? null, original_tokens: estimatedTokens, compressed_tokens: estimatedTokens, tokens_saved: 0, diff --git a/open-sse/mcp-server/README.md b/open-sse/mcp-server/README.md index 43b9852940..80803a6013 100644 --- a/open-sse/mcp-server/README.md +++ b/open-sse/mcp-server/README.md @@ -145,12 +145,15 @@ omniroute --mcp ### Cache and Compression Tools -| # | Tool | Scopes | Description | -| --- | --------------------------------- | ------------------- | ---------------------------------------------------------------------------- | -| 21 | `omniroute_cache_stats` | `read:cache` | Semantic cache, prompt-cache, and idempotency statistics | -| 22 | `omniroute_cache_flush` | `write:cache` | Flush cache entries globally or by signature/model | -| 23 | `omniroute_compression_status` | `read:compression` | Compression settings, analytics summary, and provider-aware cache statistics | -| 24 | `omniroute_compression_configure` | `write:compression` | Configure compression mode and trigger thresholds at runtime | +| # | Tool | Scopes | Description | +| --- | ----------------------------------- | ------------------- | ---------------------------------------------------------------------------- | +| 21 | `omniroute_cache_stats` | `read:cache` | Semantic cache, prompt-cache, and idempotency statistics | +| 22 | `omniroute_cache_flush` | `write:cache` | Flush cache entries globally or by signature/model | +| 23 | `omniroute_compression_status` | `read:compression` | Compression settings, analytics summary, and provider-aware cache statistics | +| 24 | `omniroute_compression_configure` | `write:compression` | Configure compression mode and trigger thresholds at runtime | +| 25 | `omniroute_set_compression_engine` | `write:compression` | Set Caveman, RTK, or stacked compression mode and pipeline | +| 26 | `omniroute_list_compression_combos` | `read:compression` | List named compression combos and routing assignments | +| 27 | `omniroute_compression_combo_stats` | `read:compression` | Read analytics grouped by compression combo and engine | --- @@ -540,12 +543,12 @@ The MCP server supports **fine-grained scope enforcement** for multi-tenant envi | `read:usage` | `cost_report`, `explain_route`, `get_session_snapshot` | | `read:models` | `list_models_catalog` | | `read:cache` | `cache_stats` | -| `read:compression` | `compression_status` | +| `read:compression` | `compression_status`, `list_compression_combos`, `compression_combo_stats` | | `write:combos` | `switch_combo` | | `write:budget` | `set_budget_guard` | | `write:resilience` | `set_resilience_profile` | | `write:cache` | `cache_flush` | -| `write:compression` | `compression_configure` | +| `write:compression` | `compression_configure`, `set_compression_engine` | | `execute:completions` | `route_request`, `test_combo` | **Wildcard scopes:** Use `read:*` to grant all read scopes, or `*` for full access. @@ -581,7 +584,7 @@ mcp-server/ ├── audit.ts # SQLite audit logger (SHA-256 input hashing) ├── scopeEnforcement.ts # Fine-grained scope enforcement ├── schemas/ -│ ├── tools.ts # Zod schemas for all 16 tools (input/output/scopes) +│ ├── tools.ts # Zod schemas for core, cache, compression, and proxy tools │ ├── a2a.ts # A2A protocol types (Agent Card, Task, JSON-RPC) │ ├── audit.ts # Audit & routing decision types + hash helpers │ └── index.ts # Schema barrel export diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index fc9d0deef9..cacce07272 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -1064,10 +1064,12 @@ export const compressionStatusTool: McpToolDefinition< export const compressionConfigureInput = z.object({ enabled: z.boolean().optional(), strategy: z - .enum(["off", "lite", "standard", "aggressive", "ultra"]) + .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked"]) .optional() .describe("Compression mode"), - autoTriggerMode: z.enum(["off", "lite", "standard", "aggressive", "ultra"]).optional(), + autoTriggerMode: z + .enum(["off", "lite", "standard", "aggressive", "ultra", "rtk", "stacked"]) + .optional(), maxTokens: z .number() .int() @@ -1099,7 +1101,7 @@ export const compressionConfigureTool: McpToolDefinition< > = { name: "omniroute_compression_configure", description: - "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", inputSchema: compressionConfigureInput, outputSchema: compressionConfigureOutput, scopes: ["write:compression"], @@ -1108,6 +1110,72 @@ export const compressionConfigureTool: McpToolDefinition< sourceEndpoints: ["/api/compression/configure"], }; +export const setCompressionEngineInput = z.object({ + engine: z.enum(["off", "caveman", "rtk", "stacked"]).optional(), + cavemanIntensity: z.enum(["lite", "full", "ultra"]).optional(), + rtkIntensity: z.enum(["minimal", "standard", "aggressive"]).optional(), + outputMode: z.boolean().optional(), +}); + +export const setCompressionEngineOutput = z.object({ + success: z.boolean(), + settings: z.record(z.string(), z.unknown()), +}); + +export const setCompressionEngineTool: McpToolDefinition< + typeof setCompressionEngineInput, + typeof setCompressionEngineOutput +> = { + name: "omniroute_set_compression_engine", + description: "Set the active compression engine and Caveman/RTK runtime options.", + inputSchema: setCompressionEngineInput, + outputSchema: setCompressionEngineOutput, + scopes: ["write:compression"], + auditLevel: "full", + phase: 2, + sourceEndpoints: ["/api/settings/compression", "/api/context/rtk/config"], +}; + +export const listCompressionCombosInput = z.object({}); +export const listCompressionCombosOutput = z.object({ + combos: z.array(z.record(z.string(), z.unknown())), +}); + +export const listCompressionCombosTool: McpToolDefinition< + typeof listCompressionCombosInput, + typeof listCompressionCombosOutput +> = { + name: "omniroute_list_compression_combos", + description: "List compression combos and their engine pipelines.", + inputSchema: listCompressionCombosInput, + outputSchema: listCompressionCombosOutput, + scopes: ["read:compression"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/context/combos"], +}; + +export const compressionComboStatsInput = z.object({ + comboId: z.string().optional(), + since: z.enum(["24h", "7d", "30d", "all"]).optional(), +}); + +export const compressionComboStatsOutput = z.record(z.string(), z.unknown()); + +export const compressionComboStatsTool: McpToolDefinition< + typeof compressionComboStatsInput, + typeof compressionComboStatsOutput +> = { + name: "omniroute_compression_combo_stats", + description: "Get compression analytics grouped by engine and compression combo.", + inputSchema: compressionComboStatsInput, + outputSchema: compressionComboStatsOutput, + scopes: ["read:compression"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/context/analytics"], +}; + // ============ 1proxy Tools ============ export const oneproxyFetchInput = z.object({ @@ -1245,6 +1313,9 @@ export const MCP_TOOLS = [ cacheFlushTool, compressionStatusTool, compressionConfigureTool, + setCompressionEngineTool, + listCompressionCombosTool, + compressionComboStatsTool, oneproxyFetchTool, oneproxyRotateTool, oneproxyStatsTool, diff --git a/open-sse/mcp-server/tools/compressionTools.ts b/open-sse/mcp-server/tools/compressionTools.ts index 8732d09376..3e8c4d1ed2 100644 --- a/open-sse/mcp-server/tools/compressionTools.ts +++ b/open-sse/mcp-server/tools/compressionTools.ts @@ -13,6 +13,7 @@ import { } from "../../../src/lib/db/compression.ts"; import { getCompressionAnalyticsSummary } from "../../../src/lib/db/compressionAnalytics.ts"; import { getCacheStatsSummary } from "../../../src/lib/db/compressionCacheStats.ts"; +import { listCompressionCombos } from "../../../src/lib/db/compressionCombos.ts"; import type { McpToolExtraLike } from "../scopeEnforcement.ts"; import { getMcpDescriptionCompressionStats } from "../descriptionCompressor.ts"; @@ -38,6 +39,8 @@ export async function handleCompressionStatus( tokensSaved: number; avgCompressionRatio: number; byMode: Record; + byEngine: Record; + byCompressionCombo: Record; validationFallbacks: number; requestsWithReceipts: number; realUsage: { @@ -89,6 +92,8 @@ export async function handleCompressionStatus( tokensSaved: analyticsSummary.totalTokensSaved, avgCompressionRatio: analyticsSummary.avgSavingsPct, byMode: analyticsSummary.byMode ?? {}, + byEngine: analyticsSummary.byEngine ?? {}, + byCompressionCombo: analyticsSummary.byCompressionCombo ?? {}, validationFallbacks: analyticsSummary.validationFallbacks, requestsWithReceipts: analyticsSummary.realUsage.requestsWithReceipts, realUsage: analyticsSummary.realUsage, @@ -213,7 +218,64 @@ export async function handleCompressionConfigure( } import { z } from "zod"; -import { compressionStatusInput, compressionConfigureInput } from "../schemas/tools.ts"; +import { + compressionStatusInput, + compressionConfigureInput, + setCompressionEngineInput, + listCompressionCombosInput, + compressionComboStatsInput, +} from "../schemas/tools.ts"; + +export async function handleSetCompressionEngine( + args: z.infer +): Promise<{ success: boolean; settings: Record }> { + const updates: Record = { enabled: true }; + if (args.engine) { + updates.defaultMode = args.engine === "caveman" ? "standard" : args.engine; + if (args.engine === "off") updates.enabled = false; + } + if (args.cavemanIntensity) { + const current = await getCompressionSettings(); + updates.cavemanConfig = { + ...(current.cavemanConfig ?? {}), + intensity: args.cavemanIntensity, + }; + } + if (args.rtkIntensity) { + const current = await getCompressionSettings(); + updates.rtkConfig = { + ...(current.rtkConfig ?? {}), + intensity: args.rtkIntensity, + }; + } + if (args.outputMode !== undefined) { + const current = await getCompressionSettings(); + updates.cavemanOutputMode = { + ...(current.cavemanOutputMode ?? {}), + enabled: args.outputMode, + }; + } + const settings = await updateCompressionSettings(updates); + return { success: true, settings: settings as unknown as Record }; +} + +export async function handleListCompressionCombos(): Promise<{ + combos: ReturnType; +}> { + return { combos: listCompressionCombos() }; +} + +export async function handleCompressionComboStats( + args: z.infer +): Promise> { + const summary = getCompressionAnalyticsSummary(args.since === "all" ? undefined : args.since); + if (!args.comboId) return summary as unknown as Record; + return { + comboId: args.comboId, + summary, + combo: summary.byCompressionCombo[args.comboId] ?? { count: 0, tokensSaved: 0 }, + }; +} export const compressionTools = { omniroute_compression_status: { @@ -226,8 +288,27 @@ export const compressionTools = { omniroute_compression_configure: { name: "omniroute_compression_configure", description: - "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", + "Configure compression settings at runtime. Supports enabling/disabling compression, changing strategy (off/lite/standard/aggressive/ultra/rtk/stacked), adjusting maxTokens threshold, targetRatio, auto-trigger mode, system prompt preservation, and MCP description compression.", inputSchema: compressionConfigureInput, handler: (args: z.infer) => handleCompressionConfigure(args), }, + omniroute_set_compression_engine: { + name: "omniroute_set_compression_engine", + description: "Set the active compression engine and Caveman/RTK runtime options.", + inputSchema: setCompressionEngineInput, + handler: (args: z.infer) => handleSetCompressionEngine(args), + }, + omniroute_list_compression_combos: { + name: "omniroute_list_compression_combos", + description: "List compression combos and their engine pipelines.", + inputSchema: listCompressionCombosInput, + handler: (_args: z.infer) => handleListCompressionCombos(), + }, + omniroute_compression_combo_stats: { + name: "omniroute_compression_combo_stats", + description: "Get compression analytics grouped by engine and compression combo.", + inputSchema: compressionComboStatsInput, + handler: (args: z.infer) => + handleCompressionComboStats(args), + }, }; diff --git a/open-sse/services/AGENTS.md b/open-sse/services/AGENTS.md index b7b99308d0..d13f4b1ea4 100644 --- a/open-sse/services/AGENTS.md +++ b/open-sse/services/AGENTS.md @@ -51,12 +51,15 @@ ### Prompt Compression Pipeline - **`compression/`** — Modular prompt compression running proactively before `contextManager.ts`. - - `strategySelector.ts` — Selects mode (off/lite/standard/aggressive/ultra) with combo overrides and auto-trigger. + - `strategySelector.ts` — Selects mode (off/lite/standard/aggressive/ultra/rtk/stacked) with compression combo assignments, combo overrides, and auto-trigger. - `lite.ts` — 5 lite techniques: whitespace collapse, system prompt dedup, tool result truncation, redundant removal, image URL placeholder. + - `caveman.ts` / `cavemanRules.ts` — Caveman-style semantic condensation with file-loaded rule packs and language-aware rule selection. + - `engines/registry.ts` — Engine registry used by standalone RTK/Caveman execution and stacked pipelines. + - `engines/rtk/` — RTK tool-output compression: command detection, JSON filter packs, deduplication, smart truncation, ANSI/code noise stripping. - `stats.ts` — Per-request compression stats (original/compressed tokens, savings %, techniques). - `types.ts` — Shared types (`CompressionMode`, `CompressionConfig`, `CompressionStats`, `CompressionResult`). - `index.ts` — Barrel re-exports. - - Phase 1: lite mode only. Standard/aggressive/ultra = Phase 2. + - Dashboard/API surface: `/dashboard/context/caveman`, `/dashboard/context/rtk`, `/dashboard/context/combos`, `/api/context/*`, and `/api/compression/preview`. ### Auto-Routing & Adaptive diff --git a/open-sse/services/compression/caveman.ts b/open-sse/services/compression/caveman.ts index 464a6ac3a8..a57195b02c 100644 --- a/open-sse/services/compression/caveman.ts +++ b/open-sse/services/compression/caveman.ts @@ -1,10 +1,17 @@ -import type { CavemanConfig, CavemanRule, CompressionResult, CompressionMode } from "./types.ts"; +import type { + CavemanConfig, + CavemanRule, + CompressionResult, + CompressionMode, + CompressionStats, +} from "./types.ts"; import { DEFAULT_CAVEMAN_CONFIG } from "./types.ts"; import { CAVEMAN_RULES, getRulesForContext } from "./cavemanRules.ts"; import { extractPreservedBlocks, restorePreservedBlocks } from "./preservation.ts"; import { createCompressionStats, estimateCompressionTokens } from "./stats.ts"; import { validateCompression } from "./validation.ts"; import { mapTextContent } from "./messageContent.ts"; +import { detectCompressionLanguage } from "./languageDetector.ts"; interface ChatMessage { role: string; @@ -18,15 +25,9 @@ interface ChatRequestBody { } const RULE_KEYWORDS: Record = { - redundant_phrasing: [ - "make sure", - "be sure", - "due to the fact", - "the reason is because", - "it is important", - "you should", - "remember to", - ], + redundant_phrasing: ["make sure", "be sure"], + redundant_because: ["due to the fact", "the reason is because"], + redundant_directive: ["it is important", "you should", "remember to"], pleasantries: ["sure", "certainly", "of course", "happy to"], polite_framing: [ "please", @@ -72,6 +73,9 @@ const RULE_KEYWORDS: Record = { self_reference: ["i am trying to", "i am working on", "i have been"], excessive_gratitude: ["thank you so much", "thanks in advance", "i really appreciate"], qualifier_removal: ["a bit", "a little", "somewhat", "kind of", "sort of"], + softeners: ["if possible", "when you get a chance", "at your convenience", "just wondering"], + uncertainty_fillers: ["i guess", "i suppose", "more or less", "in a way"], + assistant_fillers: ["here's", "below is", "this is"], compound_collapse: ["and any potential"], explanatory_prefix: [ "the function appears to be handling", @@ -98,6 +102,7 @@ const RULE_KEYWORDS: Record = { list_conjunction: ["and also", "as well as"], purpose_phrases: ["in order to", "so as to"], redundant_quantifiers: ["each and every", "any and all"], + all_quantifier: ["any and all"], verbose_connectors: ["furthermore", "additionally", "moreover", "in addition"], transition_removal: ["on the other hand", "in contrast", "however"], emphasis_removal: ["very", "really", "extremely", "highly", "quite"], @@ -122,49 +127,36 @@ const RULE_KEYWORDS: Record = { ], reestablished_context: ["going back to the code above", "referring back to", "returning to"], summary_replacement: ["to summarize", "in summary of our conversation", "to recap"], - ultra_abbreviations: [ - "database", - "configuration", - "function", - "request", - "response", + ultra_abbreviations: ["database"], + ultra_config_abbreviation: ["configuration"], + ultra_function_abbreviation: ["function"], + ultra_request_abbreviation: ["request"], + ultra_response_abbreviation: ["response"], + ultra_implementation_abbreviation: ["implementation"], + ultra_authentication_abbreviation: ["authentication"], + ultra_authorization_abbreviation: ["authorization"], + ultra_application_abbreviation: ["application"], + ultra_dependency_abbreviation: ["dependency", "dependencies"], + ultra_common_abbreviations: [ "implementation", "authentication", "authorization", "application", "dependency", + "dependencies", ], }; -const KEYWORD_WORDS = new Map(); +const ARTICLE_HINT_RE = /\b(?:a|an|the)\b/; -function extractLowerWords(lowerText: string): Set { - return new Set(lowerText.match(/[a-z]+(?:'[a-z]+)?/g) ?? []); -} - -function getKeywordWords(keyword: string): string[] { - const cached = KEYWORD_WORDS.get(keyword); - if (cached) return cached; - const words = keyword.match(/[a-z]+(?:'[a-z]+)?/g) ?? []; - KEYWORD_WORDS.set(keyword, words); - return words; -} - -function hasKeywordHint(keyword: string, words: Set): boolean { - const keywordWords = getKeywordWords(keyword); - return keywordWords.length === 0 || keywordWords.every((word) => words.has(word)); -} - -function shouldAttemptRule(ruleName: string, lowerText: string, words: Set): boolean { +function shouldAttemptRule(ruleName: string, lowerText: string): boolean { if (ruleName === "articles") { - return words.has("a") || words.has("an") || words.has("the"); + ARTICLE_HINT_RE.lastIndex = 0; + return ARTICLE_HINT_RE.test(lowerText); } const keywords = RULE_KEYWORDS[ruleName]; - return ( - !keywords || - keywords.some((keyword) => hasKeywordHint(keyword, words) && lowerText.includes(keyword)) - ); + return !keywords || keywords.some((keyword) => lowerText.includes(keyword)); } export function applyRulesToText( @@ -173,11 +165,10 @@ export function applyRulesToText( ): { text: string; appliedRules: string[] } { let result = text; const lowerResult = text.toLowerCase(); - const lowerWords = extractLowerWords(lowerResult); const appliedRules: string[] = []; for (const rule of rules) { - if (!shouldAttemptRule(rule.name, lowerResult, lowerWords)) continue; + if (!shouldAttemptRule(rule.name, lowerResult)) continue; const before = result; const { pattern, replacement } = rule; @@ -217,6 +208,29 @@ function recapitalizeSentences(text: string): string { }); } +function createCavemanStats( + originalTokens: number, + compressedTokens: number, + techniquesUsed: string[], + rulesApplied: string[] | undefined, + durationMs: number +): CompressionStats { + const savingsPercent = + originalTokens > 0 + ? Math.round(((originalTokens - compressedTokens) / originalTokens) * 10000) / 100 + : 0; + return { + originalTokens, + compressedTokens, + savingsPercent, + techniquesUsed, + mode: "standard", + timestamp: Date.now(), + ...(rulesApplied && rulesApplied.length > 0 ? { rulesApplied } : {}), + durationMs, + }; +} + function compileUserPreservePatterns(patterns: string[]): { patterns: RegExp[]; warnings: string[]; @@ -320,7 +334,16 @@ export function cavemanCompress( : { text: textPart, blocks: [] }; preservedBlockCount += blocks.length; - const rules = getRulesForContext(msg.role, config.intensity).filter( + const detectedLanguage = config.autoDetectLanguage + ? detectCompressionLanguage(textPart) + : (config.language ?? "en"); + const enabledPacks = config.enabledLanguagePacks ?? ["en", detectedLanguage]; + const language = enabledPacks.includes(detectedLanguage) + ? detectedLanguage + : enabledPacks.includes("en") + ? "en" + : detectedLanguage; + const rules = getRulesForContext(msg.role, config.intensity, language).filter( (rule) => !config.skipRules.includes(rule.name) ); const { text: rulesApplied, appliedRules } = applyRulesToText(extractedText, rules); @@ -365,10 +388,9 @@ export function cavemanCompress( const durationMs = performance.now() - startMs; const uniqueRules = [...new Set(allAppliedRules)]; - const stats = createCompressionStats( - body as unknown as Record, - { ...body, messages: compressedMessages } as unknown as Record, - "standard" as CompressionMode, + const stats = createCavemanStats( + totalOriginalTokens, + totalCompressedTokens, uniqueRules.length > 0 ? ["caveman-rules"] : [], uniqueRules.length > 0 ? uniqueRules : undefined, Math.round(durationMs * 100) / 100 diff --git a/open-sse/services/compression/cavemanRules.ts b/open-sse/services/compression/cavemanRules.ts index 906f64fc28..9e74aa3de9 100644 --- a/open-sse/services/compression/cavemanRules.ts +++ b/open-sse/services/compression/cavemanRules.ts @@ -1,4 +1,5 @@ import type { CavemanRule } from "./types.ts"; +import { loadAllRulesForLanguage } from "./ruleLoader.ts"; const CAVEMAN_RULES: CavemanRule[] = [ // ── Category 1: Filler Removal (10+ rules) ────────────────────────── @@ -392,13 +393,22 @@ const INTENSITY_RANK = { lite: 0, full: 1, ultra: 2 } as const; export function getRulesForContext( context: string, - intensity: "lite" | "full" | "ultra" = "full" + intensity: "lite" | "full" | "ultra" = "full", + language = "en" ): CavemanRule[] { const rank = INTENSITY_RANK[intensity] ?? INTENSITY_RANK.full; - return CAVEMAN_RULES.filter((rule) => { + const fileRules = language ? loadAllRulesForLanguage(language) : []; + const rules = fileRules.length > 0 ? fileRules : CAVEMAN_RULES; + const selected = rules.filter((rule) => { const minRank = INTENSITY_RANK[rule.minIntensity ?? "lite"]; return (rule.context === "all" || rule.context === context) && minRank <= rank; }); + return selected.length > 0 + ? selected + : CAVEMAN_RULES.filter((rule) => { + const minRank = INTENSITY_RANK[rule.minIntensity ?? "lite"]; + return (rule.context === "all" || rule.context === context) && minRank <= rank; + }); } export function getRuleByName(name: string): CavemanRule | undefined { diff --git a/open-sse/services/compression/diffHelper.ts b/open-sse/services/compression/diffHelper.ts index 80632c7e51..5dbebd7b1d 100644 --- a/open-sse/services/compression/diffHelper.ts +++ b/open-sse/services/compression/diffHelper.ts @@ -16,10 +16,31 @@ export interface CompressionPreviewDiff { fallbackApplied: boolean; } +export interface CompressionPreviewDiffOptions { + maxTokenProduct?: number; +} + +export const DEFAULT_MAX_PREVIEW_DIFF_TOKEN_PRODUCT = 1_000_000; + function tokenize(text: string): string[] { return text.match(/\s+|[^\s]+/g) ?? []; } +function getDiffSkipWarning( + original: string, + compressed: string, + options: CompressionPreviewDiffOptions = {} +): string | null { + const maxTokenProduct = options.maxTokenProduct ?? DEFAULT_MAX_PREVIEW_DIFF_TOKEN_PRODUCT; + if (maxTokenProduct <= 0) return null; + + const originalTokens = tokenize(original).length; + const compressedTokens = tokenize(compressed).length; + if (originalTokens * compressedTokens <= maxTokenProduct) return null; + + return `Preview diff omitted because token product ${originalTokens}x${compressedTokens} exceeds safe limit ${maxTokenProduct}.`; +} + export function buildCompressionDiff( original: string, compressed: string @@ -69,19 +90,27 @@ export function buildCompressionDiff( export function buildCompressionPreviewDiff( original: string, compressed: string, - stats: CompressionStats | null | undefined + stats: CompressionStats | null | undefined, + options: CompressionPreviewDiffOptions = {} ): CompressionPreviewDiff { const validation = validateCompression(original, compressed); const preserved = extractPreservedBlocks(original).blocks.map((block) => ({ kind: block.kind, preview: block.content.replace(/\s+/g, " ").slice(0, 120), })); + const diffSkipWarning = getDiffSkipWarning(original, compressed, options); return { - segments: buildCompressionDiff(original, compressed), + segments: diffSkipWarning + ? [{ type: "same", text: "[diff omitted: input too large]" }] + : buildCompressionDiff(original, compressed), preservedBlocks: preserved, ruleRemovals: stats?.rulesApplied ?? [], - validationWarnings: [...(stats?.validationWarnings ?? []), ...validation.warnings], + validationWarnings: [ + ...(stats?.validationWarnings ?? []), + ...validation.warnings, + ...(diffSkipWarning ? [diffSkipWarning] : []), + ], validationErrors: [...(stats?.validationErrors ?? []), ...validation.errors], fallbackApplied: Boolean(stats?.fallbackApplied || validation.fallbackApplied), }; diff --git a/open-sse/services/compression/engines/cavemanAdapter.ts b/open-sse/services/compression/engines/cavemanAdapter.ts new file mode 100644 index 0000000000..27bb44d87d --- /dev/null +++ b/open-sse/services/compression/engines/cavemanAdapter.ts @@ -0,0 +1,254 @@ +import { applyLiteCompression } from "../lite.ts"; +import { cavemanCompress } from "../caveman.ts"; +import { compressAggressive } from "../aggressive.ts"; +import { ultraCompress } from "../ultra.ts"; +import { createCompressionStats } from "../stats.ts"; +import type { CavemanIntensity } from "../types.ts"; +import type { CompressionEngine, EngineConfigField, EngineValidationResult } from "./types.ts"; + +const CAVEMAN_INTENSITIES: CavemanIntensity[] = ["lite", "full", "ultra"]; + +const CAVEMAN_SCHEMA: EngineConfigField[] = [ + { + key: "intensity", + type: "select", + label: "Intensity", + defaultValue: "full", + options: CAVEMAN_INTENSITIES.map((value) => ({ value, label: value })), + }, + { + key: "minMessageLength", + type: "number", + label: "Minimum message length", + defaultValue: 50, + min: 0, + max: 10000, + }, + { + key: "enabled", + type: "boolean", + label: "Enabled", + defaultValue: true, + }, +]; + +function ok(): EngineValidationResult { + return { valid: true, errors: [] }; +} + +function validateCavemanLikeConfig(config: Record): EngineValidationResult { + const errors: string[] = []; + if ( + config.intensity !== undefined && + !CAVEMAN_INTENSITIES.includes(config.intensity as CavemanIntensity) + ) { + errors.push("intensity must be lite, full, or ultra"); + } + if ( + config.minMessageLength !== undefined && + (typeof config.minMessageLength !== "number" || config.minMessageLength < 0) + ) { + errors.push("minMessageLength must be a non-negative number"); + } + if (config.enabled !== undefined && typeof config.enabled !== "boolean") { + errors.push("enabled must be a boolean"); + } + return { valid: errors.length === 0, errors }; +} + +export const liteEngine: CompressionEngine = { + id: "lite", + name: "Lite", + description: "Fast whitespace, tool-result and image URL reduction.", + icon: "compress", + targets: ["messages", "tool_results"], + stackable: true, + stackPriority: 5, + metadata: { + id: "lite", + name: "Lite", + description: "Fast whitespace, tool-result and image URL reduction.", + inputScope: "messages", + targetLatencyMs: 1, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + return applyLiteCompression(body, { + ...options, + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; + +export const cavemanEngine: CompressionEngine = { + id: "caveman", + name: "Caveman", + description: "Rule-based message compression with preservation and validation.", + icon: "compress", + targets: ["messages"], + stackable: true, + stackPriority: 20, + metadata: { + id: "caveman", + name: "Caveman", + description: "Rule-based message compression with preservation and validation.", + inputScope: "messages", + targetLatencyMs: 1, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const cavemanConfig = { + ...(options?.config?.cavemanConfig ?? {}), + ...(options?.stepConfig ?? {}), + ...(options?.config?.languageConfig?.enabled + ? { + language: options.config.languageConfig.defaultLanguage, + autoDetectLanguage: options.config.languageConfig.autoDetect, + enabledLanguagePacks: options.config.languageConfig.enabledPacks, + } + : {}), + ...(options?.config?.preserveSystemPrompt !== false + ? { + compressRoles: (options?.config?.cavemanConfig?.compressRoles ?? ["user"]).filter( + (role) => role !== "system" + ), + } + : {}), + }; + return cavemanCompress(body as Parameters[0], cavemanConfig); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return CAVEMAN_SCHEMA; + }, + validateConfig(config) { + return validateCavemanLikeConfig(config); + }, +}; + +export const aggressiveEngine: CompressionEngine = { + id: "aggressive", + name: "Aggressive", + description: "Summarization, tool result compression and progressive aging.", + icon: "speed", + targets: ["messages", "tool_results"], + stackable: true, + stackPriority: 30, + metadata: { + id: "aggressive", + name: "Aggressive", + description: "Summarization, tool result compression and progressive aging.", + inputScope: "messages", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const messages = (body.messages ?? []) as Array<{ + role: string; + content?: string | Array<{ type: string; text?: string }>; + [key: string]: unknown; + }>; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + const aggressiveConfig = { + ...(options?.config?.aggressive ?? {}), + ...(options?.stepConfig ?? {}), + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }; + const result = compressAggressive(messages, aggressiveConfig); + const compressedBody = { ...body, messages: result.messages }; + return { + body: compressedBody, + compressed: result.stats.savingsPercent > 0, + stats: createCompressionStats( + body, + compressedBody, + "aggressive", + ["aggressive"], + result.stats.rulesApplied, + result.stats.durationMs + ), + }; + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; + +export const ultraEngine: CompressionEngine = { + id: "ultra", + name: "Ultra", + description: "Heuristic token pruning with optional local SLM fallback.", + icon: "bolt", + targets: ["messages"], + stackable: true, + stackPriority: 40, + metadata: { + id: "ultra", + name: "Ultra", + description: "Heuristic token pruning with optional local SLM fallback.", + inputScope: "messages", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + const messages = (body.messages ?? []) as Array<{ + role: string; + content?: string | unknown[]; + [key: string]: unknown; + }>; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + const ultraConfig = { + ...(options?.config?.ultra ?? {}), + ...(options?.stepConfig ?? {}), + preserveSystemPrompt: options?.config?.preserveSystemPrompt !== false, + }; + const result = ultraCompress(messages, ultraConfig); + const compressedBody = { ...body, messages: result.messages }; + return { + body: compressedBody, + compressed: result.stats.savingsPercent > 0, + stats: createCompressionStats( + body, + compressedBody, + "ultra", + ["ultra"], + result.stats.rulesApplied, + result.stats.durationMs + ), + }; + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return []; + }, + validateConfig() { + return ok(); + }, +}; diff --git a/open-sse/services/compression/engines/index.ts b/open-sse/services/compression/engines/index.ts new file mode 100644 index 0000000000..07094ee9ac --- /dev/null +++ b/open-sse/services/compression/engines/index.ts @@ -0,0 +1,14 @@ +import { registerCompressionEngine, getCompressionEngine } from "./registry.ts"; +import { aggressiveEngine, cavemanEngine, liteEngine, ultraEngine } from "./cavemanAdapter.ts"; +import { rtkEngine } from "./rtk/index.ts"; + +let registered = false; + +export function registerBuiltinCompressionEngines(): void { + const engines = [liteEngine, cavemanEngine, aggressiveEngine, ultraEngine, rtkEngine]; + if (registered && engines.every((engine) => getCompressionEngine(engine.id))) return; + for (const engine of engines) { + if (!getCompressionEngine(engine.id)) registerCompressionEngine(engine); + } + registered = true; +} diff --git a/open-sse/services/compression/engines/registry.ts b/open-sse/services/compression/engines/registry.ts new file mode 100644 index 0000000000..85d6a50cdd --- /dev/null +++ b/open-sse/services/compression/engines/registry.ts @@ -0,0 +1,87 @@ +import type { CompressionEngine, EngineRegistryEntry, EngineValidationResult } from "./types.ts"; + +const ENGINES = new Map(); + +function assertValidEngine(engine: CompressionEngine): void { + if ( + !engine?.id || + typeof engine.apply !== "function" || + typeof engine.compress !== "function" || + typeof engine.getConfigSchema !== "function" || + typeof engine.validateConfig !== "function" + ) { + throw new Error("Invalid compression engine registration"); + } +} + +export function registerEngine( + engine: CompressionEngine, + defaultConfig: Record = {} +): void { + assertValidEngine(engine); + const validation = engine.validateConfig(defaultConfig); + if (!validation.valid) { + throw new Error(`Invalid default config for ${engine.id}: ${validation.errors.join("; ")}`); + } + ENGINES.set(engine.id, { + engine, + enabled: true, + config: { ...defaultConfig }, + }); +} + +export function registerCompressionEngine(engine: CompressionEngine): void { + registerEngine(engine); +} + +export function unregisterCompressionEngine(id: string): boolean { + return ENGINES.delete(id); +} + +export function getEngine(id: string): CompressionEngine | null { + return ENGINES.get(id)?.engine ?? null; +} + +export function getCompressionEngine(id: string): CompressionEngine | null { + return getEngine(id); +} + +export function getEngineEntry(id: string): EngineRegistryEntry | null { + return ENGINES.get(id) ?? null; +} + +export function listEngines(): EngineRegistryEntry[] { + return Array.from(ENGINES.values()); +} + +export function listCompressionEngines(): CompressionEngine[] { + return listEngines().map((entry) => entry.engine); +} + +export function listEnabledEngines(): EngineRegistryEntry[] { + return listEngines().filter((entry) => entry.enabled); +} + +export function setEngineEnabled(id: string, enabled: boolean): boolean { + const entry = ENGINES.get(id); + if (!entry) return false; + entry.enabled = enabled; + return true; +} + +export function updateEngineConfig( + id: string, + config: Record +): EngineValidationResult { + const entry = ENGINES.get(id); + if (!entry) return { valid: false, errors: [`Unknown compression engine: ${id}`] }; + const nextConfig = { ...entry.config, ...config }; + const validation = entry.engine.validateConfig(nextConfig); + if (!validation.valid) return validation; + entry.config = nextConfig; + return { valid: true, errors: [] }; +} + +export function clearCompressionEngineRegistry(): void { + ENGINES.clear(); +} diff --git a/open-sse/services/compression/engines/rtk/codeStripper.ts b/open-sse/services/compression/engines/rtk/codeStripper.ts new file mode 100644 index 0000000000..27c3512910 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/codeStripper.ts @@ -0,0 +1,128 @@ +export type CodeLanguage = + | "javascript" + | "typescript" + | "python" + | "rust" + | "go" + | "ruby" + | "java" + | "unknown"; + +export interface CodeStripperOptions { + removeComments?: boolean; + removeEmptyLines?: boolean; + collapseWhitespace?: boolean; + preserveDocstrings?: boolean; +} + +const LANGUAGE_ALIASES: Record = { + js: "javascript", + jsx: "javascript", + javascript: "javascript", + ts: "typescript", + tsx: "typescript", + typescript: "typescript", + py: "python", + python: "python", + rs: "rust", + rust: "rust", + go: "go", + rb: "ruby", + ruby: "ruby", + java: "java", +}; + +export function normalizeCodeLanguage(language?: string | null): CodeLanguage { + if (!language) return "unknown"; + return LANGUAGE_ALIASES[language.trim().toLowerCase()] ?? "unknown"; +} + +export function detectCodeLanguage(text: string): CodeLanguage { + if (/\b(?:interface|type)\s+\w+\s*=|:\s*(?:string|number|boolean)\b/.test(text)) { + return "typescript"; + } + if (/\b(?:const|let|function|import|export)\b|=>/.test(text)) return "javascript"; + if (/\bdef\s+\w+\(|\bimport\s+\w+|print\(/.test(text)) return "python"; + if (/\bfn\s+\w+\(|\blet\s+mut\b|println!\(/.test(text)) return "rust"; + if (/\bfunc\s+\w+\(|package\s+\w+/.test(text)) return "go"; + if (/\bclass\s+\w+|System\.out\.println/.test(text)) return "java"; + if (/\bdef\s+\w+|puts\s+|end\s*$/.test(text)) return "ruby"; + return "unknown"; +} + +function removeSlashComments(text: string): string { + return text + .replace(/\/\*[\s\S]*?\*\//g, "") + .replace(/^\s*\/\/.*$/gm, "") + .replace(/\s+\/\/\s.*$/gm, ""); +} + +function removeHashComments(text: string): string { + return text.replace(/^\s*#.*$/gm, "").replace(/\s+#\s.*$/gm, ""); +} + +function stripByLanguage( + text: string, + language: CodeLanguage, + options: Required +): string { + if (!options.removeComments) return text; + + if (language === "javascript" || language === "typescript" || language === "rust") { + return removeSlashComments(text); + } + if (language === "go" || language === "java") return removeSlashComments(text); + if (language === "python") { + const withoutDocstrings = options.preserveDocstrings + ? text + : text.replace(/("""[\s\S]*?"""|'''[\s\S]*?''')/g, ""); + return removeHashComments(withoutDocstrings); + } + if (language === "ruby") { + return removeHashComments(text.replace(/^=begin[\s\S]*?^=end/gm, "")); + } + return text; +} + +export function stripCode( + text: string, + language: CodeLanguage = "unknown", + options: CodeStripperOptions = {} +): { + text: string; + strippedLines: number; + language: CodeLanguage; +} { + const resolvedLanguage = language === "unknown" ? detectCodeLanguage(text) : language; + const opts: Required = { + removeComments: options.removeComments !== false, + removeEmptyLines: options.removeEmptyLines !== false, + collapseWhitespace: options.collapseWhitespace !== false, + preserveDocstrings: options.preserveDocstrings === true, + }; + const originalLines = text.split(/\r?\n/).length; + let result = stripByLanguage(text, resolvedLanguage, opts); + + if (opts.removeEmptyLines) result = result.replace(/^\s*$(?:\r?\n)?/gm, ""); + if (opts.collapseWhitespace) { + result = result + .split(/\r?\n/) + .map((line) => line.replace(/[ \t]+$/g, "")) + .join("\n"); + } + + result = result.trim(); + const strippedLines = Math.max(0, originalLines - (result ? result.split(/\r?\n/).length : 0)); + return { text: result, strippedLines, language: resolvedLanguage }; +} + +export function stripCodeComments( + text: string, + language = "typescript" +): { + text: string; + stripped: boolean; +} { + const result = stripCode(text, normalizeCodeLanguage(language)); + return { text: result.text, stripped: result.strippedLines > 0 || result.text !== text }; +} diff --git a/open-sse/services/compression/engines/rtk/commandDetector.ts b/open-sse/services/compression/engines/rtk/commandDetector.ts new file mode 100644 index 0000000000..31806c65b7 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/commandDetector.ts @@ -0,0 +1,465 @@ +export interface CommandDetectionResult { + type: string; + command: string | null; + confidence: number; + category: + | "git" + | "test" + | "build" + | "shell" + | "docker" + | "package" + | "infra" + | "cloud" + | "generic"; + matchedPatterns: string[]; +} + +type Detector = { + type: string; + category: CommandDetectionResult["category"]; + commandPatterns: RegExp[]; + contentPatterns: RegExp[]; +}; + +const COMMAND_PREFIXES = [ + "git", + "make", + "terraform", + "tofu", + "opentofu", + "systemctl", + "npm", + "pnpm", + "yarn", + "vitest", + "jest", + "pytest", + "python", + "go", + "cargo", + "tsc", + "eslint", + "webpack", + "vite", + "biome", + "prettier", + "turbo", + "nx", + "playwright", + "ruff", + "mypy", + "pip", + "uv", + "poetry", + "golangci-lint", + "bundle", + "rubocop", + "docker", + "aws", + "gcloud", + "ssh", + "rsync", + "curl", + "wget", + "ls", + "find", + "grep", + "rg", + "ag", + "ps", + "df", + "du", +]; + +const COMMAND_PREFIX_PATTERN = new RegExp(`^(?:${COMMAND_PREFIXES.join("|")})\\b`); + +const DETECTORS: Detector[] = [ + { + type: "git-status", + category: "git", + commandPatterns: [/^git\s+status\b/i], + contentPatterns: [ + /^On branch /m, + /^Changes (?:not staged|to be committed)/m, + /^Untracked files:/m, + ], + }, + { + type: "git-branch", + category: "git", + commandPatterns: [/^git\s+branch\b/i, /^git\s+checkout\b/i, /^git\s+switch\b/i], + contentPatterns: [/^\*\s+\S+/m, /Switched to (?:a new )?branch/i, /Already on ['"][^'"]+['"]/i], + }, + { + type: "git-diff", + category: "git", + commandPatterns: [/^git\s+diff\b/i, /^git\s+show\b/i], + contentPatterns: [/^diff --git /m, /^@@\s+-\d+,\d+\s+\+\d+,\d+\s+@@/m], + }, + { + type: "git-log", + category: "git", + commandPatterns: [/^git\s+log\b/i], + contentPatterns: [/^commit [0-9a-f]{7,40}/m, /^Author: /m], + }, + { + type: "make", + category: "build", + commandPatterns: [/^make\b/i], + contentPatterns: [/^make\[\d+\]: (?:Entering|Leaving) directory/m, /make: \*\*\* /], + }, + { + type: "terraform-plan", + category: "infra", + commandPatterns: [/^terraform\s+plan\b/i], + contentPatterns: [/Terraform will perform the following actions:/, /Plan: \d+ to add/i], + }, + { + type: "tofu-plan", + category: "infra", + commandPatterns: [/^(?:tofu|opentofu)\s+plan\b/i], + contentPatterns: [/OpenTofu will perform the following actions:/, /Plan: \d+ to add/i], + }, + { + type: "systemctl-status", + category: "infra", + commandPatterns: [/^systemctl\s+status\b/i], + contentPatterns: [/^\s*Loaded:\s+/m, /^\s*Active:\s+/m, /^●\s+\S+\.service/m], + }, + { + type: "test-vitest", + category: "test", + commandPatterns: [/^vitest\b/i, /^npm\s+(?:run\s+)?test:vitest\b/i], + contentPatterns: [/\bvitest\b/i, /^ ✓ /m, /^ ❯ /m, /Test Files\s+\d+\s+(?:passed|failed)/i], + }, + { + type: "test-jest", + category: "test", + commandPatterns: [/^jest\b/i, /^npm\s+(?:run\s+)?test\b/i], + contentPatterns: [/Test Suites:\s+\d+/i, /Tests:\s+\d+/i, /^PASS\s+/m, /^FAIL\s+/m], + }, + { + type: "test-pytest", + category: "test", + commandPatterns: [/^pytest\b/i, /^python\s+-m\s+pytest\b/i], + contentPatterns: [/=+\s+(?:\d+\s+)?(?:passed|failed|errors?)/i, /^E\s+/m, /^FAILED /m], + }, + { + type: "test-cargo", + category: "test", + commandPatterns: [/^cargo\s+test\b/i, /^cargo\s+nextest\b/i], + contentPatterns: [ + /^running \d+ tests?/m, + /^test\s+[\w:.-]+\s+\.\.\.\s+(?:ok|FAILED|ignored)/m, + /test result:\s+(?:ok|FAILED)/i, + ], + }, + { + type: "test-go", + category: "test", + commandPatterns: [/^go\s+test\b/i], + contentPatterns: [/^(?:ok|FAIL)\s+[\w./-]+\s+[\d.]+s/m, /^--- FAIL: /m, /^panic: /m], + }, + { + type: "build-typescript", + category: "build", + commandPatterns: [/^tsc\b/i, /^npm\s+run\s+typecheck\b/i], + contentPatterns: [/TS\d{4}:/, /error TS\d{4}/i], + }, + { + type: "build-eslint", + category: "build", + commandPatterns: [/^eslint\b/i, /^npm\s+run\s+lint\b/i], + contentPatterns: [/\s+\d+:\d+\s+(?:error|warning)\s+/, /✖\s+\d+\s+problems?/], + }, + { + type: "build-webpack", + category: "build", + commandPatterns: [/^webpack\b/i, /^npx\s+webpack\b/i, /^npm\s+run\s+build:webpack\b/i], + contentPatterns: [ + /webpack\s+\d/i, + /compiled (?:successfully|with \d+ errors?)/i, + /asset .+\.js/i, + ], + }, + { + type: "build-vite", + category: "build", + commandPatterns: [/^vite\s+build\b/i, /^npm\s+run\s+build\b/i, /^pnpm\s+build\b/i], + contentPatterns: [/vite v[\d.]+/i, /✓ built in/i, /transforming \(\d+\)/i], + }, + { + type: "biome", + category: "build", + commandPatterns: [/^biome\b/i, /^npx\s+biome\b/i], + contentPatterns: [/lint\/[A-Za-z0-9/.-]+/, /Checked \d+ files? in/i], + }, + { + type: "prettier", + category: "build", + commandPatterns: [/^prettier\b/i, /^npx\s+prettier\b/i], + contentPatterns: [/^Checking formatting\.\.\./m, /Code style issues found/i], + }, + { + type: "turbo", + category: "build", + commandPatterns: [/^turbo\b/i, /^npx\s+turbo\b/i], + contentPatterns: [/^• Packages in scope:/m, /^Tasks:\s+\d+\s+successful/m], + }, + { + type: "nx", + category: "build", + commandPatterns: [/^nx\b/i, /^npx\s+nx\b/i], + contentPatterns: [/^NX\s+/m, /^> nx run /m], + }, + { + type: "playwright", + category: "test", + commandPatterns: [/^playwright\s+test\b/i, /^npx\s+playwright\s+test\b/i], + contentPatterns: [/Running \d+ tests? using \d+ workers?/i, /^\s+\d+ failed/m], + }, + { + type: "npm-install", + category: "package", + commandPatterns: [/^(?:npm|pnpm|yarn)\s+(?:install|add|update)\b/i], + contentPatterns: [ + /added \d+ packages/i, + /packages are looking for funding/i, + /audited \d+ packages/i, + ], + }, + { + type: "npm-audit", + category: "package", + commandPatterns: [/^(?:npm|pnpm|yarn)\s+audit\b/i], + contentPatterns: [/found \d+ vulnerabilities/i, /\b(?:low|moderate|high|critical)\b/i], + }, + { + type: "ruff", + category: "build", + commandPatterns: [/^ruff\b/i, /^uv\s+run\s+ruff\b/i], + contentPatterns: [/^[\w./-]+\.py:\d+:\d+:\s+[A-Z]\d+/m, /Found \d+ errors?\./i], + }, + { + type: "mypy", + category: "build", + commandPatterns: [/^mypy\b/i, /^python\s+-m\s+mypy\b/i], + contentPatterns: [/^[\w./-]+\.py:\d+:\s+error:/m, /Found \d+ errors? in \d+ files?/i], + }, + { + type: "pip", + category: "package", + commandPatterns: [/^pip\s+(?:install|download|uninstall)\b/i, /^python\s+-m\s+pip\b/i], + contentPatterns: [/^Collecting /m, /^Successfully installed /m], + }, + { + type: "uv-sync", + category: "package", + commandPatterns: [/^uv\s+sync\b/i, /^uv\s+pip\s+install\b/i], + contentPatterns: [/^Resolved \d+ packages?/m, /^Installed \d+ packages?/m], + }, + { + type: "poetry-install", + category: "package", + commandPatterns: [/^poetry\s+install\b/i], + contentPatterns: [/^Installing dependencies from lock file/m, /^Package operations:/m], + }, + { + type: "golangci-lint", + category: "build", + commandPatterns: [/^golangci-lint\b/i], + contentPatterns: [/^[\w./-]+\.go:\d+:\d+:/m, /^\d+ issues?:/m], + }, + { + type: "bundle-install", + category: "package", + commandPatterns: [/^bundle\s+install\b/i], + contentPatterns: [/^Fetching gem metadata from /m, /^Bundle complete!/m], + }, + { + type: "rubocop", + category: "build", + commandPatterns: [/^rubocop\b/i, /^bundle\s+exec\s+rubocop\b/i], + contentPatterns: [/^Inspecting \d+ files/m, /^[\w./-]+\.rb:\d+:\d+:\s+[A-Z]:/m], + }, + { + type: "docker-ps", + category: "docker", + commandPatterns: [/^docker\s+ps\b/i], + contentPatterns: [/^CONTAINER ID\s+IMAGE\s+COMMAND/m], + }, + { + type: "docker-logs", + category: "docker", + commandPatterns: [/^docker\s+logs\b/i, /^docker\s+compose\s+logs\b/i], + contentPatterns: [ + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/m, + /\b(?:ERROR|WARN|INFO)\b/, + /^Attaching to /m, + ], + }, + { + type: "aws", + category: "cloud", + commandPatterns: [/^aws\b/i], + contentPatterns: [/An error occurred \([A-Za-z0-9]+\) when calling/, /^(?:upload|download): /m], + }, + { + type: "gcloud", + category: "cloud", + commandPatterns: [/^gcloud\b/i], + contentPatterns: [/^ERROR: \(gcloud\./m, /^Updated property \[/m], + }, + { + type: "ssh", + category: "cloud", + commandPatterns: [/^ssh\b/i], + contentPatterns: [ + /Permission denied \(/, + /Host key verification failed/, + /Connection timed out/, + ], + }, + { + type: "rsync", + category: "cloud", + commandPatterns: [/^rsync\b/i], + contentPatterns: [/^sending incremental file list/m, /^rsync error:/m], + }, + { + type: "curl", + category: "cloud", + commandPatterns: [/^curl\b/i], + contentPatterns: [/curl: \(\d+\)/, /^HTTP\/\d(?:\.\d)? \d{3}/m], + }, + { + type: "wget", + category: "cloud", + commandPatterns: [/^wget\b/i], + contentPatterns: [/^--\d{4}-\d{2}-\d{2}/m, /^ERROR \d{3}:/m], + }, + { + type: "json-output", + category: "generic", + commandPatterns: [/^jq\b/i, /^cat\s+.*\.json\b/i], + contentPatterns: [/^\s*[\[{][\s\S]*[\]}]\s*$/], + }, + { + type: "shell-ls", + category: "shell", + commandPatterns: [/^ls(?:\s+-[A-Za-z]+)?\b/i], + contentPatterns: [/^total \d+/m, /^\S+\s+\S+\s+\d+\s+\w+\s+\d{1,2}\s+/m], + }, + { + type: "shell-find", + category: "shell", + commandPatterns: [/^find\b/i], + contentPatterns: [/^(?:\.{1,2}|\/|[\w.-]+\/).+/m], + }, + { + type: "shell-grep", + category: "shell", + commandPatterns: [/^(?:grep|rg|ag)\b/i], + contentPatterns: [ + /^[\w./-]+\.(?:ts|tsx|js|jsx|py|go|rs|java|rb|md|json|ya?ml|txt):\d*:/m, + /^[\w./-]+\/[\w./-]+:\d*:/m, + ], + }, + { + type: "shell-ps", + category: "shell", + commandPatterns: [/^ps\b/i], + contentPatterns: [/^(?:USER\s+PID|\s*PID\s+)/m], + }, + { + type: "shell-df", + category: "shell", + commandPatterns: [/^df\b/i], + contentPatterns: [/^Filesystem\s+.*Use%/m], + }, + { + type: "shell-du", + category: "shell", + commandPatterns: [/^du\b/i], + contentPatterns: [/^\d+(?:\.\d+)?[KMGTP]?\s+\S+/m], + }, + { + type: "error-stacktrace", + category: "generic", + commandPatterns: [], + contentPatterns: [ + /Traceback \(most recent call last\):/, + /^\s+at\s+\S+\s+\(.+:\d+:\d+\)/m, + /^panic: /m, + /^thread '[^']+' panicked at/m, + ], + }, + { + type: "generic-error", + category: "generic", + commandPatterns: [], + contentPatterns: [/Error:/, /Exception:/, /Traceback \(most recent call last\):/], + }, +]; + +export function detectCommandFromText(text: string): string | null { + const firstLines = text.split(/\r?\n/).slice(0, 4); + for (const line of firstLines) { + const trimmed = line.trim().replace(/^\$\s+/, ""); + if (!trimmed) continue; + if (COMMAND_PREFIX_PATTERN.test(trimmed)) { + return trimmed; + } + } + return null; +} + +export function detectCommandType(text: string, command?: string | null): CommandDetectionResult { + const detectedCommand = command?.trim() || detectCommandFromText(text); + let best: CommandDetectionResult | null = null; + + for (const detector of DETECTORS) { + const matchedPatterns: string[] = []; + const commandMatched = + Boolean(detectedCommand) && + detector.commandPatterns.some((pattern) => { + const matched = pattern.test(detectedCommand ?? ""); + if (matched) matchedPatterns.push(pattern.source); + return matched; + }); + const contentMatches = detector.contentPatterns.filter((pattern) => { + const matched = pattern.test(text); + if (matched) matchedPatterns.push(pattern.source); + return matched; + }).length; + if (!commandMatched && contentMatches === 0) continue; + + const confidence = Math.min(1, (commandMatched ? 0.55 : 0) + contentMatches * 0.25); + if (!best || confidence > best.confidence) { + best = { + type: detector.type, + command: detectedCommand, + confidence, + category: detector.category, + matchedPatterns, + }; + } + } + + return ( + best ?? { + type: "unknown", + command: detectedCommand, + confidence: detectedCommand ? 0.35 : 0.1, + category: "generic", + matchedPatterns: [], + } + ); +} + +export function listCommandTypes(): string[] { + return DETECTORS.map((detector) => detector.type); +} + +export const detectCommandOutput = detectCommandType; diff --git a/open-sse/services/compression/engines/rtk/deduplicator.ts b/open-sse/services/compression/engines/rtk/deduplicator.ts new file mode 100644 index 0000000000..ea473097fd --- /dev/null +++ b/open-sse/services/compression/engines/rtk/deduplicator.ts @@ -0,0 +1,37 @@ +export interface DeduplicateOptions { + threshold?: number; +} + +export function deduplicateRepeatedLines( + text: string, + options: DeduplicateOptions = {} +): { + text: string; + collapsed: number; +} { + const threshold = Math.max(2, Math.floor(options.threshold ?? 3)); + const lines = text.split(/\r?\n/); + const output: string[] = []; + let collapsed = 0; + + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + let runLength = 1; + while (index + runLength < lines.length && lines[index + runLength] === line) { + runLength++; + } + + if (line.trim() && runLength >= threshold) { + output.push(line); + output.push(`[line repeated ${runLength - 1}x]`); + output.push(`[rtk:dropped ${runLength - 1} repeated lines]`); + collapsed += runLength - 1; + index += runLength - 1; + continue; + } + + output.push(line); + } + + return { text: output.join("\n"), collapsed }; +} diff --git a/open-sse/services/compression/engines/rtk/filterLoader.ts b/open-sse/services/compression/engines/rtk/filterLoader.ts new file mode 100644 index 0000000000..8c776a9b85 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filterLoader.ts @@ -0,0 +1,220 @@ +import fs from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import os from "node:os"; +import crypto from "node:crypto"; +import { detectCommandType } from "./commandDetector.ts"; +import { validateRtkFilter, type RtkFilterDefinition } from "./filterSchema.ts"; + +let cache: RtkFilterDefinition[] | null = null; +let cacheKey: string | null = null; +let diagnostics: RtkFilterLoadDiagnostic[] = []; + +export interface RtkFilterLoadDiagnostic { + source: "project" | "global" | "builtin"; + path?: string; + level: "warning" | "error"; + message: string; +} + +interface FilterSource { + source: "project" | "global" | "builtin"; + path: string; + trusted: boolean; +} + +interface RtkFilterLoadOptions { + refresh?: boolean; + customFiltersEnabled?: boolean; + trustProjectFilters?: boolean; +} + +function getFiltersDir(): string { + const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + const candidates = [ + path.join(moduleDir, "filters"), + path.join(moduleDir, "..", "services", "compression", "engines", "rtk", "filters"), + path.join(process.cwd(), "open-sse", "services", "compression", "engines", "rtk", "filters"), + path.join( + process.cwd(), + "app", + "open-sse", + "services", + "compression", + "engines", + "rtk", + "filters" + ), + ]; + return ( + candidates.find((candidate, index) => { + return candidates.indexOf(candidate) === index && fs.existsSync(candidate); + }) ?? candidates[0] + ); +} + +function getDataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +function sha256(value: string): string { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function projectFiltersTrusted( + filtersPath: string, + trustProjectFilters = false +): boolean | "changed" { + if (trustProjectFilters) return true; + if (process.env.OMNIROUTE_RTK_TRUST_PROJECT_FILTERS === "1") return true; + const trustPath = path.join(path.dirname(filtersPath), "trust.json"); + if (!fs.existsSync(trustPath)) return false; + try { + const filtersHash = sha256(fs.readFileSync(filtersPath, "utf8")); + const trust = JSON.parse(fs.readFileSync(trustPath, "utf8")) as Record; + const trustedHash = + typeof trust.filtersSha256 === "string" + ? trust.filtersSha256 + : typeof trust.trustedFiltersSha256 === "string" + ? trust.trustedFiltersSha256 + : null; + if (!trustedHash) return false; + return trustedHash === filtersHash ? true : "changed"; + } catch { + return false; + } +} + +function collectFilterSources(options: RtkFilterLoadOptions = {}): FilterSource[] { + const sources: FilterSource[] = []; + const projectPath = path.join(process.cwd(), ".rtk", "filters.json"); + if (options.customFiltersEnabled !== false && fs.existsSync(projectPath)) { + const trusted = projectFiltersTrusted(projectPath, options.trustProjectFilters === true); + if (trusted === true) { + sources.push({ source: "project", path: projectPath, trusted: true }); + } else { + diagnostics.push({ + source: "project", + path: projectPath, + level: "warning", + message: + trusted === "changed" + ? "Project RTK filters changed after trust and were skipped" + : "Project RTK filters are untrusted and were skipped", + }); + } + } + + const globalPath = path.join(getDataDir(), "rtk", "filters.json"); + if (options.customFiltersEnabled !== false && fs.existsSync(globalPath)) { + sources.push({ source: "global", path: globalPath, trusted: true }); + } + + const builtinDir = getFiltersDir(); + if (fs.existsSync(builtinDir)) { + for (const file of fs + .readdirSync(builtinDir) + .filter((entry) => entry.endsWith(".json")) + .sort()) { + sources.push({ + source: "builtin", + path: path.join(builtinDir, file), + trusted: true, + }); + } + } + + return sources; +} + +function parseFilterFile(source: FilterSource): RtkFilterDefinition[] { + try { + const parsed = JSON.parse(fs.readFileSync(source.path, "utf8")); + const entries = Array.isArray(parsed) ? parsed : [parsed]; + return entries.map(validateRtkFilter); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (source.source === "builtin") { + throw new Error(`Invalid RTK filter ${path.basename(source.path)}: ${message}`); + } + diagnostics.push({ + source: source.source, + path: source.path, + level: "warning", + message: `Invalid custom RTK filter skipped: ${message}`, + }); + return []; + } +} + +export function loadRtkFilters(options: RtkFilterLoadOptions = {}): RtkFilterDefinition[] { + const currentCacheKey = [ + process.cwd(), + getDataDir(), + options.customFiltersEnabled === false ? "builtin-only" : "custom", + options.trustProjectFilters === true ? "trusted-project" : "trust-file", + process.env.OMNIROUTE_RTK_TRUST_PROJECT_FILTERS === "1" ? "env-trust" : "env-normal", + ].join("|"); + if (cache && cacheKey === currentCacheKey && !options.refresh) return cache; + diagnostics = []; + const filters: RtkFilterDefinition[] = []; + for (const source of collectFilterSources(options)) { + filters.push(...parseFilterFile(source)); + } + + const sorted = filters.sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id)); + cache = sorted; + cacheKey = currentCacheKey; + return sorted; +} + +export function getRtkFilterLoadDiagnostics(): RtkFilterLoadDiagnostic[] { + loadRtkFilters(); + return diagnostics.map((entry) => ({ ...entry })); +} + +export function getRtkFilterCatalog(): Array< + Pick< + RtkFilterDefinition, + "id" | "name" | "description" | "commandTypes" | "category" | "priority" + > +> { + return loadRtkFilters().map((filter) => ({ + id: filter.id, + name: filter.name, + description: filter.description, + commandTypes: filter.commandTypes, + category: filter.category, + priority: filter.priority, + })); +} + +export function matchRtkFilter( + text: string, + command?: string | null, + options: RtkFilterLoadOptions = {} +): RtkFilterDefinition | null { + const detection = detectCommandType(text, command); + const detectedCommand = detection.command ?? command ?? ""; + const matchesPattern = (pattern: string, value: string): boolean => { + try { + return new RegExp(pattern, "im").test(value); + } catch { + return false; + } + }; + const filters = loadRtkFilters(options); + return ( + filters.find((filter) => filter.commandTypes.includes(detection.type)) ?? + filters.find( + (filter) => + detectedCommand && + filter.commandPatterns.some((pattern) => matchesPattern(pattern, detectedCommand)) + ) ?? + filters.find((filter) => + filter.matchPatterns.some((pattern) => matchesPattern(pattern, text)) + ) ?? + filters.find((filter) => filter.commandTypes.includes("generic-output")) ?? + null + ); +} diff --git a/open-sse/services/compression/engines/rtk/filterSchema.ts b/open-sse/services/compression/engines/rtk/filterSchema.ts new file mode 100644 index 0000000000..bb5f67d974 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filterSchema.ts @@ -0,0 +1,182 @@ +import { z } from "zod"; + +const rtkFilterCategorySchema = z.enum([ + "git", + "test", + "build", + "shell", + "docker", + "package", + "infra", + "cloud", + "generic", +]); + +const rtkReplaceRuleSchema = z + .object({ + pattern: z.string().min(1), + replacement: z.string(), + }) + .strict(); + +const rtkMatchOutputRuleSchema = z + .object({ + pattern: z.string().min(1), + message: z.string(), + unless: z.string().min(1).optional(), + }) + .strict(); + +const rtkInlineTestSchema = z + .object({ + name: z.string().min(1), + input: z.string(), + expected: z.string(), + command: z.string().optional(), + }) + .strict(); + +const rtkFilterMatchSchema = z + .object({ + commands: z.array(z.string().min(1)).default([]), + patterns: z.array(z.string().min(1)).default([]), + outputTypes: z.array(z.string().min(1)).default([]), + }) + .strict(); + +const rtkFilterRulesSchema = z + .object({ + stripAnsi: z.boolean().default(false), + replace: z.array(rtkReplaceRuleSchema).default([]), + matchOutput: z.array(rtkMatchOutputRuleSchema).default([]), + includePatterns: z.array(z.string()).default([]), + dropPatterns: z.array(z.string()).default([]), + collapsePatterns: z.array(z.string()).default([]), + deduplicate: z.boolean().default(false), + truncateLineAt: z.number().int().min(0).default(0), + maxLines: z.number().int().min(0).default(0), + headLines: z.number().int().min(0).default(20), + tailLines: z.number().int().min(0).default(20), + onEmpty: z.string().default(""), + filterStderr: z.boolean().default(false), + }) + .strict(); + +const rtkFilterPreserveSchema = z + .object({ + errorPatterns: z.array(z.string()).default([]), + summaryPatterns: z.array(z.string()).default([]), + }) + .strict(); + +export const rtkFilterPackSchema = z + .object({ + id: z.string().min(1), + label: z.string().min(1), + description: z.string().default(""), + category: rtkFilterCategorySchema, + priority: z.number().int().min(0).max(100).default(50), + match: rtkFilterMatchSchema, + rules: rtkFilterRulesSchema.default({}), + preserve: rtkFilterPreserveSchema.default({}), + tests: z.array(rtkInlineTestSchema).default([]), + }) + .strict(); + +const legacyRtkFilterSchema = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + description: z.string().default(""), + commandTypes: z.array(z.string().min(1)).min(1), + category: rtkFilterCategorySchema, + priority: z.number().int().min(0).max(100).default(50), + stripPatterns: z.array(z.string()).default([]), + keepPatterns: z.array(z.string()).default([]), + priorityPatterns: z.array(z.string()).default([]), + collapsePatterns: z.array(z.string()).default([]), + stripAnsi: z.boolean().default(false), + replace: z.array(rtkReplaceRuleSchema).default([]), + matchOutput: z.array(rtkMatchOutputRuleSchema).default([]), + truncateLineAt: z.number().int().min(0).default(0), + onEmpty: z.string().default(""), + filterStderr: z.boolean().default(false), + maxLines: z.number().int().min(0).default(0), + preserveHead: z.number().int().min(0).default(20), + preserveTail: z.number().int().min(0).default(20), + tests: z.array(rtkInlineTestSchema).default([]), + }) + .strict(); + +export const rtkFilterSchema = z.union([rtkFilterPackSchema, legacyRtkFilterSchema]); + +export type RtkFilterPack = z.infer; + +export interface RtkFilterDefinition { + id: string; + name: string; + description: string; + commandTypes: string[]; + commandPatterns: string[]; + matchPatterns: string[]; + category: z.infer; + priority: number; + stripPatterns: string[]; + keepPatterns: string[]; + priorityPatterns: string[]; + collapsePatterns: string[]; + stripAnsi: boolean; + replace: Array<{ pattern: string; replacement: string }>; + matchOutput: Array<{ pattern: string; message: string; unless?: string }>; + truncateLineAt: number; + onEmpty: string; + filterStderr: boolean; + deduplicate: boolean; + maxLines: number; + preserveHead: number; + preserveTail: number; + tests: Array<{ name: string; input: string; expected: string; command?: string }>; +} + +function isCanonicalFilter(value: z.infer): value is RtkFilterPack { + return "label" in value && "match" in value && "rules" in value; +} + +export function validateRtkFilter(value: unknown): RtkFilterDefinition { + const parsed = rtkFilterSchema.parse(value); + if (!isCanonicalFilter(parsed)) { + return { + ...parsed, + commandPatterns: [], + matchPatterns: [], + deduplicate: parsed.collapsePatterns.length > 0, + }; + } + + const preservePatterns = [...parsed.preserve.errorPatterns, ...parsed.preserve.summaryPatterns]; + return { + id: parsed.id, + name: parsed.label, + description: parsed.description, + commandTypes: parsed.match.outputTypes, + commandPatterns: parsed.match.commands, + matchPatterns: parsed.match.patterns, + category: parsed.category, + priority: parsed.priority, + stripPatterns: parsed.rules.dropPatterns, + keepPatterns: parsed.rules.includePatterns, + priorityPatterns: preservePatterns, + collapsePatterns: parsed.rules.collapsePatterns, + stripAnsi: parsed.rules.stripAnsi, + replace: parsed.rules.replace, + matchOutput: parsed.rules.matchOutput, + truncateLineAt: parsed.rules.truncateLineAt, + onEmpty: parsed.rules.onEmpty, + filterStderr: parsed.rules.filterStderr, + deduplicate: parsed.rules.deduplicate, + maxLines: parsed.rules.maxLines, + preserveHead: parsed.rules.headLines, + preserveTail: parsed.rules.tailLines, + tests: parsed.tests, + }; +} diff --git a/open-sse/services/compression/engines/rtk/filters/aws.json b/open-sse/services/compression/engines/rtk/filters/aws.json new file mode 100644 index 0000000000..7a3f82d887 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/aws.json @@ -0,0 +1,34 @@ +{ + "id": "aws", + "label": "AWS CLI", + "description": "Compress AWS CLI progress output while preserving errors and completed transfers.", + "category": "cloud", + "priority": 76, + "match": { + "outputTypes": ["aws"], + "commands": ["^aws\\b"], + "patterns": ["An error occurred \\([A-Za-z0-9]+\\) when calling", "^(?:upload|download): "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Completed \\d", "^\\s*$"], + "includePatterns": ["^(?:upload|download): ", "An error occurred \\("], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["An error occurred", "AccessDenied", "Throttling", "ValidationError"], + "summaryPatterns": ["^(?:upload|download): "] + }, + "tests": [ + { + "name": "inline sample", + "command": "aws s3 cp", + "input": "Completed 1.0 MiB/2.0 MiB\nupload: ./app.log to s3://bucket/app.log\nAn error occurred (AccessDenied) when calling the PutObject operation: denied\n", + "expected": "upload: ./app.log to s3://bucket/app.log\nAn error occurred (AccessDenied) when calling the PutObject operation: denied" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/biome.json b/open-sse/services/compression/engines/rtk/filters/biome.json new file mode 100644 index 0000000000..d0c7564b39 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/biome.json @@ -0,0 +1,35 @@ +{ + "id": "biome", + "label": "Biome", + "description": "Compact Biome lint/format output.", + "category": "build", + "priority": 80, + "match": { + "outputTypes": [], + "commands": ["^biome\\b", "^npx\\s+biome\\b"], + "patterns": ["Checked \\d+ files?", "biome"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^Checked \\d+ files? in"], + "includePatterns": ["error", "warning", "✖", "Checked", "Fixed", "^\\s*>"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40, + "onEmpty": "biome: ok" + }, + "preserve": { + "errorPatterns": ["error", "✖"], + "summaryPatterns": ["Checked", "Fixed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "biome", + "input": "Checked 10 files in 20ms\nerror lint/a.ts:1:1 boom\nFixed 1 file\n", + "expected": "error lint/a.ts:1:1 boom\nFixed 1 file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-eslint.json b/open-sse/services/compression/engines/rtk/filters/build-eslint.json new file mode 100644 index 0000000000..610e616363 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-eslint.json @@ -0,0 +1,37 @@ +{ + "id": "build-eslint", + "label": "ESLint", + "description": "Keep lint diagnostics and problem summary.", + "category": "build", + "priority": 88, + "match": { + "outputTypes": ["build-eslint"], + "commands": ["^(?:eslint|npm\\s+run\\s+lint)\\b"], + "patterns": ["\\s+\\d+:\\d+\\s+(?:error|warning)\\s+", "✖\\s+\\d+\\s+problems?"] + }, + "rules": { + "includePatterns": [ + "\\s+\\d+:\\d+\\s+(error|warning)\\s+", + "✖\\s+\\d+\\s+problems?", + "^/.*\\.(ts|tsx|js|jsx)$" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["error", "warning", "problems?"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:eslint|npm run lint)", + "input": "/repo/a.ts\n 1:1 error boom no-boom\n✖ 1 problem (1 error, 0 warnings)\n", + "expected": "/repo/a.ts\n 1:1 error boom no-boom\n✖ 1 problem (1 error, 0 warnings)" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-typescript.json b/open-sse/services/compression/engines/rtk/filters/build-typescript.json new file mode 100644 index 0000000000..5ef3e314c5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-typescript.json @@ -0,0 +1,33 @@ +{ + "id": "build-typescript", + "label": "TypeScript", + "description": "Keep TypeScript diagnostics and file locations.", + "category": "build", + "priority": 93, + "match": { + "outputTypes": ["build-typescript"], + "commands": ["^(?:tsc|npm\\s+run\\s+typecheck)\\b"], + "patterns": ["TS\\d{4}:", "error TS\\d{4}"] + }, + "rules": { + "includePatterns": ["TS\\d{4}:", "error TS\\d{4}", "^Found \\d+ errors?"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["TS\\d{4}", "Found \\d+"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:tsc|npm run typecheck)", + "input": "src/a.ts:1:1 - error TS2322: Type string is not assignable\nFound 1 error.\n", + "expected": "src/a.ts:1:1 - error TS2322: Type string is not assignable\nFound 1 error." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-vite.json b/open-sse/services/compression/engines/rtk/filters/build-vite.json new file mode 100644 index 0000000000..7e81845b81 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-vite.json @@ -0,0 +1,41 @@ +{ + "id": "build-vite", + "label": "Vite Build", + "description": "Keep Vite errors and final build summary.", + "category": "build", + "priority": 83, + "match": { + "outputTypes": ["build-vite"], + "commands": ["^vite\\s+build\\b", "^npm\\s+run\\s+build\\b", "^pnpm\\s+build\\b"], + "patterns": ["vite v[\\d.]+", "✓ built in"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "✓ built in", + "message": "vite: build ok", + "unless": "error|failed" + } + ], + "includePatterns": ["error", "failed", "✓ built in", "dist/", "rendering chunks"], + "dropPatterns": ["^transforming ", "^✓ \\d+ modules transformed", "^\\s*$"], + "collapsePatterns": ["^dist/"], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["✓ built in"] + }, + "tests": [ + { + "name": "inline sample", + "command": "vite build", + "input": "vite v5.0.0 building for production...\n✓ 10 modules transformed.\n✓ built in 100ms\n", + "expected": "vite: build ok" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/build-webpack.json b/open-sse/services/compression/engines/rtk/filters/build-webpack.json new file mode 100644 index 0000000000..55e1eb845b --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/build-webpack.json @@ -0,0 +1,34 @@ +{ + "id": "build-webpack", + "label": "Webpack Build", + "description": "Keep webpack errors/warnings and compilation summary.", + "category": "build", + "priority": 84, + "match": { + "outputTypes": ["build-webpack"], + "commands": ["^webpack\\b", "^npx\\s+webpack\\b"], + "patterns": ["webpack \\d", "compiled (?:successfully|with)"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": ["ERROR in", "WARNING in", "compiled ", "asset ", "Entrypoint"], + "dropPatterns": ["^\\s*$", "^cacheable modules"], + "collapsePatterns": ["^asset "], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["ERROR in"], + "summaryPatterns": ["compiled "] + }, + "tests": [ + { + "name": "inline sample", + "command": "webpack", + "input": "asset main.js 10 KiB\nERROR in ./src/a.ts\ncompiled with 1 error\n", + "expected": "asset main.js 10 KiB\nERROR in ./src/a.ts\ncompiled with 1 error" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/bundle-install.json b/open-sse/services/compression/engines/rtk/filters/bundle-install.json new file mode 100644 index 0000000000..7b126ae117 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/bundle-install.json @@ -0,0 +1,35 @@ +{ + "id": "bundle-install", + "label": "Bundle Install", + "description": "Strip bundle install noise and keep final status.", + "category": "package", + "priority": 70, + "match": { + "outputTypes": [], + "commands": ["^bundle\\s+install\\b"], + "patterns": ["Bundle complete", "Using "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Using ", "^Fetching ", "^Installing ", "^\\s*$"], + "includePatterns": ["Bundle complete", "Your bundle is complete", "ERROR", "Gem::"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "bundle: ok" + }, + "preserve": { + "errorPatterns": ["ERROR", "Gem::"], + "summaryPatterns": ["Bundle complete", "Your bundle is complete"] + }, + "tests": [ + { + "name": "inline sample", + "command": "bundle install", + "input": "Using rake 13.0\nInstalling pg 1.5\nBundle complete! 4 Gemfile dependencies\n", + "expected": "Bundle complete! 4 Gemfile dependencies" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/curl.json b/open-sse/services/compression/engines/rtk/filters/curl.json new file mode 100644 index 0000000000..936f290706 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/curl.json @@ -0,0 +1,34 @@ +{ + "id": "curl", + "label": "curl", + "description": "Remove curl progress noise and keep HTTP status/errors.", + "category": "cloud", + "priority": 70, + "match": { + "outputTypes": ["curl"], + "commands": ["^curl\\b"], + "patterns": ["curl: \\(\\d+\\)", "^HTTP/\\d(?:\\.\\d)? \\d{3}"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*% Total\\s+% Received", "^\\s*\\d+\\s+\\d+", "^\\s*$"], + "includePatterns": ["^HTTP/\\d(?:\\.\\d)? \\d{3}", "^curl: \\(\\d+\\)", "^[<{]"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["curl: \\(\\d+\\)", "^HTTP/\\d(?:\\.\\d)? [45]\\d\\d"], + "summaryPatterns": ["^HTTP/\\d(?:\\.\\d)? \\d{3}"] + }, + "tests": [ + { + "name": "inline sample", + "command": "curl -i", + "input": "% Total % Received\n 0 0 0 0\nHTTP/1.1 500 Internal Server Error\ncurl: (22) The requested URL returned error: 500\n", + "expected": "HTTP/1.1 500 Internal Server Error\ncurl: (22) The requested URL returned error: 500" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/df.json b/open-sse/services/compression/engines/rtk/filters/df.json new file mode 100644 index 0000000000..5be798a273 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/df.json @@ -0,0 +1,34 @@ +{ + "id": "df", + "label": "df", + "description": "Keep filesystem usage table headers and mounted filesystems.", + "category": "shell", + "priority": 67, + "match": { + "outputTypes": ["shell-df"], + "commands": ["^df\\b"], + "patterns": ["^Filesystem\\s+.*Use%"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^tmpfs\\s", "^udev\\s", "^\\s*$"], + "includePatterns": ["^Filesystem\\s+", "^/dev/\\S+\\s+"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 25, + "tailLines": 25 + }, + "preserve": { + "errorPatterns": ["No such file", "Permission denied"], + "summaryPatterns": ["Use%", "^/dev/"] + }, + "tests": [ + { + "name": "inline sample", + "command": "df -h", + "input": "Filesystem Size Used Avail Use% Mounted on\nudev 2.0G 0 2.0G 0% /dev\n/dev/sda1 50G 45G 5.0G 90% /\n", + "expected": "Filesystem Size Used Avail Use% Mounted on\n/dev/sda1 50G 45G 5.0G 90% /" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/docker-logs.json b/open-sse/services/compression/engines/rtk/filters/docker-logs.json new file mode 100644 index 0000000000..10b5d11f43 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/docker-logs.json @@ -0,0 +1,42 @@ +{ + "id": "docker-logs", + "label": "Docker Logs", + "description": "Keep warnings/errors and recent Docker log context.", + "category": "docker", + "priority": 84, + "match": { + "outputTypes": ["docker-logs"], + "commands": ["^docker\\s+(?:compose\\s+)?logs\\b"], + "patterns": ["\\b(?:ERROR|WARN|INFO)\\b", "^Attaching to "] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Attaching to ", "^\\s*$"], + "includePatterns": [ + "ERROR", + "WARN", + "Exception", + "Traceback", + "failed", + "listening", + "started" + ], + "collapsePatterns": ["INFO"], + "deduplicate": true, + "maxLines": 120, + "headLines": 20, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["ERROR", "Exception", "failed"], + "summaryPatterns": ["started", "listening"] + }, + "tests": [ + { + "name": "inline sample", + "command": "docker (?:compose )?logs", + "input": "Attaching to api\nINFO started\nINFO started\nERROR failed to connect\n", + "expected": "INFO started\nERROR failed to connect" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/docker-ps.json b/open-sse/services/compression/engines/rtk/filters/docker-ps.json new file mode 100644 index 0000000000..b7dc4efee0 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/docker-ps.json @@ -0,0 +1,33 @@ +{ + "id": "docker-ps", + "label": "Docker PS", + "description": "Keep docker container table header and rows under a fixed cap.", + "category": "docker", + "priority": 60, + "match": { + "outputTypes": ["docker-ps"], + "commands": ["^docker\\s+ps\\b"], + "patterns": ["^CONTAINER ID\\s+IMAGE\\s+COMMAND"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 30, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^CONTAINER ID"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "docker ps", + "input": "CONTAINER ID IMAGE COMMAND\nabc123 node \"npm start\"\n", + "expected": "CONTAINER ID IMAGE COMMAND\nabc123 node \"npm start\"" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/du.json b/open-sse/services/compression/engines/rtk/filters/du.json new file mode 100644 index 0000000000..54cca64e89 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/du.json @@ -0,0 +1,33 @@ +{ + "id": "du", + "label": "du", + "description": "Summarize disk-usage listings while preserving largest entries.", + "category": "shell", + "priority": 66, + "match": { + "outputTypes": ["shell-du"], + "commands": ["^du\\b"], + "patterns": ["^\\d+(?:\\.\\d+)?[KMGTP]?\\s+\\S+"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["Permission denied", "No such file"], + "summaryPatterns": ["\\s+\\.$", "\\s+total$"] + }, + "tests": [ + { + "name": "inline sample", + "command": "du -sh *", + "input": "4.0K package.json\n120M node_modules\n120M node_modules\n", + "expected": "4.0K package.json\n120M node_modules\n120M node_modules" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json b/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json new file mode 100644 index 0000000000..0d5fd3b8bf --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/error-stacktrace.json @@ -0,0 +1,41 @@ +{ + "id": "error-stacktrace", + "label": "Error Stacktrace", + "description": "Collapse stack traces while preserving error text and top frames.", + "category": "generic", + "priority": 70, + "match": { + "outputTypes": ["error-stacktrace"], + "commands": [], + "patterns": ["Traceback \\(most recent call last\\):", "^\\s+at\\s+", "^panic: "] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "Error:", + "Exception:", + "Traceback", + "^\\s+at ", + "^panic:", + "^thread ", + "^Caused by:" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 80, + "headLines": 25, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["Error:", "Exception:", "^panic:", "Traceback"], + "summaryPatterns": ["Caused by:"] + }, + "tests": [ + { + "name": "inline sample", + "input": "Error: boom\n at fn (/repo/a.ts:1:1)\n at fn (/repo/a.ts:1:1)\nCaused by: bad\n", + "expected": "Error: boom\n at fn (/repo/a.ts:1:1)\nCaused by: bad" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/gcloud.json b/open-sse/services/compression/engines/rtk/filters/gcloud.json new file mode 100644 index 0000000000..19c5dd09a9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/gcloud.json @@ -0,0 +1,34 @@ +{ + "id": "gcloud", + "label": "gcloud", + "description": "Compress Google Cloud CLI output and preserve actionable errors.", + "category": "cloud", + "priority": 75, + "match": { + "outputTypes": ["gcloud"], + "commands": ["^gcloud\\b"], + "patterns": ["^ERROR: \\(gcloud\\.", "^Updated property \\["] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Waiting for operation", "^\\s*$"], + "includePatterns": ["^ERROR: \\(gcloud\\.", "^Updated property \\[", "^Created \\["], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^ERROR: \\(gcloud\\.", "PERMISSION_DENIED", "NOT_FOUND"], + "summaryPatterns": ["^Updated property \\[", "^Created \\["] + }, + "tests": [ + { + "name": "inline sample", + "command": "gcloud run deploy", + "input": "Waiting for operation [deploy]\nUpdated property [core/project].\nERROR: (gcloud.run.deploy) PERMISSION_DENIED: denied\n", + "expected": "Updated property [core/project].\nERROR: (gcloud.run.deploy) PERMISSION_DENIED: denied" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/generic-output.json b/open-sse/services/compression/engines/rtk/filters/generic-output.json new file mode 100644 index 0000000000..da93022888 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/generic-output.json @@ -0,0 +1,32 @@ +{ + "id": "generic-output", + "label": "Generic Output", + "description": "Fallback filter for unknown command output.", + "category": "generic", + "priority": 10, + "match": { + "outputTypes": ["generic-output", "generic-error"], + "commands": [], + "patterns": ["Error:", "Exception:", "Traceback \\(most recent call last\\):"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 35, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["error", "failed", "exception", "traceback"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "input": "\nError: boom\n\nTraceback (most recent call last):\n file.py:1\n", + "expected": "Error: boom\nTraceback (most recent call last):\n file.py:1" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-branch.json b/open-sse/services/compression/engines/rtk/filters/git-branch.json new file mode 100644 index 0000000000..b333932f08 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-branch.json @@ -0,0 +1,45 @@ +{ + "id": "git-branch", + "label": "Git Branch", + "description": "Compact git branch/switch output.", + "category": "git", + "priority": 94, + "match": { + "outputTypes": ["git-branch"], + "commands": ["^git\\s+(?:branch|checkout|switch)\\b"], + "patterns": ["^\\*\\s+\\S+", "Switched to (?:a new )?branch", "Already on"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "Switched to (?:a new )?branch", + "message": "git: switched branch", + "unless": "error|failed" + }, + { + "pattern": "Already on", + "message": "git: already on branch" + } + ], + "includePatterns": ["^\\*\\s+\\S+", "^\\s{2}\\S+", "Switched to", "Already on"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 60, + "headLines": 20, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["Switched to", "Already on"] + }, + "tests": [ + { + "name": "inline sample", + "command": "git (?:branch|checkout|switch)", + "input": " main\n* feature/rtk\n release\n", + "expected": " main\n* feature/rtk\n release" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-diff.json b/open-sse/services/compression/engines/rtk/filters/git-diff.json new file mode 100644 index 0000000000..a97df1fbac --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-diff.json @@ -0,0 +1,33 @@ +{ + "id": "git-diff", + "label": "Git Diff", + "description": "Drop low-signal diff metadata while preserving files, hunks, and changed lines.", + "category": "git", + "priority": 94, + "match": { + "outputTypes": ["git-diff"], + "commands": ["^git\\s+(?:diff|show)\\b"], + "patterns": ["^diff --git ", "^@@\\s+-\\d+,\\d+\\s+\\+\\d+,\\d+\\s+@@"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^index [0-9a-f]+\\.\\.[0-9a-f]+.*$", "^--- a/", "^\\+\\+\\+ b/"], + "collapsePatterns": ["^\\s*$"], + "deduplicate": true, + "maxLines": 180, + "headLines": 40, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["^diff --git ", "^@@ ", "^[+-](?![+-]{2})"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git (?:diff|show)", + "input": "diff --git a/a.ts b/a.ts\n@@ -1,2 +1,2 @@\n-console.log(\"old\")\n+console.log(\"new\")\n", + "expected": "diff --git a/a.ts b/a.ts\n@@ -1,2 +1,2 @@\n-console.log(\"old\")\n+console.log(\"new\")\n" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-log.json b/open-sse/services/compression/engines/rtk/filters/git-log.json new file mode 100644 index 0000000000..4a54160537 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-log.json @@ -0,0 +1,33 @@ +{ + "id": "git-log", + "label": "Git Log", + "description": "Keep commit identifiers and subjects from verbose logs.", + "category": "git", + "priority": 70, + "match": { + "outputTypes": ["git-log"], + "commands": ["^git\\s+log\\b"], + "patterns": ["^commit [0-9a-f]{7,40}", "^Author: "] + }, + "rules": { + "includePatterns": ["^commit [0-9a-f]{7,40}", "^\\s{4}\\S"], + "dropPatterns": ["^Author: ", "^Date: ", "^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 40, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^commit "], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git log", + "input": "commit abcdef1234567890\nAuthor: Dev \nDate: today\n\n feat: add rtk\n", + "expected": "commit abcdef1234567890\n feat: add rtk" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/git-status.json b/open-sse/services/compression/engines/rtk/filters/git-status.json new file mode 100644 index 0000000000..d2d21a66f5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/git-status.json @@ -0,0 +1,40 @@ +{ + "id": "git-status", + "label": "Git Status", + "description": "Keep branch, staged/unstaged headings, and changed file lines.", + "category": "git", + "priority": 95, + "match": { + "outputTypes": ["git-status"], + "commands": ["^git\\s+status\\b"], + "patterns": ["^On branch ", "^Changes (?:not staged|to be committed)", "^Untracked files:"] + }, + "rules": { + "includePatterns": [ + "^On branch ", + "^Your branch ", + "^Changes ", + "^Untracked files:", + "^\\s*(modified|new file|deleted|renamed|both modified):", + "^\\s*[MADRCU?!]{1,2}\\s+" + ], + "dropPatterns": ["^\\s*$", "^\\s*\\(use .*\\)$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 12, + "tailLines": 20 + }, + "preserve": { + "errorPatterns": ["^Changes ", "^Untracked files:"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "git status", + "input": "On branch main\nChanges not staged for commit:\n (use \"git add\" to update)\n\tmodified: src/app.ts\nnothing added to commit\n", + "expected": "On branch main\nChanges not staged for commit:\n\tmodified: src/app.ts" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/golangci-lint.json b/open-sse/services/compression/engines/rtk/filters/golangci-lint.json new file mode 100644 index 0000000000..8bffbee1af --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/golangci-lint.json @@ -0,0 +1,34 @@ +{ + "id": "golangci-lint", + "label": "golangci-lint", + "description": "Keep golangci-lint findings and summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^golangci-lint\\b"], + "patterns": ["issues:", "level=error", ":\\d+:\\d+:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [":\\d+:\\d+:", "level=error", "issues:", "Error:"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["level=error", "Error:"], + "summaryPatterns": ["issues:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "golangci-lint", + "input": "main.go:1:1: issue text\nlevel=error msg=\"runner failed\"\nissues: 1\n", + "expected": "main.go:1:1: issue text\nlevel=error msg=\"runner failed\"\nissues: 1" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/json-output.json b/open-sse/services/compression/engines/rtk/filters/json-output.json new file mode 100644 index 0000000000..1ab8769266 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/json-output.json @@ -0,0 +1,40 @@ +{ + "id": "json-output", + "label": "JSON Output", + "description": "Keep actionable JSON status, error and message fields while preserving structure.", + "category": "generic", + "priority": 70, + "match": { + "outputTypes": ["json-output"], + "commands": ["^(?:jq\\b|cat\\s+.*\\.json\\b)"], + "patterns": ["^\\s*[\\[{][\\s\\S]*[\\]}]\\s*$"] + }, + "rules": { + "includePatterns": [ + "\"status\"\\s*:", + "\"error\"\\s*:", + "\"errors\"\\s*:", + "\"code\"\\s*:", + "\"message\"\\s*:", + "[{}\\[\\],]" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["\"error\"", "\"errors\"", "\"message\""], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:jq\\b|cat .*\\.json\\b)", + "input": "{\n \"status\": \"ok\",\n \"items\": [1,2,3]\n}\n", + "expected": "{\n \"status\": \"ok\",\n \"items\": [1,2,3]\n}" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/make.json b/open-sse/services/compression/engines/rtk/filters/make.json new file mode 100644 index 0000000000..7b9d3a6b24 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/make.json @@ -0,0 +1,38 @@ +{ + "id": "make", + "label": "Make", + "description": "Strip make directory chatter and empty lines.", + "category": "build", + "priority": 78, + "match": { + "outputTypes": [], + "commands": ["^make\\b"], + "patterns": ["make\\[\\d+\\]: (?:Entering|Leaving) directory"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": [ + "^make\\[\\d+\\]: (?:Entering|Leaving) directory", + "^\\s*$", + "Nothing to be done for" + ], + "collapsePatterns": ["^(?:cc|gcc|clang|g\\+\\+)\\b"], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40, + "onEmpty": "make: ok" + }, + "preserve": { + "errorPatterns": ["error:", "failed", "\\*\\*\\*"], + "summaryPatterns": ["built", "linking"] + }, + "tests": [ + { + "name": "inline sample", + "command": "make", + "input": "make[1]: Entering directory '/repo'\ngcc -O2 foo.c\nmake[1]: Leaving directory '/repo'\n", + "expected": "gcc -O2 foo.c" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/mypy.json b/open-sse/services/compression/engines/rtk/filters/mypy.json new file mode 100644 index 0000000000..861bbe3e9b --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/mypy.json @@ -0,0 +1,34 @@ +{ + "id": "mypy", + "label": "mypy", + "description": "Keep mypy errors grouped by file and final summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^mypy\\b", "^python\\s+-m\\s+mypy\\b"], + "patterns": [": error:", "Found \\d+ errors? in"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [": error:", ": note:", "Found \\d+ errors?", "Success: no issues found"], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 160, + "headLines": 40, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": [": error:", "Found \\d+ errors?"], + "summaryPatterns": ["Success:", "Found \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "mypy", + "input": "app.py:1: error: Incompatible types\nFound 1 error in 1 file\n", + "expected": "app.py:1: error: Incompatible types\nFound 1 error in 1 file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/npm-audit.json b/open-sse/services/compression/engines/rtk/filters/npm-audit.json new file mode 100644 index 0000000000..19fa224a38 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/npm-audit.json @@ -0,0 +1,43 @@ +{ + "id": "npm-audit", + "label": "npm audit", + "description": "Keep vulnerability counts and remediation hints.", + "category": "package", + "priority": 84, + "match": { + "outputTypes": ["npm-audit"], + "commands": ["^(?:npm|pnpm|yarn)\\s+audit\\b"], + "patterns": ["found \\d+ vulnerabilities", "\\bcritical\\b"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "vulnerabilities", + "critical", + "high", + "moderate", + "low", + "fix available", + "npm audit fix", + "No vulnerabilities" + ], + "dropPatterns": ["^\\s*$", "^# npm audit report"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": ["critical", "high"], + "summaryPatterns": ["vulnerabilities", "No vulnerabilities"] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:npm|pnpm|yarn) audit", + "input": "# npm audit report\ncritical vulnerability in foo\nfound 1 vulnerabilities\nnpm audit fix\n", + "expected": "critical vulnerability in foo\nfound 1 vulnerabilities\nnpm audit fix" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/npm-install.json b/open-sse/services/compression/engines/rtk/filters/npm-install.json new file mode 100644 index 0000000000..df001e67b2 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/npm-install.json @@ -0,0 +1,41 @@ +{ + "id": "npm-install", + "label": "Package Install", + "description": "Keep install summaries, warnings and audit results.", + "category": "package", + "priority": 65, + "match": { + "outputTypes": ["npm-install"], + "commands": ["^(?:npm|pnpm|yarn)\\s+(?:install|add|update)\\b"], + "patterns": ["added \\d+ packages", "packages are looking for funding", "audited \\d+ packages"] + }, + "rules": { + "includePatterns": [ + "added \\d+ packages", + "removed \\d+ packages", + "changed \\d+ packages", + "audited \\d+ packages", + "vulnerabilities", + "WARN", + "ERR!" + ], + "dropPatterns": ["^\\s*$", "^\\s*\\|", "^Progress:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["WARN", "ERR", "vulnerabilities"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:npm|pnpm|yarn) (?:install|add|update)", + "input": "added 3 packages\npackages are looking for funding\naudited 3 packages\n", + "expected": "added 3 packages\naudited 3 packages" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/nx.json b/open-sse/services/compression/engines/rtk/filters/nx.json new file mode 100644 index 0000000000..6973433228 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/nx.json @@ -0,0 +1,34 @@ +{ + "id": "nx", + "label": "Nx", + "description": "Compact Nx task graph and output.", + "category": "build", + "priority": 76, + "match": { + "outputTypes": [], + "commands": ["^nx\\b", "^npx\\s+nx\\b"], + "patterns": ["NX ", "Successfully ran target"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^> nx run"], + "includePatterns": ["NX ", "ERROR", "Failed", "Successfully ran", "Running target"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 25, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["ERROR", "Failed"], + "summaryPatterns": ["Successfully ran"] + }, + "tests": [ + { + "name": "inline sample", + "command": "nx", + "input": "> nx run app:build\nNX Failed tasks:\nERROR boom\n", + "expected": "NX Failed tasks:\nERROR boom" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/pip.json b/open-sse/services/compression/engines/rtk/filters/pip.json new file mode 100644 index 0000000000..91edf38ca3 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/pip.json @@ -0,0 +1,41 @@ +{ + "id": "pip", + "label": "pip", + "description": "Strip pip install progress and keep install/failure summary.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^pip\\s+(?:install|sync|download)\\b", "^python\\s+-m\\s+pip\\s+install\\b"], + "patterns": ["Successfully installed", "Requirement already satisfied"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": [ + "^Collecting ", + "^Downloading ", + "^Installing collected packages:", + "^Requirement already satisfied", + "^\\s*$" + ], + "includePatterns": ["Successfully installed", "ERROR:", "WARNING:", "failed", "Could not find"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "pip: ok" + }, + "preserve": { + "errorPatterns": ["ERROR:", "failed", "Could not find"], + "summaryPatterns": ["Successfully installed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "pip (?:install|sync|download)", + "input": "Collecting requests\nDownloading requests.whl\nSuccessfully installed requests-2.0\n", + "expected": "Successfully installed requests-2.0" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/playwright.json b/open-sse/services/compression/engines/rtk/filters/playwright.json new file mode 100644 index 0000000000..1899b6367c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/playwright.json @@ -0,0 +1,42 @@ +{ + "id": "playwright", + "label": "Playwright", + "description": "Keep failed specs, error context, and summary.", + "category": "test", + "priority": 86, + "match": { + "outputTypes": [], + "commands": ["^playwright\\s+test\\b", "^npx\\s+playwright\\s+test\\b"], + "patterns": ["Running \\d+ tests?", "Error Context:", "failed"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$", "^Running \\d+ tests? using"], + "includePatterns": [ + "✘", + "Error:", + "Error Context:", + "failed", + "passed", + "Slow test file", + "^\\s*\\d+\\)" + ], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 160, + "headLines": 35, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["Error:", "✘", "failed"], + "summaryPatterns": ["passed", "failed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "playwright test", + "input": "Running 2 tests using 1 worker\n ✘ 1 login.spec.ts:3:1 › login\nError: expected visible\n 1 failed\n", + "expected": " ✘ 1 login.spec.ts:3:1 › login\nError: expected visible\n 1 failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/poetry-install.json b/open-sse/services/compression/engines/rtk/filters/poetry-install.json new file mode 100644 index 0000000000..f40169365e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/poetry-install.json @@ -0,0 +1,35 @@ +{ + "id": "poetry-install", + "label": "Poetry Install", + "description": "Strip Poetry package operation noise.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^poetry\\s+(?:install|update)\\b"], + "patterns": ["Package operations:", "Installing dependencies"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^ - Installing ", "^ - Updating ", "^Installing dependencies", "^\\s*$"], + "includePatterns": ["Package operations:", "Writing lock file", "Error", "failed"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "poetry: ok" + }, + "preserve": { + "errorPatterns": ["Error", "failed"], + "summaryPatterns": ["Package operations:", "Writing lock file"] + }, + "tests": [ + { + "name": "inline sample", + "command": "poetry (?:install|update)", + "input": "Installing dependencies\nPackage operations: 2 installs, 0 updates\n - Installing foo (1.0)\nWriting lock file\n", + "expected": "Package operations: 2 installs, 0 updates\nWriting lock file" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/prettier.json b/open-sse/services/compression/engines/rtk/filters/prettier.json new file mode 100644 index 0000000000..0a1c894fc6 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/prettier.json @@ -0,0 +1,40 @@ +{ + "id": "prettier", + "label": "Prettier", + "description": "Keep files that would be changed and final summary.", + "category": "build", + "priority": 79, + "match": { + "outputTypes": [], + "commands": ["^prettier\\b", "^npx\\s+prettier\\b"], + "patterns": ["\\[warn\\]", "All matched files use Prettier"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "All matched files use Prettier", + "message": "prettier: ok" + } + ], + "dropPatterns": ["^\\s*$"], + "includePatterns": ["\\[warn\\]", "Code style issues", "All matched files"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 80, + "headLines": 20, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["\\[error\\]", "Code style issues"], + "summaryPatterns": ["All matched files"] + }, + "tests": [ + { + "name": "inline sample", + "command": "prettier", + "input": "[warn] src/a.ts\n[warn] Code style issues found in the above file. Forgot to run Prettier?\n", + "expected": "[warn] src/a.ts\n[warn] Code style issues found in the above file. Forgot to run Prettier?" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ps.json b/open-sse/services/compression/engines/rtk/filters/ps.json new file mode 100644 index 0000000000..843866d70e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ps.json @@ -0,0 +1,34 @@ +{ + "id": "ps", + "label": "ps", + "description": "Keep process table headers and process rows, dropping empty noise.", + "category": "shell", + "priority": 68, + "match": { + "outputTypes": ["shell-ps"], + "commands": ["^ps\\b"], + "patterns": ["^(?:USER\\s+PID|\\s*PID\\s+)"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^(?:USER\\s+PID|\\s*PID\\s+)", "^\\S+\\s+\\d+\\s+", "^\\s*\\d+\\s+"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 120, + "headLines": 30, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["defunct", "Permission denied"], + "summaryPatterns": ["^(?:USER\\s+PID|\\s*PID\\s+)"] + }, + "tests": [ + { + "name": "inline sample", + "command": "ps aux", + "input": "USER PID %CPU COMMAND\nroot 1 0.0 init\nnode 1234 5.0 node server.js\n", + "expected": "USER PID %CPU COMMAND\nroot 1 0.0 init\nnode 1234 5.0 node server.js" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/rsync.json b/open-sse/services/compression/engines/rtk/filters/rsync.json new file mode 100644 index 0000000000..36ecd135bc --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/rsync.json @@ -0,0 +1,33 @@ +{ + "id": "rsync", + "label": "rsync", + "description": "Drop rsync transfer chatter while preserving copied files and errors.", + "category": "cloud", + "priority": 73, + "match": { + "outputTypes": ["rsync"], + "commands": ["^rsync\\b"], + "patterns": ["^sending incremental file list", "^rsync error:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^sending incremental file list", "^sent \\d", "^total size is", "^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 140, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["^rsync error:", "Permission denied", "No such file"], + "summaryPatterns": ["^sent \\d", "^total size is"] + }, + "tests": [ + { + "name": "inline sample", + "command": "rsync -av", + "input": "sending incremental file list\napp.js\nsent 120 bytes received 20 bytes\nrsync error: some files could not be transferred (code 23)\n", + "expected": "app.js\nrsync error: some files could not be transferred (code 23)" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/rubocop.json b/open-sse/services/compression/engines/rtk/filters/rubocop.json new file mode 100644 index 0000000000..da92cc4beb --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/rubocop.json @@ -0,0 +1,34 @@ +{ + "id": "rubocop", + "label": "RuboCop", + "description": "Keep RuboCop offenses and final summary.", + "category": "build", + "priority": 74, + "match": { + "outputTypes": ["rubocop"], + "commands": ["^rubocop\\b", "^bundle\\s+exec\\s+rubocop\\b"], + "patterns": ["^Inspecting \\d+ files", "^\\S+\\.rb:\\d+:\\d+: [A-Z]:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Inspecting \\d+ files", "^\\.+$", "^\\s*$"], + "includePatterns": ["^\\S+\\.rb:\\d+:\\d+: [A-Z]:", "^\\d+ files inspected"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 45 + }, + "preserve": { + "errorPatterns": [" [EF]: ", "Offenses:"], + "summaryPatterns": ["^\\d+ files inspected"] + }, + "tests": [ + { + "name": "inline sample", + "command": "rubocop", + "input": "Inspecting 2 files\n.C\napp/models/user.rb:12:3: C: Style/IfUnlessModifier: Favor modifier if usage.\n2 files inspected, 1 offense detected\n", + "expected": "app/models/user.rb:12:3: C: Style/IfUnlessModifier: Favor modifier if usage.\n2 files inspected, 1 offense detected" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ruff.json b/open-sse/services/compression/engines/rtk/filters/ruff.json new file mode 100644 index 0000000000..0e3d80d6ee --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ruff.json @@ -0,0 +1,40 @@ +{ + "id": "ruff", + "label": "Ruff", + "description": "Keep Ruff diagnostics and summary.", + "category": "build", + "priority": 82, + "match": { + "outputTypes": [], + "commands": ["^ruff\\b", "^uv\\s+run\\s+ruff\\b"], + "patterns": ["Found \\d+ errors?", "All checks passed"] + }, + "rules": { + "stripAnsi": true, + "matchOutput": [ + { + "pattern": "All checks passed", + "message": "ruff: ok" + } + ], + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^[A-Z]\\d{3}", "Found \\d+", "error:", "warning:", "-->"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^[A-Z]\\d{3}", "error:"], + "summaryPatterns": ["Found \\d+", "All checks passed"] + }, + "tests": [ + { + "name": "inline sample", + "command": "ruff", + "input": "F401 app.py:1:1 unused import\nFound 1 error.\n", + "expected": "F401 app.py:1:1 unused import\nFound 1 error." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-find.json b/open-sse/services/compression/engines/rtk/filters/shell-find.json new file mode 100644 index 0000000000..e95ce818d9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-find.json @@ -0,0 +1,33 @@ +{ + "id": "shell-find", + "label": "find", + "description": "Limit large find output while preserving head and tail.", + "category": "shell", + "priority": 80, + "match": { + "outputTypes": ["shell-find"], + "commands": ["^find\\b"], + "patterns": ["^(?:\\./|/|[A-Za-z0-9_.-]+/)"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 120, + "headLines": 40, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["Permission denied", "No such file"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "find", + "input": "./src/a.ts\n./src/b.ts\n./node_modules/pkg/index.js\n", + "expected": "./src/a.ts\n./src/b.ts\n./node_modules/pkg/index.js" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-grep.json b/open-sse/services/compression/engines/rtk/filters/shell-grep.json new file mode 100644 index 0000000000..6429ff82f9 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-grep.json @@ -0,0 +1,37 @@ +{ + "id": "shell-grep", + "label": "grep/rg", + "description": "Deduplicate grep output while preserving matches.", + "category": "shell", + "priority": 82, + "match": { + "outputTypes": ["shell-grep"], + "commands": ["^(?:grep|rg|ag)\\b"], + "patterns": [ + "^[\\w./-]+\\.(?:ts|tsx|js|jsx|py|go|rs|java|rb|md|json|ya?ml|txt):\\d*:", + "^[\\w./-]+/[\\w./-]+:\\d*:" + ] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*$"], + "includePatterns": ["^[\\w./-]+(?::\\d+)?:"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 160, + "headLines": 40, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["No such file", "Permission denied"], + "summaryPatterns": ["^[^:\\n]+(?::\\d+)?:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:grep|rg|ag)", + "input": "src/a.ts:1:match\nsrc/a.ts:1:match\nsrc/b.ts:2:other\n", + "expected": "src/a.ts:1:match\nsrc/a.ts:1:match\nsrc/b.ts:2:other" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/shell-ls.json b/open-sse/services/compression/engines/rtk/filters/shell-ls.json new file mode 100644 index 0000000000..d8bb51cf8e --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/shell-ls.json @@ -0,0 +1,33 @@ +{ + "id": "shell-ls", + "label": "Shell Listing", + "description": "Keep directory listing boundaries without flooding the prompt.", + "category": "shell", + "priority": 50, + "match": { + "outputTypes": ["shell-ls"], + "commands": ["^(?:ls(?:\\s+-[A-Za-z]+)?|find)\\b"], + "patterns": ["^total \\d+", "^\\S+\\s+\\S+\\s+\\d+\\s+\\w+\\s+\\d{1,2}\\s+"] + }, + "rules": { + "includePatterns": [], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 100, + "headLines": 40, + "tailLines": 30 + }, + "preserve": { + "errorPatterns": ["^total \\d+"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:ls(?: -[A-Za-z]+)?|find)", + "input": "total 8\ndrwxr-xr-x 2 user user 4096 May 1 src\n-rw-r--r-- 1 user user 10 May 1 package.json\n", + "expected": "total 8\ndrwxr-xr-x 2 user user 4096 May 1 src\n-rw-r--r-- 1 user user 10 May 1 package.json" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/ssh.json b/open-sse/services/compression/engines/rtk/filters/ssh.json new file mode 100644 index 0000000000..2b67759776 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/ssh.json @@ -0,0 +1,43 @@ +{ + "id": "ssh", + "label": "ssh", + "description": "Compress SSH connection logs while preserving access and host-key failures.", + "category": "cloud", + "priority": 72, + "match": { + "outputTypes": ["ssh"], + "commands": ["^ssh\\b"], + "patterns": [ + "Permission denied \\(", + "Host key verification failed", + "Connection (?:closed|timed out)" + ] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^debug\\d:", "^\\s*$"], + "includePatterns": [ + "Permission denied", + "Host key verification failed", + "Connection (?:closed|timed out)", + "^Welcome " + ], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["Permission denied", "Host key verification failed", "Connection timed out"], + "summaryPatterns": ["^Welcome "] + }, + "tests": [ + { + "name": "inline sample", + "command": "ssh host", + "input": "debug1: Connecting to host\nWelcome to Ubuntu\nPermission denied (publickey).\n", + "expected": "Welcome to Ubuntu\nPermission denied (publickey)." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/systemctl-status.json b/open-sse/services/compression/engines/rtk/filters/systemctl-status.json new file mode 100644 index 0000000000..f75b502bc1 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/systemctl-status.json @@ -0,0 +1,43 @@ +{ + "id": "systemctl-status", + "label": "systemctl status", + "description": "Preserve unit status, active state, and journal errors.", + "category": "infra", + "priority": 77, + "match": { + "outputTypes": [], + "commands": ["^systemctl\\s+status\\b"], + "patterns": ["Loaded:", "Active:", "Main PID:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "Loaded:", + "Active:", + "Main PID:", + "CGroup:", + "error", + "failed", + "Started", + "Stopped" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^\\s+Docs:"], + "deduplicate": true, + "maxLines": 100, + "headLines": 30, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["failed", "error"], + "summaryPatterns": ["Active:", "Loaded:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "systemctl status", + "input": "Loaded: loaded (/etc/systemd/system/app.service)\nActive: failed (Result: exit-code)\nMain PID: 123\nMay 1 app[123]: error boot failed\n", + "expected": "Loaded: loaded (/etc/systemd/system/app.service)\nActive: failed (Result: exit-code)\nMain PID: 123\nMay 1 app[123]: error boot failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/terraform-plan.json b/open-sse/services/compression/engines/rtk/filters/terraform-plan.json new file mode 100644 index 0000000000..3f9d417253 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/terraform-plan.json @@ -0,0 +1,34 @@ +{ + "id": "terraform-plan", + "label": "Terraform Plan", + "description": "Keep plan actions and summary while stripping refresh noise.", + "category": "infra", + "priority": 88, + "match": { + "outputTypes": [], + "commands": ["^terraform\\s+plan\\b"], + "patterns": ["Terraform will perform the following actions", "Plan: \\d+ to add"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["Refreshing state", "Acquiring state lock", "Releasing state lock", "^\\s*$"], + "includePatterns": ["Terraform will perform", "^[ ]*[#~+\\-/]", "Plan: \\d+", "Error:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 160, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["Error:", "failed"], + "summaryPatterns": ["Plan: \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "terraform plan", + "input": "Refreshing state... [id=a]\nTerraform will perform the following actions:\n # aws_instance.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy.\n", + "expected": "Terraform will perform the following actions:\n # aws_instance.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-cargo.json b/open-sse/services/compression/engines/rtk/filters/test-cargo.json new file mode 100644 index 0000000000..e7aecaca5c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-cargo.json @@ -0,0 +1,42 @@ +{ + "id": "test-cargo", + "label": "Cargo Test", + "description": "Keep Cargo test failures and final summary.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-cargo"], + "commands": ["^cargo\\s+(?:test|nextest)\\b"], + "patterns": ["^running \\d+ tests?", "test result:"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "^running \\d+ tests?", + "^test .* \\.\\.\\. FAILED", + "^failures:", + "^---- ", + "^thread ", + "^test result:", + "^error:" + ], + "dropPatterns": ["^test .* \\.\\.\\. ok$", "^\\s*$"], + "collapsePatterns": ["^\\s+at "], + "deduplicate": true, + "maxLines": 180, + "headLines": 40, + "tailLines": 70 + }, + "preserve": { + "errorPatterns": ["FAILED", "^error:", "^thread "], + "summaryPatterns": ["test result:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "cargo (?:test|nextest)", + "input": "running 2 tests\ntest a ... ok\ntest b ... FAILED\nfailures:\n---- b stdout ----\nthread b panicked at boom\ntest result: FAILED. 1 passed; 1 failed\n", + "expected": "running 2 tests\ntest b ... FAILED\nfailures:\n---- b stdout ----\nthread b panicked at boom\ntest result: FAILED. 1 passed; 1 failed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-go.json b/open-sse/services/compression/engines/rtk/filters/test-go.json new file mode 100644 index 0000000000..5e4f05e50d --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-go.json @@ -0,0 +1,41 @@ +{ + "id": "test-go", + "label": "Go Test", + "description": "Keep failing Go tests and package summaries.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-go"], + "commands": ["^go\\s+test\\b"], + "patterns": ["^--- FAIL:", "^(?:ok|FAIL)\\s+"] + }, + "rules": { + "stripAnsi": true, + "includePatterns": [ + "^--- FAIL:", + "^FAIL", + "^ok\\s+", + "^panic:", + "^\\s+.*_test\\.go:\\d+", + "^\\s*Error" + ], + "dropPatterns": ["^\\s*$"], + "collapsePatterns": ["^=== RUN"], + "deduplicate": true, + "maxLines": 160, + "headLines": 35, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["^--- FAIL:", "^panic:", "^FAIL"], + "summaryPatterns": ["^ok\\s+", "^FAIL\\s+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "go test", + "input": "=== RUN TestA\n--- FAIL: TestA (0.00s)\n a_test.go:1: boom\nFAIL\t./pkg\t0.1s\n", + "expected": "--- FAIL: TestA (0.00s)\n a_test.go:1: boom\nFAIL\t./pkg\t0.1s" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-jest.json b/open-sse/services/compression/engines/rtk/filters/test-jest.json new file mode 100644 index 0000000000..62f4c0a5d0 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-jest.json @@ -0,0 +1,43 @@ +{ + "id": "test-jest", + "label": "Jest", + "description": "Keep Jest failures and summary while removing passing noise.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-jest"], + "commands": ["^(?:jest|npm\\s+(?:run\\s+)?test)\\b"], + "patterns": ["Test Suites:\\s+\\d+", "Tests:\\s+\\d+", "^PASS\\s+", "^FAIL\\s+"] + }, + "rules": { + "includePatterns": [ + "^FAIL\\s+", + "^\\s*●\\s+", + "Expected:", + "Received:", + "Test Suites:", + "Tests:", + "Snapshots:", + "Time:", + "\\s+at\\s+" + ], + "dropPatterns": ["^PASS\\s+", "^\\s*✓\\s+"], + "collapsePatterns": ["^\\s*at "], + "deduplicate": true, + "maxLines": 140, + "headLines": 24, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["FAIL", "Expected:", "Received:", "Test Suites:"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:jest|npm (?:run )?test)", + "input": "PASS src/a.test.ts\nFAIL src/b.test.ts\nError: boom\nTest Suites: 1 failed, 1 passed\nTests: 1 failed, 1 passed\n", + "expected": "FAIL src/b.test.ts\nTest Suites: 1 failed, 1 passed\nTests: 1 failed, 1 passed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-pytest.json b/open-sse/services/compression/engines/rtk/filters/test-pytest.json new file mode 100644 index 0000000000..4e62bcc34c --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-pytest.json @@ -0,0 +1,41 @@ +{ + "id": "test-pytest", + "label": "Pytest", + "description": "Keep pytest failures, tracebacks and terminal summary.", + "category": "test", + "priority": 90, + "match": { + "outputTypes": ["test-pytest"], + "commands": ["^(?:pytest|python\\s+-m\\s+pytest)\\b"], + "patterns": ["=+\\s+(?:\\d+\\s+)?(?:passed|failed|errors?)", "^E\\s+", "^FAILED "] + }, + "rules": { + "includePatterns": [ + "^FAILED ", + "^ERROR ", + "^E\\s+", + "Traceback \\(most recent call last\\):", + "=+ short test summary info =+", + "=+ .*failed", + "AssertionError" + ], + "dropPatterns": ["^\\.", "^\\s*$"], + "collapsePatterns": ["^E\\s+"], + "deduplicate": true, + "maxLines": 160, + "headLines": 28, + "tailLines": 60 + }, + "preserve": { + "errorPatterns": ["FAILED", "ERROR", "Traceback", "AssertionError"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:pytest|python -m pytest)", + "input": "FAILED tests/test_a.py::test_a - AssertionError\nE assert 1 == 2\n================ 1 failed, 2 passed ================\n", + "expected": "FAILED tests/test_a.py::test_a - AssertionError\nE assert 1 == 2\n================ 1 failed, 2 passed ================" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/test-vitest.json b/open-sse/services/compression/engines/rtk/filters/test-vitest.json new file mode 100644 index 0000000000..bb8a7480de --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/test-vitest.json @@ -0,0 +1,43 @@ +{ + "id": "test-vitest", + "label": "Vitest", + "description": "Keep failing suites, failed tests, stack headers, and final summary.", + "category": "test", + "priority": 92, + "match": { + "outputTypes": ["test-vitest"], + "commands": ["^(?:vitest|npm\\s+(?:run\\s+)?test:vitest)\\b"], + "patterns": ["\\bvitest\\b", "^ ✓ ", "^ ❯ ", "Test Files\\s+\\d+\\s+(?:passed|failed)"] + }, + "rules": { + "includePatterns": [ + "^\\s*FAIL\\s+", + "^\\s*❯\\s+", + "^\\s*×\\s+", + "Test Files\\s+", + "Tests\\s+", + "Duration\\s+", + "Error:", + "AssertionError", + "\\s+at\\s+" + ], + "dropPatterns": ["^\\s*✓\\s+", "^\\s*stdout \\|", "^\\s*stderr \\|"], + "collapsePatterns": ["^\\s*at "], + "deduplicate": true, + "maxLines": 140, + "headLines": 24, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["FAIL", "Error:", "AssertionError", "Test Files"], + "summaryPatterns": [] + }, + "tests": [ + { + "name": "inline sample", + "command": "(?:vitest|npm (?:run )?test:vitest)", + "input": " ✓ src/a.test.ts (1)\n ❯ src/b.test.ts (1)\n FAIL src/b.test.ts > fails\nError: boom\nTest Files 1 failed | 1 passed\nTests 1 failed | 1 passed\n", + "expected": " ❯ src/b.test.ts (1)\n FAIL src/b.test.ts > fails\nError: boom\nTest Files 1 failed | 1 passed\nTests 1 failed | 1 passed" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/tofu-plan.json b/open-sse/services/compression/engines/rtk/filters/tofu-plan.json new file mode 100644 index 0000000000..d6b8c69ee5 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/tofu-plan.json @@ -0,0 +1,34 @@ +{ + "id": "tofu-plan", + "label": "OpenTofu Plan", + "description": "Keep OpenTofu plan actions and summary while stripping refresh noise.", + "category": "infra", + "priority": 87, + "match": { + "outputTypes": [], + "commands": ["^tofu\\s+plan\\b"], + "patterns": ["OpenTofu will perform the following actions", "Plan: \\d+ to add"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["Refreshing state", "Acquiring state lock", "Releasing state lock", "^\\s*$"], + "includePatterns": ["OpenTofu will perform", "^[ ]*[#~+\\-/]", "Plan: \\d+", "Error:"], + "collapsePatterns": [], + "deduplicate": false, + "maxLines": 160, + "headLines": 35, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["Error:", "failed"], + "summaryPatterns": ["Plan: \\d+"] + }, + "tests": [ + { + "name": "inline sample", + "command": "tofu plan", + "input": "Refreshing state... [id=a]\nOpenTofu will perform the following actions:\n # tofu_resource.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy.\n", + "expected": "OpenTofu will perform the following actions:\n # tofu_resource.web will be created\nPlan: 1 to add, 0 to change, 0 to destroy." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/turbo.json b/open-sse/services/compression/engines/rtk/filters/turbo.json new file mode 100644 index 0000000000..0727717b65 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/turbo.json @@ -0,0 +1,34 @@ +{ + "id": "turbo", + "label": "Turborepo", + "description": "Compact turbo task cache and failure output.", + "category": "build", + "priority": 76, + "match": { + "outputTypes": [], + "commands": ["^turbo\\b", "^npx\\s+turbo\\b"], + "patterns": ["Tasks:", "Remote caching"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["cache hit", "cache miss", "^\\s*$"], + "includePatterns": ["failed", "error", "Tasks:", "Cached:", "Time:", "successful"], + "collapsePatterns": ["^• Packages in scope"], + "deduplicate": true, + "maxLines": 120, + "headLines": 25, + "tailLines": 50 + }, + "preserve": { + "errorPatterns": ["failed", "error"], + "summaryPatterns": ["Tasks:", "Cached:", "Time:"] + }, + "tests": [ + { + "name": "inline sample", + "command": "turbo", + "input": "cache hit app#build\napp#test: failed\nTasks: 1 successful, 1 failed\nTime: 2s\n", + "expected": "app#test: failed\nTasks: 1 successful, 1 failed\nTime: 2s" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/uv-sync.json b/open-sse/services/compression/engines/rtk/filters/uv-sync.json new file mode 100644 index 0000000000..ecac0a0321 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/uv-sync.json @@ -0,0 +1,35 @@ +{ + "id": "uv-sync", + "label": "uv sync", + "description": "Compact uv sync/install output.", + "category": "package", + "priority": 72, + "match": { + "outputTypes": [], + "commands": ["^uv\\s+sync\\b", "^uv\\s+pip\\s+install\\b"], + "patterns": ["Resolved \\d+ packages?", "Prepared \\d+ packages?"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^Resolved ", "^Prepared ", "^Installed ", "^ \\+", "^\\s*$"], + "includePatterns": ["error", "failed", "Installed", "Uninstalled", "Resolved"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 35, + "onEmpty": "uv: ok" + }, + "preserve": { + "errorPatterns": ["error", "failed"], + "summaryPatterns": ["Installed", "Resolved"] + }, + "tests": [ + { + "name": "inline sample", + "command": "uv sync", + "input": "Resolved 10 packages in 1ms\nInstalled 10 packages in 2ms\n", + "expected": "uv: ok" + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/filters/wget.json b/open-sse/services/compression/engines/rtk/filters/wget.json new file mode 100644 index 0000000000..b5de52bb99 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/filters/wget.json @@ -0,0 +1,34 @@ +{ + "id": "wget", + "label": "wget", + "description": "Remove wget progress output and keep final errors or saved-file summaries.", + "category": "cloud", + "priority": 70, + "match": { + "outputTypes": ["wget"], + "commands": ["^wget\\b"], + "patterns": ["^--\\d{4}-\\d{2}-\\d{2}", "^ERROR \\d{3}:"] + }, + "rules": { + "stripAnsi": true, + "dropPatterns": ["^\\s*\\d+K", "^Saving to:", "^\\s*$"], + "includePatterns": ["^ERROR \\d{3}:", "saved \\[\\d", "^HTTP request sent"], + "collapsePatterns": [], + "deduplicate": true, + "maxLines": 100, + "headLines": 25, + "tailLines": 40 + }, + "preserve": { + "errorPatterns": ["^ERROR \\d{3}:", "failed", "unable to resolve"], + "summaryPatterns": ["saved \\[\\d", "^HTTP request sent"] + }, + "tests": [ + { + "name": "inline sample", + "command": "wget", + "input": "--2026-05-02-- https://example.test/file\nSaving to: 'file'\nHTTP request sent, awaiting response... 404 Not Found\nERROR 404: Not Found.\n", + "expected": "HTTP request sent, awaiting response... 404 Not Found\nERROR 404: Not Found." + } + ] +} diff --git a/open-sse/services/compression/engines/rtk/index.ts b/open-sse/services/compression/engines/rtk/index.ts new file mode 100644 index 0000000000..61fd9d535a --- /dev/null +++ b/open-sse/services/compression/engines/rtk/index.ts @@ -0,0 +1,404 @@ +import { createCompressionStats, estimateCompressionTokens } from "../../stats.ts"; +import { DEFAULT_RTK_CONFIG, type CompressionResult, type RtkConfig } from "../../types.ts"; +import type { CompressionEngine, EngineConfigField, EngineValidationResult } from "../types.ts"; +import { detectCommandType } from "./commandDetector.ts"; +import { deduplicateRepeatedLines } from "./deduplicator.ts"; +import { matchRtkFilter } from "./filterLoader.ts"; +import { applyLineFilter } from "./lineFilter.ts"; +import { smartTruncate } from "./smartTruncate.ts"; +import { normalizeCodeLanguage, stripCode } from "./codeStripper.ts"; +import { maybePersistRtkRawOutput, type RtkRawOutputPointer } from "./rawOutput.ts"; + +type Message = { + role: string; + content?: string | Array<{ type?: string; text?: string; [key: string]: unknown }>; + [key: string]: unknown; +}; + +const RTK_SCHEMA: EngineConfigField[] = [ + { + key: "intensity", + type: "select", + label: "Intensity", + defaultValue: DEFAULT_RTK_CONFIG.intensity, + options: [ + { value: "minimal", label: "minimal" }, + { value: "standard", label: "standard" }, + { value: "aggressive", label: "aggressive" }, + ], + }, + { + key: "applyToToolResults", + type: "boolean", + label: "Apply to tool results", + defaultValue: DEFAULT_RTK_CONFIG.applyToToolResults, + }, + { + key: "applyToAssistantMessages", + type: "boolean", + label: "Apply to assistant messages", + defaultValue: DEFAULT_RTK_CONFIG.applyToAssistantMessages, + }, + { + key: "applyToCodeBlocks", + type: "boolean", + label: "Apply to code blocks", + defaultValue: DEFAULT_RTK_CONFIG.applyToCodeBlocks, + }, + { + key: "maxLinesPerResult", + type: "number", + label: "Max lines per result", + defaultValue: DEFAULT_RTK_CONFIG.maxLinesPerResult, + min: 0, + max: 5000, + }, + { + key: "maxCharsPerResult", + type: "number", + label: "Max chars per result", + defaultValue: DEFAULT_RTK_CONFIG.maxCharsPerResult, + min: 0, + max: 500000, + }, + { + key: "deduplicateThreshold", + type: "number", + label: "Deduplicate threshold", + defaultValue: DEFAULT_RTK_CONFIG.deduplicateThreshold, + min: 2, + max: 100, + }, + { + key: "rawOutputRetention", + type: "select", + label: "Raw output retention", + defaultValue: DEFAULT_RTK_CONFIG.rawOutputRetention, + options: [ + { value: "never", label: "never" }, + { value: "failures", label: "failures" }, + { value: "always", label: "always" }, + ], + }, +]; + +function validateRtkEngineConfig(config: Record): EngineValidationResult { + const errors: string[] = []; + if ( + config.intensity !== undefined && + config.intensity !== "minimal" && + config.intensity !== "standard" && + config.intensity !== "aggressive" + ) { + errors.push("intensity must be minimal, standard, or aggressive"); + } + for (const key of [ + "enabled", + "applyToToolResults", + "applyToAssistantMessages", + "applyToCodeBlocks", + ]) { + if (config[key] !== undefined && typeof config[key] !== "boolean") { + errors.push(`${key} must be a boolean`); + } + } + for (const key of ["maxLinesPerResult", "maxCharsPerResult", "deduplicateThreshold"]) { + if (config[key] !== undefined && (typeof config[key] !== "number" || config[key] < 0)) { + errors.push(`${key} must be a non-negative number`); + } + } + if (config.enabledFilters !== undefined && !Array.isArray(config.enabledFilters)) { + errors.push("enabledFilters must be an array"); + } + if (config.disabledFilters !== undefined && !Array.isArray(config.disabledFilters)) { + errors.push("disabledFilters must be an array"); + } + if ( + config.rawOutputRetention !== undefined && + config.rawOutputRetention !== "never" && + config.rawOutputRetention !== "failures" && + config.rawOutputRetention !== "always" + ) { + errors.push("rawOutputRetention must be never, failures, or always"); + } + return { valid: errors.length === 0, errors }; +} + +export interface RtkProcessResult { + text: string; + compressed: boolean; + originalTokens: number; + compressedTokens: number; + techniquesUsed: string[]; + rulesApplied: string[]; + rawOutputPointers?: RtkRawOutputPointer[]; +} + +function mergeRtkConfig(base?: Partial, override?: Record): RtkConfig { + const merged = { ...DEFAULT_RTK_CONFIG, ...(base ?? {}), ...(override ?? {}) }; + return { + ...merged, + intensity: + merged.intensity === "minimal" || + merged.intensity === "standard" || + merged.intensity === "aggressive" + ? merged.intensity + : DEFAULT_RTK_CONFIG.intensity, + enabledFilters: Array.isArray(merged.enabledFilters) + ? merged.enabledFilters.filter((id): id is string => typeof id === "string") + : [], + disabledFilters: Array.isArray(merged.disabledFilters) + ? merged.disabledFilters.filter((id): id is string => typeof id === "string") + : [], + maxLinesPerResult: + typeof merged.maxLinesPerResult === "number" && Number.isFinite(merged.maxLinesPerResult) + ? Math.max(0, Math.floor(merged.maxLinesPerResult)) + : DEFAULT_RTK_CONFIG.maxLinesPerResult, + maxCharsPerResult: + typeof merged.maxCharsPerResult === "number" && Number.isFinite(merged.maxCharsPerResult) + ? Math.max(0, Math.floor(merged.maxCharsPerResult)) + : DEFAULT_RTK_CONFIG.maxCharsPerResult, + deduplicateThreshold: + typeof merged.deduplicateThreshold === "number" && + Number.isFinite(merged.deduplicateThreshold) + ? Math.max(2, Math.floor(merged.deduplicateThreshold)) + : DEFAULT_RTK_CONFIG.deduplicateThreshold, + customFiltersEnabled: + typeof merged.customFiltersEnabled === "boolean" + ? merged.customFiltersEnabled + : DEFAULT_RTK_CONFIG.customFiltersEnabled, + trustProjectFilters: + typeof merged.trustProjectFilters === "boolean" + ? merged.trustProjectFilters + : DEFAULT_RTK_CONFIG.trustProjectFilters, + rawOutputRetention: + merged.rawOutputRetention === "never" || + merged.rawOutputRetention === "failures" || + merged.rawOutputRetention === "always" + ? merged.rawOutputRetention + : DEFAULT_RTK_CONFIG.rawOutputRetention, + rawOutputMaxBytes: + typeof merged.rawOutputMaxBytes === "number" && Number.isFinite(merged.rawOutputMaxBytes) + ? Math.max(1024, Math.floor(merged.rawOutputMaxBytes)) + : DEFAULT_RTK_CONFIG.rawOutputMaxBytes, + }; +} + +function shouldCompressMessage(message: Message, config: RtkConfig): boolean { + if (message.role === "tool") return config.applyToToolResults || config.applyToCodeBlocks; + if (message.role === "assistant") + return config.applyToAssistantMessages || config.applyToCodeBlocks; + return false; +} + +function mapStringContent( + content: Message["content"], + transform: (text: string) => string +): Message["content"] { + if (typeof content === "string") return transform(content); + if (!Array.isArray(content)) return content; + return content.map((part) => { + if (part && typeof part === "object" && part.type === "text" && typeof part.text === "string") { + return { ...part, text: transform(part.text) }; + } + return part; + }); +} + +function extractContentText(content: Message["content"]): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .map((part) => + part && typeof part === "object" && typeof part.text === "string" ? part.text : "" + ) + .filter(Boolean) + .join("\n"); +} + +export function processRtkText( + text: string, + options: { command?: string | null; config?: Partial } = {} +): RtkProcessResult { + const config = mergeRtkConfig(options.config); + const originalTokens = estimateCompressionTokens(text); + const techniquesUsed: string[] = []; + const rulesApplied: string[] = []; + const rawOutputPointers: RtkRawOutputPointer[] = []; + let result = text; + + const detection = detectCommandType(text, options.command); + const filter = matchRtkFilter(text, detection.command, { + customFiltersEnabled: config.customFiltersEnabled, + trustProjectFilters: config.trustProjectFilters, + }); + if (filter && !config.disabledFilters.includes(filter.id)) { + if (config.enabledFilters.length === 0 || config.enabledFilters.includes(filter.id)) { + const filtered = applyLineFilter(result, { + ...filter, + maxLines: filter.maxLines || config.maxLinesPerResult, + }); + result = filtered.text; + if (filtered.appliedRules.length > 0) { + techniquesUsed.push("rtk-filter"); + rulesApplied.push(...filtered.appliedRules); + } + } + } + + if (config.applyToCodeBlocks) { + let strippedCodeBlocks = 0; + result = result.replace( + /```([A-Za-z0-9_+.-]*)\r?\n([\s\S]*?)```/g, + (match, languageHint: string, code: string) => { + const stripped = stripCode(code, normalizeCodeLanguage(languageHint)); + if (stripped.strippedLines <= 0 && stripped.text === code.trim()) return match; + strippedCodeBlocks++; + const fenceLanguage = languageHint?.trim() || stripped.language; + return `\`\`\`${fenceLanguage}\n${stripped.text}\n\`\`\``; + } + ); + if (strippedCodeBlocks > 0) { + techniquesUsed.push("rtk-code-strip"); + rulesApplied.push("rtk:code-strip"); + } + } + + if (config.intensity !== "minimal") { + const deduped = deduplicateRepeatedLines(result, { threshold: config.deduplicateThreshold }); + if (deduped.collapsed > 0) { + result = deduped.text; + techniquesUsed.push("rtk-dedup"); + rulesApplied.push("rtk:dedup"); + } + } + + const truncated = smartTruncate(result, { + maxLines: config.maxLinesPerResult, + maxChars: config.maxCharsPerResult, + preserveHead: config.intensity === "aggressive" ? 16 : 24, + preserveTail: config.intensity === "aggressive" ? 16 : 24, + priorityPatterns: [/error|failed|exception|traceback|TS\d{4}|FAIL|✖/i], + }); + if (truncated.truncated) { + result = truncated.text; + techniquesUsed.push("rtk-truncate"); + rulesApplied.push("rtk:truncate"); + } + + const compressedTokens = estimateCompressionTokens(result); + if (compressedTokens < originalTokens) { + const pointer = maybePersistRtkRawOutput(text, { + retention: config.rawOutputRetention, + command: detection.command, + maxBytes: config.rawOutputMaxBytes, + }); + if (pointer) { + rawOutputPointers.push(pointer); + techniquesUsed.push("rtk-raw-output-retention"); + rulesApplied.push("rtk:raw-output-retention"); + } + } + return { + text: result, + compressed: compressedTokens < originalTokens, + originalTokens, + compressedTokens, + techniquesUsed: [...new Set(techniquesUsed)], + rulesApplied: [...new Set(rulesApplied)], + ...(rawOutputPointers.length > 0 ? { rawOutputPointers } : {}), + }; +} + +export function applyRtkCompression( + body: Record, + options: { config?: Partial; stepConfig?: Record } = {} +): CompressionResult { + const start = performance.now(); + const config = mergeRtkConfig(options.config, options.stepConfig); + if (!config.enabled) return { body, compressed: false, stats: null }; + + const messages = body.messages as Message[] | undefined; + if (!Array.isArray(messages) || messages.length === 0) { + return { body, compressed: false, stats: null }; + } + + const allTechniques: string[] = []; + const allRules: string[] = []; + const rawOutputPointers: RtkRawOutputPointer[] = []; + const compressedMessages = messages.map((message) => { + if (!shouldCompressMessage(message, config)) return message; + const text = extractContentText(message.content); + if (!text) return message; + const processed = processRtkText(text, { config }); + allTechniques.push(...processed.techniquesUsed); + allRules.push(...processed.rulesApplied); + if (processed.rawOutputPointers) rawOutputPointers.push(...processed.rawOutputPointers); + if (!processed.compressed) return message; + return { + ...message, + content: mapStringContent(message.content, () => processed.text), + }; + }); + + const compressedBody = { ...body, messages: compressedMessages }; + const stats = createCompressionStats( + body, + compressedBody, + "rtk", + [...new Set(allTechniques)], + allRules.length > 0 ? [...new Set(allRules)] : undefined, + Math.round((performance.now() - start) * 100) / 100 + ); + stats.engine = "rtk"; + if (rawOutputPointers.length > 0) { + stats.rtkRawOutputPointers = rawOutputPointers; + } + return { + body: compressedBody, + compressed: stats.compressedTokens < stats.originalTokens, + stats, + }; +} + +export const rtkEngine: CompressionEngine = { + id: "rtk", + name: "RTK", + description: "Command-aware tool output compression with declarative filters.", + icon: "filter_alt", + targets: ["tool_results", "code_blocks"], + stackable: true, + stackPriority: 10, + metadata: { + id: "rtk", + name: "RTK", + description: "Command-aware tool output compression with declarative filters.", + inputScope: "tool-results", + targetLatencyMs: 5, + supportsPreview: true, + stable: true, + }, + apply(body, options) { + return applyRtkCompression(body, { + config: options?.config?.rtkConfig, + stepConfig: options?.stepConfig, + }); + }, + compress(body, config) { + return this.apply(body, { stepConfig: config }); + }, + getConfigSchema() { + return RTK_SCHEMA; + }, + validateConfig(config) { + return validateRtkEngineConfig(config); + }, +}; + +export { + detectCommandFromText, + detectCommandOutput, + detectCommandType, +} from "./commandDetector.ts"; +export { runRtkFilterTests } from "./verify.ts"; +export { maybePersistRtkRawOutput, readRtkRawOutput, redactRtkRawOutput } from "./rawOutput.ts"; diff --git a/open-sse/services/compression/engines/rtk/lineFilter.ts b/open-sse/services/compression/engines/rtk/lineFilter.ts new file mode 100644 index 0000000000..589b618086 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/lineFilter.ts @@ -0,0 +1,155 @@ +import type { RtkFilterDefinition } from "./filterSchema.ts"; +import { smartTruncate } from "./smartTruncate.ts"; + +export interface LineFilterResult { + text: string; + strippedLines: number; + keptByRule: boolean; + appliedRules: string[]; +} + +function compilePatterns(patterns: string[]): RegExp[] { + return patterns.flatMap((pattern) => { + try { + return [new RegExp(pattern, "i")]; + } catch { + return []; + } + }); +} + +function compileGlobalPattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "g"); + } catch { + return null; + } +} + +function compileBlobPattern(pattern: string): RegExp | null { + try { + return new RegExp(pattern, "im"); + } catch { + return null; + } +} + +function stripAnsi(text: string): string { + return text.replace(/\u001b\[[0-?]*[ -/]*[@-~]/g, ""); +} + +function normalizeStderrPrefix(line: string): string { + return line.replace(/^\s*(?:stderr|err)\s*(?:\||:)\s*/i, ""); +} + +function truncateUnicodeSafe(line: string, maxChars: number): string { + if (maxChars <= 0) return line; + const chars = Array.from(line); + if (chars.length <= maxChars) return line; + if (maxChars <= 3) return chars.slice(0, maxChars).join(""); + return `${chars.slice(0, maxChars - 3).join("")}...`; +} + +export function applyLineFilter(text: string, filter: RtkFilterDefinition): LineFilterResult { + const stripPatterns = compilePatterns(filter.stripPatterns); + const keepPatterns = compilePatterns(filter.keepPatterns); + const collapsePatterns = compilePatterns(filter.collapsePatterns); + const priorityPatterns = compilePatterns(filter.priorityPatterns); + const appliedRules: string[] = []; + + let lines = text.split(/\r?\n/); + const originalLineCount = lines.length; + + if (filter.stripAnsi) { + const stripped = lines.map(stripAnsi); + if (stripped.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:strip-ansi`); + } + lines = stripped; + } + + if (filter.filterStderr) { + const normalized = lines.map(normalizeStderrPrefix); + if (normalized.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:filter-stderr`); + } + lines = normalized; + } + + for (const rule of filter.replace) { + const pattern = compileGlobalPattern(rule.pattern); + if (!pattern) continue; + const replaced = lines.map((line) => line.replace(pattern, rule.replacement)); + if (replaced.join("\n") !== lines.join("\n")) { + appliedRules.push(`${filter.id}:replace`); + } + lines = replaced; + } + + if (filter.matchOutput.length > 0) { + const blob = lines.join("\n"); + for (const rule of filter.matchOutput) { + const pattern = compileBlobPattern(rule.pattern); + if (!pattern?.test(blob)) continue; + const unless = rule.unless ? compileBlobPattern(rule.unless) : null; + if (unless?.test(blob)) continue; + appliedRules.push(`${filter.id}:match-output`); + return { + text: rule.message, + strippedLines: Math.max(0, originalLineCount - 1), + keptByRule: false, + appliedRules, + }; + } + } + + if (stripPatterns.length > 0) { + lines = lines.filter((line) => !stripPatterns.some((pattern) => pattern.test(line))); + if (lines.length !== originalLineCount) appliedRules.push(`${filter.id}:strip`); + } + + if (keepPatterns.length > 0) { + const kept = lines.filter((line) => keepPatterns.some((pattern) => pattern.test(line))); + if (kept.length > 0) { + lines = kept; + appliedRules.push(`${filter.id}:keep`); + } + } + + if (collapsePatterns.length > 0) { + const seen = new Set(); + lines = lines.filter((line) => { + if (!collapsePatterns.some((pattern) => pattern.test(line))) return true; + const key = line.trim(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + appliedRules.push(`${filter.id}:collapse`); + } + + if (filter.truncateLineAt > 0) { + const truncatedLines = lines.map((line) => truncateUnicodeSafe(line, filter.truncateLineAt)); + if (truncatedLines.join("\n") !== lines.join("\n")) { + lines = truncatedLines; + appliedRules.push(`${filter.id}:truncate-line`); + } + } + + const truncated = smartTruncate(lines.join("\n"), { + maxLines: filter.maxLines, + preserveHead: filter.preserveHead, + preserveTail: filter.preserveTail, + priorityPatterns, + }); + if (truncated.truncated) appliedRules.push(`${filter.id}:truncate`); + const output = + truncated.text.trim().length === 0 && filter.onEmpty ? filter.onEmpty : truncated.text; + + return { + text: output, + strippedLines: Math.max(0, originalLineCount - output.split(/\r?\n/).length), + keptByRule: keepPatterns.length > 0, + appliedRules, + }; +} diff --git a/open-sse/services/compression/engines/rtk/rawOutput.ts b/open-sse/services/compression/engines/rtk/rawOutput.ts new file mode 100644 index 0000000000..7e40d6a9cb --- /dev/null +++ b/open-sse/services/compression/engines/rtk/rawOutput.ts @@ -0,0 +1,112 @@ +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import crypto from "node:crypto"; + +export type RtkRawOutputRetention = "never" | "failures" | "always"; + +export interface RtkRawOutputPointer { + id: string; + path: string; + bytes: number; + sha256: string; + redacted: boolean; +} + +const SECRET_PATTERNS: Array<[RegExp, string]> = [ + [/\b(sk-[A-Za-z0-9_-]{16,})\b/g, "[REDACTED_OPENAI_KEY]"], + [/\b(xox[baprs]-[A-Za-z0-9-]{16,})\b/g, "[REDACTED_SLACK_TOKEN]"], + [/\b(AKIA[0-9A-Z]{16})\b/g, "[REDACTED_AWS_KEY]"], + [/((?:api[_-]?key|token|secret|password)\s*[:=]\s*)("[^"]+"|'[^']+'|[^\s]+)/gi, "$1[REDACTED]"], + [/(Authorization:\s*Bearer\s+)[A-Za-z0-9._~+/-]+=*/gi, "$1[REDACTED]"], +]; + +function dataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +function safeId(seed: string): string { + return crypto.createHash("sha256").update(seed).digest("hex").slice(0, 24); +} + +function safeUtf8Slice(value: string, maxBytes: number): string { + if (maxBytes <= 0 || Buffer.byteLength(value, "utf8") <= maxBytes) return value; + let bytes = 0; + let output = ""; + for (const char of value) { + const len = Buffer.byteLength(char, "utf8"); + if (bytes + len > maxBytes) break; + output += char; + bytes += len; + } + return `${output}\n\n--- truncated at ${maxBytes} bytes ---`; +} + +export function redactRtkRawOutput(value: string): { text: string; redacted: boolean } { + let redacted = false; + let text = value; + for (const [pattern, replacement] of SECRET_PATTERNS) { + const next = text.replace(pattern, (...args: string[]) => { + redacted = true; + return typeof replacement === "string" + ? replacement.replace("$1", args[1] ?? "") + : replacement; + }); + text = next; + } + return { text, redacted }; +} + +export function isLikelyFailureOutput(value: string): boolean { + return /\b(error|failed|failure|exception|traceback|panic|fatal|critical|TS\d{4}|FAIL)\b/i.test( + value + ); +} + +export function maybePersistRtkRawOutput( + raw: string, + options: { + retention: RtkRawOutputRetention; + command?: string | null; + maxBytes?: number; + failure?: boolean; + } +): RtkRawOutputPointer | null { + if (options.retention === "never") return null; + const failure = options.failure ?? isLikelyFailureOutput(raw); + if (options.retention === "failures" && !failure) return null; + if (raw.trim().length === 0) return null; + + const maxBytes = Math.max(1024, Math.floor(options.maxBytes ?? 1_048_576)); + const redaction = redactRtkRawOutput(safeUtf8Slice(raw, maxBytes)); + const now = Date.now(); + const commandSlug = (options.command || "tool-output") + .replace(/[^A-Za-z0-9_-]+/g, "_") + .replace(/^_+|_+$/g, "") + .slice(0, 48); + const id = safeId(`${now}:${commandSlug}:${raw.length}:${redaction.text}`); + const dir = path.join(dataDir(), "rtk", "raw-output"); + const filePath = path.join(dir, `${now}-${commandSlug || "tool-output"}-${id}.log`); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(filePath, redaction.text); + + return { + id, + path: filePath, + bytes: Buffer.byteLength(redaction.text, "utf8"), + sha256: crypto.createHash("sha256").update(redaction.text).digest("hex"), + redacted: redaction.redacted, + }; +} + +export function readRtkRawOutput(pointerId: string): string | null { + const dir = path.join(dataDir(), "rtk", "raw-output"); + if (!fs.existsSync(dir)) return null; + const entry = fs + .readdirSync(dir) + .find((file) => file.endsWith(".log") && file.includes(pointerId)); + if (!entry) return null; + const fullPath = path.join(dir, entry); + if (!fullPath.startsWith(dir)) return null; + return fs.readFileSync(fullPath, "utf8"); +} diff --git a/open-sse/services/compression/engines/rtk/smartTruncate.ts b/open-sse/services/compression/engines/rtk/smartTruncate.ts new file mode 100644 index 0000000000..8af0e708c2 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/smartTruncate.ts @@ -0,0 +1,67 @@ +export interface SmartTruncateOptions { + maxLines?: number; + maxChars?: number; + preserveHead?: number; + preserveTail?: number; + priorityPatterns?: RegExp[]; +} + +export function smartTruncate( + text: string, + options: SmartTruncateOptions = {} +): { + text: string; + truncated: boolean; + droppedLines: number; +} { + const maxChars = Math.max(0, Math.floor(options.maxChars ?? 0)); + const maxLines = Math.max(0, Math.floor(options.maxLines ?? 0)); + const lines = text.split(/\r?\n/); + const overLineLimit = maxLines > 0 && lines.length > maxLines; + const overCharLimit = maxChars > 0 && text.length > maxChars; + if (!overLineLimit && !overCharLimit) { + return { text, truncated: false, droppedLines: 0 }; + } + + const preserveHead = Math.max(0, Math.floor(options.preserveHead ?? 20)); + const preserveTail = Math.max(0, Math.floor(options.preserveTail ?? 20)); + const priorityPatterns = options.priorityPatterns ?? []; + const priorityLines = priorityPatterns.length + ? lines.filter((line) => priorityPatterns.some((pattern) => pattern.test(line))) + : []; + + const head = lines.slice(0, preserveHead); + const tail = preserveTail > 0 ? lines.slice(-preserveTail) : []; + const selected = [...head]; + for (const line of priorityLines) { + if (!selected.includes(line)) selected.push(line); + } + const tailStart = lines.length - tail.length; + tail.forEach((line, offset) => { + const originalIndex = tailStart + offset; + if (originalIndex >= preserveHead && !selected.includes(line)) selected.push(line); + }); + + const droppedLines = Math.max(0, lines.length - selected.length); + let result = [ + ...selected.slice(0, head.length), + `[rtk:truncated ${droppedLines} lines]`, + ...selected.slice(head.length), + ].join("\n"); + + if (maxChars > 0 && result.length > maxChars) { + const marker = "\n[rtk:truncated by chars]\n"; + const budget = Math.max(0, maxChars - marker.length); + if (budget === 0) { + result = marker.slice(0, maxChars); + return { text: result, truncated: true, droppedLines }; + } + const headChars = Math.ceil(budget * 0.55); + const tailChars = Math.max(0, budget - headChars); + const tailText = tailChars > 0 ? result.slice(-tailChars) : ""; + result = `${result.slice(0, headChars)}${marker}${tailText}`; + if (result.length > maxChars) result = result.slice(0, maxChars); + } + + return { text: result, truncated: true, droppedLines }; +} diff --git a/open-sse/services/compression/engines/rtk/verify.ts b/open-sse/services/compression/engines/rtk/verify.ts new file mode 100644 index 0000000000..c59a5e9102 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/verify.ts @@ -0,0 +1,107 @@ +import { estimateCompressionTokens } from "../../stats.ts"; +import { + getRtkFilterLoadDiagnostics, + loadRtkFilters, + type RtkFilterLoadDiagnostic, +} from "./filterLoader.ts"; +import { applyLineFilter } from "./lineFilter.ts"; + +export interface RtkFilterTestOutcome { + filterId: string; + testName: string; + passed: boolean; + actual: string; + expected: string; +} + +export interface RtkFilterBenchmarkRow { + category: string; + filters: number; + tests: number; + averageSavingsPercent: number; +} + +export interface RtkVerifyResult { + passed: boolean; + outcomes: RtkFilterTestOutcome[]; + filtersWithoutTests: string[]; + benchmark: RtkFilterBenchmarkRow[]; + diagnostics: RtkFilterLoadDiagnostic[]; +} + +function trimComparable(value: string): string { + return value.replace(/\n+$/g, ""); +} + +export function runRtkFilterTests( + options: { + requireAll?: boolean; + customFiltersEnabled?: boolean; + trustProjectFilters?: boolean; + } = {} +): RtkVerifyResult { + const filters = loadRtkFilters({ + refresh: true, + customFiltersEnabled: options.customFiltersEnabled, + trustProjectFilters: options.trustProjectFilters, + }); + const outcomes: RtkFilterTestOutcome[] = []; + const filtersWithoutTests: string[] = []; + const benchmarkByCategory = new Map< + string, + { filters: Set; tests: number; savingsTotal: number } + >(); + + for (const filter of filters) { + const categoryStats = benchmarkByCategory.get(filter.category) ?? { + filters: new Set(), + tests: 0, + savingsTotal: 0, + }; + categoryStats.filters.add(filter.id); + benchmarkByCategory.set(filter.category, categoryStats); + + if (filter.tests.length === 0) { + filtersWithoutTests.push(filter.id); + continue; + } + + for (const test of filter.tests) { + const result = applyLineFilter(test.input, filter).text; + const actual = trimComparable(result); + const expected = trimComparable(test.expected); + const originalTokens = estimateCompressionTokens(test.input); + const compressedTokens = estimateCompressionTokens(result); + const savings = + originalTokens > 0 ? ((originalTokens - compressedTokens) / originalTokens) * 100 : 0; + categoryStats.tests += 1; + categoryStats.savingsTotal += Math.max(0, savings); + outcomes.push({ + filterId: filter.id, + testName: test.name, + passed: actual === expected, + actual, + expected, + }); + } + } + + const benchmark = Array.from(benchmarkByCategory.entries()) + .map(([category, value]) => ({ + category, + filters: value.filters.size, + tests: value.tests, + averageSavingsPercent: + value.tests > 0 ? Math.round((value.savingsTotal / value.tests) * 100) / 100 : 0, + })) + .sort((a, b) => a.category.localeCompare(b.category)); + + const failed = outcomes.some((outcome) => !outcome.passed); + return { + passed: !failed && (!options.requireAll || filtersWithoutTests.length === 0), + outcomes, + filtersWithoutTests, + benchmark, + diagnostics: options.customFiltersEnabled === false ? [] : getRtkFilterLoadDiagnostics(), + }; +} diff --git a/open-sse/services/compression/engines/types.ts b/open-sse/services/compression/engines/types.ts new file mode 100644 index 0000000000..b270d8d1d5 --- /dev/null +++ b/open-sse/services/compression/engines/types.ts @@ -0,0 +1,59 @@ +import type { CompressionConfig, CompressionResult } from "../types.ts"; + +export type CompressionEngineTarget = "messages" | "tool_results" | "code_blocks"; + +export interface EngineConfigField { + key: string; + type: "boolean" | "number" | "string" | "select" | "multiselect"; + label: string; + i18nKey?: string; + description?: string; + defaultValue: unknown; + options?: Array<{ value: string; label: string }>; + min?: number; + max?: number; +} + +export interface EngineValidationResult { + valid: boolean; + errors: string[]; +} + +export interface CompressionEngineMetadata { + id: string; + name: string; + description: string; + inputScope: "messages" | "tool-results" | "mixed"; + targetLatencyMs: number; + supportsPreview: boolean; + stable: boolean; +} + +export interface CompressionEngineApplyOptions { + model?: string; + supportsVision?: boolean | null; + config?: CompressionConfig; + compressionComboId?: string | null; + stepConfig?: Record; +} + +export interface CompressionEngine { + id: string; + name: string; + description: string; + icon: string; + targets: CompressionEngineTarget[]; + stackable: boolean; + stackPriority: number; + metadata: CompressionEngineMetadata; + apply(body: Record, options?: CompressionEngineApplyOptions): CompressionResult; + compress(body: Record, config?: Record): CompressionResult; + getConfigSchema(): EngineConfigField[]; + validateConfig(config: Record): EngineValidationResult; +} + +export interface EngineRegistryEntry { + engine: CompressionEngine; + enabled: boolean; + config: Record; +} diff --git a/open-sse/services/compression/index.ts b/open-sse/services/compression/index.ts index efc5b4ddb6..7f16cd9f00 100644 --- a/open-sse/services/compression/index.ts +++ b/open-sse/services/compression/index.ts @@ -7,6 +7,12 @@ export type { CavemanRule, CavemanIntensity, CavemanOutputModeConfig, + RtkConfig, + RtkIntensity, + RtkRawOutputRetention, + CompressionEngineId, + CompressionLanguageConfig, + CompressionPipelineStep, AggressiveConfig, AgingThresholds, ToolStrategiesConfig, @@ -18,6 +24,8 @@ export { DEFAULT_COMPRESSION_CONFIG, DEFAULT_CAVEMAN_CONFIG, DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG, + DEFAULT_RTK_CONFIG, + DEFAULT_COMPRESSION_LANGUAGE_CONFIG, DEFAULT_AGGRESSIVE_CONFIG, } from "./types.ts"; @@ -32,6 +40,19 @@ export { export { cavemanCompress, applyRulesToText } from "./caveman.ts"; export { getRulesForContext, getCavemanRuleMetadata, CAVEMAN_RULES } from "./cavemanRules.ts"; +export { + getAvailableLanguagePacks, + listCavemanRulePacks, + loadAllRulesForLanguage, + loadCavemanFileRules, + loadRulePack, + validateRulePack, +} from "./ruleLoader.ts"; +export type { RulePackMetadata } from "./ruleLoader.ts"; +export { + detectCompressionLanguage, + listSupportedCompressionLanguages, +} from "./languageDetector.ts"; export { extractPreservedBlocks, restorePreservedBlocks, @@ -62,8 +83,62 @@ export { applyCompression, checkComboOverride, shouldAutoTrigger, + applyStackedCompression, } from "./strategySelector.ts"; +export type { + CompressionEngine, + CompressionEngineApplyOptions, + CompressionEngineMetadata, + CompressionEngineTarget, + EngineConfigField, + EngineRegistryEntry, + EngineValidationResult, +} from "./engines/types.ts"; + +export { + registerEngine, + registerCompressionEngine, + unregisterCompressionEngine, + getEngine, + getEngineEntry, + getCompressionEngine, + listEngines, + listCompressionEngines, + listEnabledEngines, + setEngineEnabled, + updateEngineConfig, + clearCompressionEngineRegistry, +} from "./engines/registry.ts"; +export { registerBuiltinCompressionEngines } from "./engines/index.ts"; + +export { applyRtkCompression, processRtkText, rtkEngine } from "./engines/rtk/index.ts"; +export { + detectCommandFromText, + detectCommandOutput, + detectCommandType, + type CommandDetectionResult, +} from "./engines/rtk/commandDetector.ts"; +export { + loadRtkFilters, + getRtkFilterCatalog, + matchRtkFilter, + getRtkFilterLoadDiagnostics, +} from "./engines/rtk/filterLoader.ts"; +export { runRtkFilterTests } from "./engines/rtk/verify.ts"; +export { + maybePersistRtkRawOutput, + readRtkRawOutput, + redactRtkRawOutput, +} from "./engines/rtk/rawOutput.ts"; +export { + detectCodeLanguage, + normalizeCodeLanguage, + stripCode, + stripCodeComments, +} from "./engines/rtk/codeStripper.ts"; +export type { CodeLanguage, CodeStripperOptions } from "./engines/rtk/codeStripper.ts"; + export { RuleBasedSummarizer, createSummarizer } from "./summarizer.ts"; export { compressToolResult } from "./toolResultCompressor.ts"; diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts new file mode 100644 index 0000000000..3b3aa9b002 --- /dev/null +++ b/open-sse/services/compression/languageDetector.ts @@ -0,0 +1,18 @@ +const LANGUAGE_HINTS: Record = { + "pt-BR": [/\b(?:voce|você|preciso|arquivo|codigo|código|erro|falha|obrigado)\b/i], + es: [/\b(?:necesito|archivo|codigo|código|error|fallo|gracias|puedes)\b/i], + de: [/\b(?:ich|datei|fehler|bitte|kannst|konfiguration|danke)\b/i], + fr: [/\b(?:fichier|erreur|merci|peux|configuration|besoin)\b/i], + ja: [/[\u3040-\u30ff]/], +}; + +export function detectCompressionLanguage(text: string): string { + for (const [language, patterns] of Object.entries(LANGUAGE_HINTS)) { + if (patterns.some((pattern) => pattern.test(text))) return language; + } + return "en"; +} + +export function listSupportedCompressionLanguages(): string[] { + return ["en", ...Object.keys(LANGUAGE_HINTS)]; +} diff --git a/open-sse/services/compression/outputMode.ts b/open-sse/services/compression/outputMode.ts index 478ae863f2..9f4e7bfe45 100644 --- a/open-sse/services/compression/outputMode.ts +++ b/open-sse/services/compression/outputMode.ts @@ -19,11 +19,43 @@ export interface CavemanOutputModeResult { skippedReason?: string; } -const CAVEMAN_INSTRUCTION_BY_INTENSITY = { - lite: "Respond concise. Drop filler, pleasantries, hedging. Keep full sentences, technical terms, code, errors, URLs, and identifiers exact.", - full: "Respond terse like smart caveman. Drop articles, filler, pleasantries, hedging. Fragments OK. Keep all technical substance, code, errors, URLs, identifiers exact.", - ultra: - "Respond ultra terse. Use short technical prose, arrows for causality, common abbreviations like DB/auth/config/req/res/fn. Never abbreviate code symbols, API names, error strings, URLs, or identifiers.", +const CAVEMAN_INSTRUCTION_BY_LANGUAGE = { + en: { + lite: "Respond concise. Drop filler, pleasantries, hedging. Keep full sentences, technical terms, code, errors, URLs, and identifiers exact.", + full: "Respond terse like smart caveman. Drop articles, filler, pleasantries, hedging. Fragments OK. Keep all technical substance, code, errors, URLs, identifiers exact.", + ultra: + "Respond ultra terse. Use short technical prose, arrows for causality, common abbreviations like DB/auth/config/req/res/fn. Never abbreviate code symbols, API names, error strings, URLs, or identifiers.", + }, + "pt-BR": { + lite: "Responda conciso. Remova enrolacao, cortesias e incerteza. Preserve termos tecnicos, codigo, erros, URLs e identificadores exatamente.", + full: "Responda seco e compacto. Frases curtas OK. Preserve todo conteudo tecnico, codigo, erros, URLs e identificadores exatamente.", + ultra: + "Responda ultra compacto. Use prosa tecnica curta e abreviacoes comuns como DB/auth/config/req/res/fn. Nunca abrevie simbolos de codigo, APIs, erros, URLs ou identificadores.", + }, + es: { + lite: "Responde conciso. Quita relleno, cortesias y dudas. Conserva terminos tecnicos, codigo, errores, URLs e identificadores exactos.", + full: "Responde seco y compacto. Fragmentos OK. Conserva todo el contenido tecnico, codigo, errores, URLs e identificadores exactos.", + ultra: + "Responde ultra compacto. Usa prosa tecnica corta y abreviaturas comunes como DB/auth/config/req/res/fn. Nunca abrevies simbolos de codigo, APIs, errores, URLs o identificadores.", + }, + de: { + lite: "Antworte knapp. Entferne Fuellwoerter, Hoeflichkeit und Unsicherheit. Bewahre Fachbegriffe, Code, Fehler, URLs und Bezeichner exakt.", + full: "Antworte sehr knapp. Fragmente OK. Bewahre alle technischen Inhalte, Code, Fehler, URLs und Bezeichner exakt.", + ultra: + "Antworte ultra knapp. Nutze kurze technische Prosa und uebliche Abkuerzungen wie DB/auth/config/req/res/fn. Code-Symbole, APIs, Fehler, URLs und Bezeichner nie abkuerzen.", + }, + fr: { + lite: "Reponds concis. Retire remplissage, politesses et hesitations. Garde termes techniques, code, erreurs, URLs et identifiants exacts.", + full: "Reponds tres compact. Fragments OK. Garde tout le contenu technique, code, erreurs, URLs et identifiants exacts.", + ultra: + "Reponds ultra compact. Utilise une prose technique courte et des abreviations communes comme DB/auth/config/req/res/fn. N'abrege jamais symboles de code, APIs, erreurs, URLs ou identifiants.", + }, + ja: { + lite: "簡潔に回答。冗長表現、挨拶、曖昧表現を削る。技術用語、コード、エラー、URL、識別子は正確に保持。", + full: "短く圧縮して回答。断片文可。技術内容、コード、エラー、URL、識別子は正確に保持。", + ultra: + "超短く回答。DB/auth/config/req/res/fn など一般的な略語は可。コード記号、API名、エラー文字列、URL、識別子は省略しない。", + }, } as const; const CAVEMAN_OUTPUT_MARKER = "[OmniRoute Caveman Output Mode]"; @@ -60,14 +92,21 @@ export function shouldBypassCavemanOutputMode(messages: ChatMessage[]): string | return null; } -export function buildCavemanOutputInstruction(config: CavemanOutputModeConfig): string { +export function buildCavemanOutputInstruction( + config: CavemanOutputModeConfig, + language = "en" +): string { const intensity = config.intensity ?? "full"; - return `${CAVEMAN_OUTPUT_MARKER}\n${CAVEMAN_INSTRUCTION_BY_INTENSITY[intensity]}`; + const instructions = + CAVEMAN_INSTRUCTION_BY_LANGUAGE[language as keyof typeof CAVEMAN_INSTRUCTION_BY_LANGUAGE] ?? + CAVEMAN_INSTRUCTION_BY_LANGUAGE.en; + return `${CAVEMAN_OUTPUT_MARKER}\n${instructions[intensity]}`; } export function applyCavemanOutputMode( body: ChatRequestBody, - options?: Partial + options?: Partial, + language = "en" ): CavemanOutputModeResult { const config: CavemanOutputModeConfig = { ...DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG, @@ -84,7 +123,7 @@ export function applyCavemanOutputMode( return { body: { ...body, - instructions: `${body.instructions.trim()}\n\n${buildCavemanOutputInstruction(config)}`, + instructions: `${body.instructions.trim()}\n\n${buildCavemanOutputInstruction(config, language)}`, }, applied: true, }; @@ -105,7 +144,7 @@ export function applyCavemanOutputMode( ); if (alreadyApplied) return { body, applied: false, skippedReason: "already_applied" }; - const instruction = buildCavemanOutputInstruction(config); + const instruction = buildCavemanOutputInstruction(config, language); const nextMessages = [...messages]; const first = nextMessages[0]; diff --git a/open-sse/services/compression/ruleLoader.ts b/open-sse/services/compression/ruleLoader.ts new file mode 100644 index 0000000000..ba149d8515 --- /dev/null +++ b/open-sse/services/compression/ruleLoader.ts @@ -0,0 +1,206 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { CavemanIntensity, CavemanRule } from "./types.ts"; + +type CavemanRuleCategory = NonNullable; +type CavemanRuleContext = CavemanRule["context"]; + +interface FileRule { + name: string; + pattern: string; + replacement: string; + context?: CavemanRuleContext; + category?: CavemanRuleCategory; + minIntensity?: CavemanIntensity; + description?: string; +} + +interface RulePack { + language: string; + category: string; + rules: FileRule[]; +} + +export interface RulePackMetadata { + language: string; + categories: string[]; + ruleCount: number; +} + +const VALID_CONTEXTS = new Set(["all", "user", "system", "assistant"]); +const VALID_CATEGORIES = new Set(["filler", "context", "structural", "dedup", "terse", "ultra"]); +const VALID_INTENSITIES = new Set(["lite", "full", "ultra"]); +const cache = new Map(); + +function getRulesDir(): string { + const moduleDir = path.dirname(fileURLToPath(import.meta.url)); + const candidates = [ + path.join(moduleDir, "rules"), + path.join(moduleDir, "..", "services", "compression", "rules"), + path.join(process.cwd(), "open-sse", "services", "compression", "rules"), + path.join(process.cwd(), "app", "open-sse", "services", "compression", "rules"), + ]; + return ( + candidates.find((candidate, index) => { + return candidates.indexOf(candidate) === index && fs.existsSync(candidate); + }) ?? candidates[0] + ); +} + +function compileRule(rule: FileRule, source: string): CavemanRule { + try { + return { + name: rule.name, + pattern: new RegExp(rule.pattern, "gi"), + replacement: rule.replacement, + context: rule.context ?? "all", + category: rule.category ?? "filler", + minIntensity: rule.minIntensity ?? "lite", + description: rule.description, + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid Caveman rule pattern in ${source}:${rule.name}: ${message}`); + } +} + +export function validateRulePack(pack: unknown): { valid: boolean; errors: string[] } { + const errors: string[] = []; + if (!pack || typeof pack !== "object") { + return { valid: false, errors: ["Rule pack must be an object"] }; + } + + const value = pack as Partial; + if (typeof value.language !== "string" || !value.language.trim()) { + errors.push("language must be a non-empty string"); + } + if (typeof value.category !== "string" || !value.category.trim()) { + errors.push("category must be a non-empty string"); + } + if (!Array.isArray(value.rules)) { + errors.push("rules must be an array"); + } else { + value.rules.forEach((rule, index) => { + if (!rule || typeof rule !== "object") { + errors.push(`rules[${index}] must be an object`); + return; + } + const entry = rule as Partial; + if (typeof entry.name !== "string" || !entry.name.trim()) { + errors.push(`rules[${index}].name must be a non-empty string`); + } + if (typeof entry.pattern !== "string" || !entry.pattern.trim()) { + errors.push(`rules[${index}].pattern must be a non-empty string`); + } else { + try { + new RegExp(entry.pattern, "gi"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + errors.push(`rules[${index}].pattern is invalid: ${message}`); + } + } + if (typeof entry.replacement !== "string") { + errors.push(`rules[${index}].replacement must be a string`); + } + if (entry.context !== undefined && !VALID_CONTEXTS.has(entry.context)) { + errors.push(`rules[${index}].context is invalid`); + } + if (entry.category !== undefined && !VALID_CATEGORIES.has(entry.category)) { + errors.push(`rules[${index}].category is invalid`); + } + if (entry.minIntensity !== undefined && !VALID_INTENSITIES.has(entry.minIntensity)) { + errors.push(`rules[${index}].minIntensity is invalid`); + } + }); + } + return { valid: errors.length === 0, errors }; +} + +function readPack(language: string, category: string): RulePack | null { + const filename = path.join(getRulesDir(), language, `${category}.json`); + if (!fs.existsSync(filename)) return null; + const parsed = JSON.parse(fs.readFileSync(filename, "utf8")) as unknown; + const validation = validateRulePack(parsed); + if (!validation.valid) { + throw new Error( + `Invalid Caveman rule pack ${language}/${category}: ${validation.errors.join("; ")}` + ); + } + return parsed as RulePack; +} + +export function loadRulePack( + language: string, + category: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + const key = `${getRulesDir()}:${language}:${category}`; + if (cache.has(key) && !options.refresh) return cache.get(key) ?? []; + + const pack = readPack(language, category); + if (!pack) { + cache.set(key, []); + return []; + } + + const rules = pack.rules.map((rule) => compileRule(rule, `${language}/${category}`)); + cache.set(key, rules); + return rules; +} + +export function loadAllRulesForLanguage( + language: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + const key = `${getRulesDir()}:${language}:*`; + if (cache.has(key) && !options.refresh) return cache.get(key) ?? []; + + const languageDir = path.join(getRulesDir(), language); + if (!fs.existsSync(languageDir)) { + cache.set(key, []); + return []; + } + + const rules = fs + .readdirSync(languageDir) + .filter((entry) => entry.endsWith(".json")) + .sort() + .flatMap((entry) => loadRulePack(language, path.basename(entry, ".json"), options)); + + cache.set(key, rules); + return rules; +} + +export function getAvailableLanguagePacks(): RulePackMetadata[] { + const root = getRulesDir(); + if (!fs.existsSync(root)) return []; + + return fs + .readdirSync(root) + .filter((entry) => fs.statSync(path.join(root, entry)).isDirectory()) + .map((language) => { + const categories = fs + .readdirSync(path.join(root, language)) + .filter((entry) => entry.endsWith(".json")) + .map((entry) => path.basename(entry, ".json")) + .sort(); + const ruleCount = categories.reduce( + (count, category) => count + loadRulePack(language, category).length, + 0 + ); + return { language, categories, ruleCount }; + }) + .sort((a, b) => a.language.localeCompare(b.language)); +} + +export function loadCavemanFileRules( + language: string, + options: { refresh?: boolean } = {} +): CavemanRule[] { + return loadAllRulesForLanguage(language, options); +} + +export function listCavemanRulePacks(): RulePackMetadata[] { + return getAvailableLanguagePacks(); +} diff --git a/open-sse/services/compression/rules/_schema.json b/open-sse/services/compression/rules/_schema.json new file mode 100644 index 0000000000..5182229389 --- /dev/null +++ b/open-sse/services/compression/rules/_schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OmniRoute Caveman Rule Pack", + "type": "object", + "required": ["language", "category", "rules"], + "properties": { + "language": { "type": "string" }, + "category": { "type": "string" }, + "rules": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "pattern", "replacement"], + "properties": { + "name": { "type": "string" }, + "pattern": { "type": "string" }, + "replacement": { "type": "string" }, + "context": { "enum": ["all", "user", "system", "assistant"] }, + "category": { "enum": ["filler", "context", "structural", "dedup", "terse", "ultra"] }, + "minIntensity": { "enum": ["lite", "full", "ultra"] }, + "description": { "type": "string" } + } + } + } + } +} diff --git a/open-sse/services/compression/rules/de/context.json b/open-sse/services/compression/rules/de/context.json new file mode 100644 index 0000000000..64173166bf --- /dev/null +++ b/open-sse/services/compression/rules/de/context.json @@ -0,0 +1,38 @@ +{ + "language": "de", + "category": "context", + "rules": [ + { + "name": "de_context_setup", + "pattern": "\\b(?:hier ist der code|unten ist der code|dies ist der code)\\b\\s*[:.]?\\s*", + "replacement": "Code:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_intent", + "pattern": "\\b(?:mein ziel ist|was ich brauche ist|ich versuche|die idee ist)\\b\\s*", + "replacement": "Ziel:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_question_directive", + "pattern": "\\b(?:kannst du erklaeren warum|kannst du erklären warum|kannst du zeigen wie)\\b\\s*", + "replacement": "Erklaere ", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "de_background", + "pattern": "\\b(?:wie du weisst|wie du weißt|wie bereits erwaehnt|wie bereits erwähnt)\\b[,.]?\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/de/filler.json b/open-sse/services/compression/rules/de/filler.json new file mode 100644 index 0000000000..f4f2e6eb62 --- /dev/null +++ b/open-sse/services/compression/rules/de/filler.json @@ -0,0 +1,70 @@ +{ + "language": "de", + "category": "filler", + "rules": [ + { + "name": "de_polite_framing", + "pattern": "\\b(?:bitte|kannst du|koenntest du|könntest du|ich moechte|ich möchte|ich brauche)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_pleasantries", + "pattern": "\\b(?:hallo|guten morgen|guten tag|danke|vielen dank)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_hedging", + "pattern": "\\b(?:ich denke dass|ich glaube dass|es scheint dass|vielleicht|wahrscheinlich|moeglicherweise|möglicherweise)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_filler_adverbs", + "pattern": "\\b(?:grundsaetzlich|grundsätzlich|eigentlich|buchstaeblich|buchstäblich|einfach)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_self_reference", + "pattern": "^(?:ich versuche|ich will|ich moechte|ich möchte|ich brauche)\\b\\s*", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_verbose_request", + "pattern": "\\b(?:kannst du erklaeren|kannst du erklären|koenntest du erklaeren|könntest du erklären)\\b\\s*", + "replacement": "Erklaere ", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_qualifiers", + "pattern": "\\b(?:ein bisschen|etwas|irgendwie|mehr oder weniger)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "de_softeners", + "pattern": "\\b(?:wenn moeglich|wenn möglich|wenn du zeit hast|ohne eile)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/de/structural.json b/open-sse/services/compression/rules/de/structural.json new file mode 100644 index 0000000000..7107c977b3 --- /dev/null +++ b/open-sse/services/compression/rules/de/structural.json @@ -0,0 +1,30 @@ +{ + "language": "de", + "category": "structural", + "rules": [ + { + "name": "de_purpose", + "pattern": "\\b(?:um zu|mit dem ziel|damit ich)\\b\\s*", + "replacement": "um ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "de_connectors", + "pattern": "\\b(?:ausserdem|außerdem|zusaetzlich|zusätzlich|darueber hinaus|darüber hinaus)\\b\\s*", + "replacement": "auch ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "de_emphasis", + "pattern": "\\b(?:sehr|wirklich|extrem|ziemlich)\\s+(?=[a-zäöüß])", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/context.json b/open-sse/services/compression/rules/en/context.json new file mode 100644 index 0000000000..9f0964692e --- /dev/null +++ b/open-sse/services/compression/rules/en/context.json @@ -0,0 +1,70 @@ +{ + "language": "en", + "category": "context", + "rules": [ + { + "name": "compound_collapse", + "pattern": "\\band any potential\\b", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "full" + }, + { + "name": "explanatory_prefix", + "pattern": "\\b(?:The function appears to be handling|The code seems to|The class is|This module is)\\b", + "replacement": "Function:", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "question_to_directive", + "pattern": "\\b(?:Can you explain why|Could you show me how|Would you tell me|Can you tell me)\\b\\s*", + "replacement": "Explain why", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "context_setup", + "pattern": "\\b(?:I have the following code|Here is my code|Below is the code)\\b\\s*[:.]?\\s*", + "replacement": "Code:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "intent_clarification", + "pattern": "\\b(?:What I'm trying to do is|My objective is to|What I need is|I'm aiming to)\\b\\s*", + "replacement": "Goal:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "background_removal", + "pattern": "\\b(?:As you may know,?\\s*|As we discussed earlier,?\\s*)", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "meta_commentary", + "pattern": "^(?:Note that|Keep in mind that|Remember that)\\b\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "purpose_statement", + "pattern": "\\b(?:for the purpose of|with the goal of|in an effort to|for every)\\b", + "replacement": "for", + "context": "all", + "category": "context", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/dedup.json b/open-sse/services/compression/rules/en/dedup.json new file mode 100644 index 0000000000..23f0820bfe --- /dev/null +++ b/open-sse/services/compression/rules/en/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "en", + "category": "dedup", + "rules": [ + { + "name": "repeated_context", + "pattern": "\\b(?:As we discussed earlier|As mentioned before|As previously stated|As I said before)\\b[,.]?\\s*", + "replacement": "See above. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "repeated_question", + "pattern": "\\b(?:Same question as before|I asked this earlier|This is the same question)\\b[,.]?\\s*", + "replacement": "[same question] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "reestablished_context", + "pattern": "\\b(?:Going back to the code above|Referring back to|Returning to)\\b\\s*", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "summary_replacement", + "pattern": "\\b(?:To summarize what we've discussed|In summary of our conversation|To recap)\\b[,.]?\\s*", + "replacement": "Summary: ", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/en/filler.json b/open-sse/services/compression/rules/en/filler.json new file mode 100644 index 0000000000..975d7c324c --- /dev/null +++ b/open-sse/services/compression/rules/en/filler.json @@ -0,0 +1,118 @@ +{ + "language": "en", + "category": "filler", + "rules": [ + { + "name": "pleasantries", + "pattern": "(?, + pipeline?: Array, + options?: { + model?: string; + supportsVision?: boolean | null; + config?: CompressionConfig; + compressionComboId?: string | null; + } +): CompressionResult { + const steps = + pipeline && pipeline.length > 0 + ? pipeline.map(normalizePipelineStep) + : [ + { engine: "rtk" as const, intensity: "standard" as const }, + { engine: "caveman" as const, intensity: "full" as const }, + ]; + registerBuiltinCompressionEngines(); + + let currentBody = body; + let compressed = false; + const techniques = new Set(); + const rules = new Set(); + const breakdown: NonNullable = []; + const rtkRawOutputPointers: NonNullable = []; + const start = performance.now(); + + for (const step of steps) { + const engine = getCompressionEngine(step.engine); + if (!engine) continue; + const result = engine.apply(currentBody, { + ...options, + compressionComboId: options?.compressionComboId ?? options?.config?.compressionComboId, + stepConfig: { + ...(step.config ?? {}), + ...(step.intensity ? { intensity: step.intensity } : {}), + }, + }); + if (result.stats) { + result.stats.techniquesUsed.forEach((technique) => techniques.add(technique)); + result.stats.rulesApplied?.forEach((rule) => rules.add(rule)); + result.stats.rtkRawOutputPointers?.forEach((pointer) => { + rtkRawOutputPointers.push(pointer); + }); + breakdown.push({ + engine: step.engine, + originalTokens: result.stats.originalTokens, + compressedTokens: result.stats.compressedTokens, + savingsPercent: result.stats.savingsPercent, + techniquesUsed: result.stats.techniquesUsed, + ...(result.stats.rulesApplied ? { rulesApplied: result.stats.rulesApplied } : {}), + ...(result.stats.durationMs !== undefined ? { durationMs: result.stats.durationMs } : {}), + }); + } + if (result.compressed) { + currentBody = result.body; + compressed = true; + } + } + + const stats = createCompressionStats( + body, + currentBody, + "stacked", + Array.from(techniques), + rules.size > 0 ? Array.from(rules) : undefined, + Math.round((performance.now() - start) * 100) / 100 + ); + stats.engine = "stacked"; + stats.compressionComboId = + options?.compressionComboId ?? options?.config?.compressionComboId ?? null; + stats.engineBreakdown = breakdown; + if (rtkRawOutputPointers.length > 0) { + const seenPointers = new Set(); + stats.rtkRawOutputPointers = rtkRawOutputPointers.filter((pointer) => { + if (seenPointers.has(pointer.id)) return false; + seenPointers.add(pointer.id); + return true; + }); + } + + return { + body: currentBody, + compressed, + stats, + }; +} diff --git a/open-sse/services/compression/types.ts b/open-sse/services/compression/types.ts index 443cc93fdf..abe842fb35 100644 --- a/open-sse/services/compression/types.ts +++ b/open-sse/services/compression/types.ts @@ -1,15 +1,26 @@ /** - * Compression Pipeline Types — Phase 1 (Lite) + Phase 2 (Standard/Caveman) + Phase 3 (Aggressive) + Phase 4 (Ultra) + * Compression Pipeline Types — Lite, Caveman, Aggressive, Ultra, RTK, and Stacked modes. * * Shared type definitions for the compression pipeline. * Phase 1: 'off' and 'lite' modes. * Phase 2: 'standard' mode (caveman engine). * Phase 3: 'aggressive' mode (summarization + tool compression + aging). * Phase 4: 'ultra' mode (heuristic token pruning + optional SLM tier). + * Phase 5: 'rtk' and 'stacked' modes (tool-output filters + multi-engine pipeline). */ -export type CompressionMode = "off" | "lite" | "standard" | "aggressive" | "ultra"; +export type CompressionMode = + | "off" + | "lite" + | "standard" + | "aggressive" + | "ultra" + | "rtk" + | "stacked"; export type CavemanIntensity = "lite" | "full" | "ultra"; +export type RtkIntensity = "minimal" | "standard" | "aggressive"; +export type RtkRawOutputRetention = "never" | "failures" | "always"; +export type CompressionEngineId = "lite" | "caveman" | "aggressive" | "ultra" | "rtk"; export interface CavemanRule { name: string; @@ -29,6 +40,9 @@ export interface CavemanConfig { minMessageLength: number; preservePatterns: string[]; intensity: CavemanIntensity; + language?: string; + autoDetectLanguage?: boolean; + enabledLanguagePacks?: string[]; } export interface CavemanOutputModeConfig { @@ -37,6 +51,36 @@ export interface CavemanOutputModeConfig { autoClarity: boolean; } +export interface RtkConfig { + enabled: boolean; + intensity: RtkIntensity; + applyToToolResults: boolean; + applyToCodeBlocks: boolean; + applyToAssistantMessages: boolean; + enabledFilters: string[]; + disabledFilters: string[]; + maxLinesPerResult: number; + maxCharsPerResult: number; + deduplicateThreshold: number; + customFiltersEnabled: boolean; + trustProjectFilters: boolean; + rawOutputRetention: RtkRawOutputRetention; + rawOutputMaxBytes: number; +} + +export interface CompressionLanguageConfig { + enabled: boolean; + defaultLanguage: string; + autoDetect: boolean; + enabledPacks: string[]; +} + +export interface CompressionPipelineStep { + engine: CompressionEngineId; + intensity?: CavemanIntensity | RtkIntensity; + config?: Record; +} + export interface CompressionConfig { enabled: boolean; defaultMode: CompressionMode; @@ -46,8 +90,12 @@ export interface CompressionConfig { preserveSystemPrompt: boolean; mcpDescriptionCompressionEnabled?: boolean; comboOverrides: Record; + compressionComboId?: string | null; + stackedPipeline?: CompressionPipelineStep[]; cavemanConfig?: CavemanConfig; cavemanOutputMode?: CavemanOutputModeConfig; + rtkConfig?: RtkConfig; + languageConfig?: CompressionLanguageConfig; aggressive?: AggressiveConfig; ultra?: UltraConfig; } @@ -58,6 +106,8 @@ export interface CompressionStats { savingsPercent: number; techniquesUsed: string[]; mode: CompressionMode; + engine?: string; + compressionComboId?: string | null; timestamp: number; rulesApplied?: string[]; durationMs?: number; @@ -65,6 +115,13 @@ export interface CompressionStats { validationErrors?: string[]; fallbackApplied?: boolean; preservedBlockCount?: number; + rtkRawOutputPointers?: Array<{ + id: string; + path: string; + bytes: number; + sha256: string; + redacted: boolean; + }>; realUsage?: { promptTokens?: number; completionTokens?: number; @@ -76,6 +133,15 @@ export interface CompressionStats { toolResultSavings: number; agingSavings: number; }; + engineBreakdown?: Array<{ + engine: string; + originalTokens: number; + compressedTokens: number; + savingsPercent: number; + techniquesUsed: string[]; + rulesApplied?: string[]; + durationMs?: number; + }>; } export interface CompressionResult { @@ -93,6 +159,11 @@ export const DEFAULT_COMPRESSION_CONFIG: CompressionConfig = { preserveSystemPrompt: true, mcpDescriptionCompressionEnabled: true, comboOverrides: {}, + compressionComboId: null, + stackedPipeline: [ + { engine: "rtk", intensity: "standard" }, + { engine: "caveman", intensity: "full" }, + ], }; export const DEFAULT_CAVEMAN_CONFIG: CavemanConfig = { @@ -110,6 +181,30 @@ export const DEFAULT_CAVEMAN_OUTPUT_MODE_CONFIG: CavemanOutputModeConfig = { autoClarity: true, }; +export const DEFAULT_RTK_CONFIG: RtkConfig = { + enabled: true, + intensity: "standard", + applyToToolResults: true, + applyToCodeBlocks: false, + applyToAssistantMessages: false, + enabledFilters: [], + disabledFilters: [], + maxLinesPerResult: 120, + maxCharsPerResult: 12000, + deduplicateThreshold: 3, + customFiltersEnabled: true, + trustProjectFilters: false, + rawOutputRetention: "never", + rawOutputMaxBytes: 1_048_576, +}; + +export const DEFAULT_COMPRESSION_LANGUAGE_CONFIG: CompressionLanguageConfig = { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], +}; + /** Aging thresholds for progressive message degradation (Phase 3) */ export interface AgingThresholds { fullSummary: number; diff --git a/package-lock.json b/package-lock.json index c9be8e0b17..8753eccd38 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,6 +79,7 @@ "husky": "^9.1.7", "jsdom": "^29.0.1", "lint-staged": "^16.2.7", + "node-loader": "^2.1.0", "prettier": "^3.8.1", "tailwindcss": "^4", "typescript": "^6.0.2", @@ -5569,6 +5570,16 @@ "require-from-string": "^2.0.2" } }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", @@ -6773,6 +6784,16 @@ "dev": true, "license": "MIT" }, + "node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -10080,6 +10101,21 @@ "dev": true, "license": "MIT" }, + "node_modules/loader-utils": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-2.0.4.tgz", + "integrity": "sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^2.1.2" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -11260,6 +11296,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/node-loader": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-loader/-/node-loader-2.1.0.tgz", + "integrity": "sha512-OwjPkyh8+7jW8DMd/iq71uU1Sspufr/C2+c3t0p08J3CrM9ApZ4U53xuisNrDXOHyGi5OYHgtfmmh+aK9zJA6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "loader-utils": "^2.0.3" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, "node_modules/node-machine-id": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/node-machine-id/-/node-machine-id-1.1.12.tgz", diff --git a/package.json b/package.json index 9e6f28ddce..24568eb1fb 100644 --- a/package.json +++ b/package.json @@ -174,6 +174,7 @@ "husky": "^9.1.7", "jsdom": "^29.0.1", "lint-staged": "^16.2.7", + "node-loader": "^2.1.0", "prettier": "^3.8.1", "tailwindcss": "^4", "typescript": "^6.0.2", diff --git a/scripts/pack-artifact-policy.ts b/scripts/pack-artifact-policy.ts index b2e55c0232..06793c10c6 100644 --- a/scripts/pack-artifact-policy.ts +++ b/scripts/pack-artifact-policy.ts @@ -42,6 +42,8 @@ export const APP_STAGING_ALLOWED_PATH_PREFIXES: string[] = [ ".next/", "data/", "node_modules/", + "open-sse/services/compression/engines/rtk/filters/", + "open-sse/services/compression/rules/", "public/", "src/lib/db/migrations/", "src/mitm/", @@ -88,6 +90,8 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_PATH_PREFIXES: string[] = [ ]; export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [ + "app/open-sse/services/compression/engines/rtk/filters/generic-output.json", + "app/open-sse/services/compression/rules/en/filler.json", "app/server.js", "app/server-ws.mjs", "app/responses-ws-proxy.mjs", diff --git a/scripts/prepublish.ts b/scripts/prepublish.ts index 199c76c68d..8b54f60067 100644 --- a/scripts/prepublish.ts +++ b/scripts/prepublish.ts @@ -475,6 +475,23 @@ if (existsSync(migrationsSrc)) { cpSync(migrationsSrc, migrationsDest, { recursive: true, force: true }); } +const runtimeAssetDirs = [ + { + source: join(ROOT, "open-sse", "services", "compression", "engines", "rtk", "filters"), + destination: join(APP_DIR, "open-sse", "services", "compression", "engines", "rtk", "filters"), + }, + { + source: join(ROOT, "open-sse", "services", "compression", "rules"), + destination: join(APP_DIR, "open-sse", "services", "compression", "rules"), + }, +]; +for (const assetDir of runtimeAssetDirs) { + if (existsSync(assetDir.source)) { + mkdirSync(dirname(assetDir.destination), { recursive: true }); + cpSync(assetDir.source, assetDir.destination, { recursive: true, force: true }); + } +} + // ── Step 10: Ensure data/ directory exists ────────────────── mkdirSync(join(APP_DIR, "data"), { recursive: true }); diff --git a/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx b/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx index c0d08da196..6c84c6b077 100644 --- a/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx +++ b/src/app/(dashboard)/dashboard/analytics/CompressionAnalyticsTab.tsx @@ -358,8 +358,8 @@ export default function CompressionAnalyticsTab() { info Compression analytics: Token savings tracked per mode (off, lite, - standard, aggressive, ultra) and provider. Hover over charts for details. Use the time - selector to view different time periods. + standard, aggressive, ultra, RTK, stacked), engine, compression combo, and provider. Hover + over charts for details. Use the time selector to view different time periods. diff --git a/src/app/(dashboard)/dashboard/compression/page.tsx b/src/app/(dashboard)/dashboard/compression/page.tsx index 7f4c6363b6..a347872010 100644 --- a/src/app/(dashboard)/dashboard/compression/page.tsx +++ b/src/app/(dashboard)/dashboard/compression/page.tsx @@ -1,20 +1,5 @@ -"use client"; - -import CompressionSettingsTab from "@/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab"; +import { redirect } from "next/navigation"; export default function CompressionPage() { - return ( -
-
-

- compress - Compression -

-

- Configure context compression settings to reduce token usage and costs. -

-
- -
- ); + redirect("/dashboard/context/caveman"); } diff --git a/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx b/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx new file mode 100644 index 0000000000..cfac49fa4c --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/caveman/CavemanContextPageClient.tsx @@ -0,0 +1,262 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useTranslations } from "next-intl"; +import CompressionSettingsTab from "@/app/(dashboard)/dashboard/settings/components/CompressionSettingsTab"; + +type AnalyticsSummary = { + totalRequests: number; + totalTokensSaved: number; + avgSavingsPct: number; + avgDurationMs: number; + byEngine?: Record; + byMode?: Record; + last24h?: Array<{ hour: string; count: number; tokensSaved: number }>; +}; + +type LanguageConfig = { + enabled: boolean; + defaultLanguage: string; + autoDetect: boolean; + enabledPacks: string[]; +}; + +type OutputModeConfig = { + enabled: boolean; + intensity: "lite" | "full" | "ultra"; + autoClarity: boolean; +}; + +type CompressionSettings = { + languageConfig?: LanguageConfig; + cavemanOutputMode?: OutputModeConfig; +}; + +type LanguagePack = { language: string; ruleCount: number; categories?: string[] }; + +function formatNumber(value: number | undefined): string { + return new Intl.NumberFormat().format(value ?? 0); +} + +export default function CavemanContextPageClient() { + const t = useTranslations("contextCaveman"); + const [analytics, setAnalytics] = useState(null); + const [settings, setSettings] = useState(null); + const [languagePacks, setLanguagePacks] = useState([]); + const [saving, setSaving] = useState(false); + + const refreshSettings = () => { + fetch("/api/context/caveman/config") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setSettings(data)) + .catch(() => setSettings(null)); + }; + + useEffect(() => { + fetch("/api/context/analytics?since=7d") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setAnalytics(data)) + .catch(() => setAnalytics(null)); + fetch("/api/compression/language-packs") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setLanguagePacks(Array.isArray(data?.packs) ? data.packs : [])) + .catch(() => setLanguagePacks([])); + refreshSettings(); + }, []); + + const languageConfig: LanguageConfig = settings?.languageConfig ?? { + enabled: false, + defaultLanguage: "en", + autoDetect: true, + enabledPacks: ["en"], + }; + const outputMode: OutputModeConfig = settings?.cavemanOutputMode ?? { + enabled: false, + intensity: "full", + autoClarity: true, + }; + + const saveSettings = async (patch: Partial) => { + setSaving(true); + try { + const res = await fetch("/api/context/caveman/config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }); + if (res.ok) setSettings(await res.json()); + } finally { + setSaving(false); + } + }; + + const updateLanguageConfig = (patch: Partial) => { + saveSettings({ languageConfig: { ...languageConfig, ...patch } }); + }; + + const updateOutputMode = (patch: Partial) => { + saveSettings({ cavemanOutputMode: { ...outputMode, ...patch } }); + }; + + const togglePack = (language: string, enabled: boolean) => { + const enabledPacks = enabled + ? [...new Set([...languageConfig.enabledPacks, language])] + : languageConfig.enabledPacks.filter((pack) => pack !== language && pack !== "en"); + updateLanguageConfig({ enabledPacks }); + }; + + const cavemanStats = analytics?.byEngine?.caveman ?? analytics?.byEngine?.standard; + const modeBreakdown = useMemo(() => Object.entries(analytics?.byMode ?? {}), [analytics]); + const statCards = [ + [t("requests"), formatNumber(cavemanStats?.count ?? analytics?.totalRequests)], + [t("tokensSaved"), formatNumber(cavemanStats?.tokensSaved ?? analytics?.totalTokensSaved)], + [t("savingsPercent"), `${cavemanStats?.avgSavingsPct ?? analytics?.avgSavingsPct ?? 0}%`], + [t("avgLatency"), `${analytics?.avgDurationMs ?? 0}ms`], + ]; + const previewPrompt = `[OmniRoute Caveman Output Mode]\n${t(`preview.${outputMode.intensity}`)}`; + + return ( +
+
+
+ compress +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+ {statCards.map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))} +
+ +
+
+

{t("languagePacks")}

+

{t("languagePacksDesc")}

+
+
+ + + +
+
+ {languagePacks.map((pack) => ( + + ))} +
+
+ +
+
+

{t("analyticsTitle")}

+
+ {modeBreakdown.length === 0 ? ( +

{t("noAnalytics")}

+ ) : ( + modeBreakdown.map(([mode, stats]) => ( +
+ {mode} + + {formatNumber(stats.tokensSaved)} / {stats.avgSavingsPct}% + +
+ )) + )} +
+
+ +
+

{t("outputModeTitle")}

+

{t("outputModeDesc")}

+
+ + + +
+ 
+            {previewPrompt}
+
+

+ {t("bypassConditions")}: security, irreversible, clarification, order-sensitive +

+
+
+ + +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/context/caveman/page.tsx b/src/app/(dashboard)/dashboard/context/caveman/page.tsx new file mode 100644 index 0000000000..bfeae4c1cb --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/caveman/page.tsx @@ -0,0 +1,5 @@ +import CavemanContextPageClient from "./CavemanContextPageClient"; + +export default function CavemanContextPage() { + return ; +} diff --git a/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx b/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx new file mode 100644 index 0000000000..58149a9aa6 --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/combos/CompressionCombosPageClient.tsx @@ -0,0 +1,391 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useTranslations } from "next-intl"; + +type PipelineStep = { engine: string; intensity?: string }; +type CompressionCombo = { + id: string; + name: string; + description: string; + pipeline: PipelineStep[]; + languagePacks: string[]; + outputMode: boolean; + outputModeIntensity: string; + isDefault: boolean; +}; +type RoutingCombo = { id?: string; name?: string }; +type LanguagePack = { language: string; ruleCount: number }; + +const EMPTY_PIPELINE: PipelineStep[] = [ + { engine: "rtk", intensity: "standard" }, + { engine: "caveman", intensity: "full" }, +]; + +const ENGINE_INTENSITIES: Record = { + rtk: ["minimal", "standard", "aggressive"], + caveman: ["lite", "full", "ultra"], + lite: ["lite"], + aggressive: ["standard"], + ultra: ["ultra"], +}; + +export default function CompressionCombosPageClient() { + const t = useTranslations("contextCombos"); + const [combos, setCombos] = useState([]); + const [routingCombos, setRoutingCombos] = useState([]); + const [languagePacks, setLanguagePacks] = useState([]); + const [editingId, setEditingId] = useState(null); + const [name, setName] = useState(""); + const [description, setDescription] = useState(""); + const [pipeline, setPipeline] = useState(EMPTY_PIPELINE); + const [selectedPacks, setSelectedPacks] = useState(["en"]); + const [outputMode, setOutputMode] = useState(false); + const [outputModeIntensity, setOutputModeIntensity] = useState("full"); + const [assignmentIds, setAssignmentIds] = useState([]); + const [saving, setSaving] = useState(false); + + const refresh = () => { + fetch("/api/context/combos") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setCombos(Array.isArray(data?.combos) ? data.combos : [])) + .catch(() => {}); + }; + + useEffect(() => { + refresh(); + fetch("/api/combos") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setRoutingCombos(Array.isArray(data?.combos) ? data.combos : [])) + .catch(() => {}); + fetch("/api/compression/language-packs") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setLanguagePacks(Array.isArray(data?.packs) ? data.packs : [])) + .catch(() => {}); + }, []); + + const resetForm = () => { + setEditingId(null); + setName(""); + setDescription(""); + setPipeline(EMPTY_PIPELINE); + setSelectedPacks(["en"]); + setOutputMode(false); + setOutputModeIntensity("full"); + setAssignmentIds([]); + }; + + const loadAssignments = async (id: string) => { + const res = await fetch(`/api/context/combos/${id}/assignments`); + if (!res.ok) return []; + const data = await res.json(); + return Array.isArray(data?.assignments) + ? data.assignments.map((item: { routingComboId: string }) => item.routingComboId) + : []; + }; + + const editCombo = async (combo: CompressionCombo) => { + setEditingId(combo.id); + setName(combo.name); + setDescription(combo.description ?? ""); + setPipeline(combo.pipeline.length > 0 ? combo.pipeline : EMPTY_PIPELINE); + setSelectedPacks(combo.languagePacks?.length ? combo.languagePacks : ["en"]); + setOutputMode(Boolean(combo.outputMode)); + setOutputModeIntensity(combo.outputModeIntensity ?? "full"); + setAssignmentIds(await loadAssignments(combo.id)); + }; + + const saveCombo = async () => { + const trimmed = name.trim(); + if (!trimmed) return; + setSaving(true); + try { + const payload = { + name: trimmed, + description, + pipeline, + languagePacks: selectedPacks, + outputMode, + outputModeIntensity, + }; + const res = await fetch( + editingId ? `/api/context/combos/${editingId}` : "/api/context/combos", + { + method: editingId ? "PUT" : "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + } + ); + if (!res.ok) return; + const combo = await res.json(); + await fetch(`/api/context/combos/${combo.id}/assignments`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ routingComboIds: assignmentIds }), + }); + resetForm(); + refresh(); + } finally { + setSaving(false); + } + }; + + const deleteCombo = async (combo: CompressionCombo) => { + if (!confirm(t("deleteConfirm"))) return; + const res = await fetch(`/api/context/combos/${combo.id}`, { method: "DELETE" }); + if (res.ok) refresh(); + }; + + const setDefault = async (id: string) => { + const res = await fetch(`/api/context/combos/${id}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ isDefault: true }), + }); + if (res.ok) refresh(); + }; + + const updateStep = (index: number, patch: Partial) => { + setPipeline((current) => + current.map((step, stepIndex) => { + if (stepIndex !== index) return step; + const next = { ...step, ...patch }; + const allowed = ENGINE_INTENSITIES[next.engine] ?? ["standard"]; + return { + ...next, + intensity: allowed.includes(next.intensity ?? "") ? next.intensity : allowed[0], + }; + }) + ); + }; + + const togglePack = (language: string, enabled: boolean) => { + setSelectedPacks((current) => + enabled + ? [...new Set([...current, language])] + : current.filter((item) => item !== language && item !== "en") + ); + }; + + const toggleAssignment = (id: string, enabled: boolean) => { + setAssignmentIds((current) => + enabled ? [...new Set([...current, id])] : current.filter((item) => item !== id) + ); + }; + + return ( +
+
+
+ hub +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+
+ setName(event.target.value)} + placeholder={t("name")} + className="rounded-lg border border-border bg-bg px-3 py-2 text-sm text-text-main" + /> + setDescription(event.target.value)} + placeholder={t("descriptionField")} + className="rounded-lg border border-border bg-bg px-3 py-2 text-sm text-text-main" + /> +
+ +
+
+

{t("pipeline")}

+ +
+ {pipeline.map((step, index) => ( +
+ + + +
+ ))} +
+ +
+
+

{t("languagePacks")}

+
+ {languagePacks.map((pack) => ( + + ))} +
+
+
+

{t("outputMode")}

+ + +
+
+

{t("assignToRouting")}

+
+ {routingCombos.length === 0 ? ( +

{t("noAssignments")}

+ ) : ( + routingCombos.map((combo) => { + const id = combo.id ?? combo.name ?? ""; + if (!id) return null; + return ( + + ); + }) + )} +
+
+
+ +
+ + {editingId && ( + + )} +
+
+ +
+ {combos.map((combo) => ( +
+
+
+

{combo.name}

+

{combo.description}

+
+ {combo.isDefault && ( + + {t("default")} + + )} +
+
+ {combo.pipeline.map((step, index) => ( + + {index + 1}. {step.engine} + {step.intensity ? `:${step.intensity}` : ""} + + ))} +
+

+ {t("languagePacks")}: {combo.languagePacks.join(", ")} +

+
+ + {!combo.isDefault && ( + + )} + {!combo.isDefault && ( + + )} +
+
+ ))} +
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/context/combos/page.tsx b/src/app/(dashboard)/dashboard/context/combos/page.tsx new file mode 100644 index 0000000000..934ab7d451 --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/combos/page.tsx @@ -0,0 +1,5 @@ +import CompressionCombosPageClient from "./CompressionCombosPageClient"; + +export default function CompressionCombosPage() { + return ; +} diff --git a/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx new file mode 100644 index 0000000000..a7b148be8e --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx @@ -0,0 +1,299 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useTranslations } from "next-intl"; + +type RtkFilter = { + id: string; + name: string; + description: string; + commandTypes: string[]; + category: string; + priority: number; +}; + +type RtkConfig = { + enabled: boolean; + intensity: "minimal" | "standard" | "aggressive"; + applyToToolResults: boolean; + applyToAssistantMessages: boolean; + applyToCodeBlocks: boolean; + enabledFilters: string[]; + disabledFilters: string[]; + maxLinesPerResult: number; + maxCharsPerResult: number; + deduplicateThreshold: number; +}; + +type AnalyticsSummary = { + totalRequests: number; + totalTokensSaved: number; + avgSavingsPct: number; + byEngine?: Record; +}; + +type PreviewResult = { + text?: string; + compressed?: boolean; + originalTokens?: number; + compressedTokens?: number; + techniquesUsed?: string[]; + detection?: { type: string; confidence: number; category: string }; + error?: string; +}; + +const SAMPLE_OUTPUT = `$ npm run typecheck +src/lib/example.ts:10:15 - error TS2322: Type 'string' is not assignable to type 'number'. + +10 const value: number = "bad"; + ~~~~~ + +Found 1 error in src/lib/example.ts:10`; + +function formatNumber(value: number | undefined): string { + return new Intl.NumberFormat().format(value ?? 0); +} + +export default function RtkContextPageClient() { + const t = useTranslations("contextRtk"); + const [filters, setFilters] = useState([]); + const [config, setConfig] = useState(null); + const [analytics, setAnalytics] = useState(null); + const [sample, setSample] = useState(SAMPLE_OUTPUT); + const [preview, setPreview] = useState(null); + const [saving, setSaving] = useState(false); + + useEffect(() => { + fetch("/api/context/rtk/filters") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setFilters(Array.isArray(data?.filters) ? data.filters : [])) + .catch(() => {}); + fetch("/api/context/rtk/config") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setConfig(data)) + .catch(() => {}); + fetch("/api/context/analytics?since=7d") + .then((res) => (res.ok ? res.json() : null)) + .then((data) => setAnalytics(data)) + .catch(() => {}); + }, []); + + const groupedFilters = useMemo(() => { + return filters.reduce>((groups, filter) => { + groups[filter.category] = [...(groups[filter.category] ?? []), filter]; + return groups; + }, {}); + }, [filters]); + + const activeFilterCount = useMemo(() => { + if (!config) return 0; + if (config.enabledFilters.length > 0) return config.enabledFilters.length; + return filters.filter((filter) => !config.disabledFilters.includes(filter.id)).length; + }, [config, filters]); + + const saveConfig = async (patch: Partial) => { + if (!config) return; + const nextConfig = { ...config, ...patch }; + setConfig(nextConfig); + setSaving(true); + try { + const res = await fetch("/api/context/rtk/config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }); + if (res.ok) setConfig(await res.json()); + } finally { + setSaving(false); + } + }; + + const toggleFilter = (filterId: string, enabled: boolean) => { + if (!config) return; + const disabledFilters = enabled + ? config.disabledFilters.filter((id) => id !== filterId) + : [...new Set([...config.disabledFilters, filterId])]; + saveConfig({ disabledFilters }); + }; + + const runPreview = async () => { + const res = await fetch("/api/context/rtk/test", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text: sample, config: config ?? undefined }), + }); + setPreview(res.ok ? await res.json() : { error: await res.text() }); + }; + + const rtkStats = analytics?.byEngine?.rtk; + const statCards = [ + [t("tokensFiltered"), formatNumber(rtkStats?.tokensSaved ?? analytics?.totalTokensSaved)], + [t("filtersActive"), formatNumber(activeFilterCount)], + [t("requests"), formatNumber(rtkStats?.count ?? analytics?.totalRequests)], + [t("avgSavings"), `${rtkStats?.avgSavingsPct ?? analytics?.avgSavingsPct ?? 0}%`], + ]; + + return ( +
+
+
+ filter_alt +
+

{t("title")}

+

{t("description")}

+
+
+
+ +
+ {statCards.map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))} +
+ + {config && ( +
+
+ + + + +
+
+ {[ + ["applyToToolResults", t("toolResults")], + ["applyToAssistantMessages", t("assistantMessages")], + ["applyToCodeBlocks", t("codeBlocks")], + ].map(([key, label]) => ( + + ))} +
+
+ )} + +
+
+
+

{t("filterTesting")}

+ +
+