From c78f150ac3944c2d4c731a6260cfdebfd3ba660f Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Sat, 18 Jul 2026 20:12:43 +0200 Subject: [PATCH] feat(compression): support RTK TOML schema v1 filters (#7281) * feat(compression): support RTK TOML filters * chore(ci): sync RTK skill and dependency allowlist * refactor(compression): reduce RTK import complexity * fix(i18n): add Portuguese RTK import translations * fix(compression): improve RTK TOML import validation feedback --- config/quality/dependency-allowlist.json | 1 + docs/compression/COMPRESSION_GUIDE.md | 2 + docs/compression/RTK_COMPRESSION.md | 52 ++- docs/openapi.yaml | 30 ++ open-sse/package.json | 3 +- .../compression/engines/rtk/filterLoader.ts | 171 ++++++--- .../compression/engines/rtk/filterSchema.ts | 6 + .../compression/engines/rtk/lineFilter.ts | 48 +++ .../engines/rtk/tomlCompatibility.ts | 334 ++++++++++++++++++ package-lock.json | 5 +- package.json | 1 + skills/omni-context-rtk/SKILL.md | 11 + .../context/rtk/RtkContextPageClient.tsx | 8 +- .../context/rtk/RtkTomlImportCard.tsx | 255 +++++++++++++ src/app/api/context/rtk/import/route.ts | 81 +++++ src/i18n/messages/de.json | 21 +- src/i18n/messages/en.json | 21 +- src/i18n/messages/pt-BR.json | 21 +- .../compression/rtk-toml-import-route.test.ts | 125 +++++++ .../unit/compression/rtk-line-filter.test.ts | 16 + .../rtk-toml-compatibility.test.ts | 313 ++++++++++++++++ tests/unit/ui/rtkTomlImportCard.test.tsx | 179 ++++++++++ 22 files changed, 1651 insertions(+), 53 deletions(-) create mode 100644 open-sse/services/compression/engines/rtk/tomlCompatibility.ts create mode 100644 src/app/(dashboard)/dashboard/context/rtk/RtkTomlImportCard.tsx create mode 100644 src/app/api/context/rtk/import/route.ts create mode 100644 tests/unit/api/compression/rtk-toml-import-route.test.ts create mode 100644 tests/unit/compression/rtk-toml-compatibility.test.ts create mode 100644 tests/unit/ui/rtkTomlImportCard.test.tsx diff --git a/config/quality/dependency-allowlist.json b/config/quality/dependency-allowlist.json index 5ecfb4329a..6c7e2e3f45 100644 --- a/config/quality/dependency-allowlist.json +++ b/config/quality/dependency-allowlist.json @@ -114,6 +114,7 @@ "safe-regex", "selfsigned", "size-limit", + "smol-toml", "socks", "sql.js", "sqlite-vec", diff --git a/docs/compression/COMPRESSION_GUIDE.md b/docs/compression/COMPRESSION_GUIDE.md index 326a042a1c..314458f287 100644 --- a/docs/compression/COMPRESSION_GUIDE.md +++ b/docs/compression/COMPRESSION_GUIDE.md @@ -93,6 +93,8 @@ RTK mode is optimized for verbose tool outputs that appear in coding-agent sessi TypeScript/Vite/Webpack builds, ESLint/Biome/Prettier, npm audit/installs, Docker logs, infra output, and generic shell output - Applies JSON filter packs from `open-sse/services/compression/engines/rtk/filters/` +- Imports RTK TOML schema v1 filters from project or global `filters.toml` files, with inline-test + validation and trust-gating for project files - Ships 49 built-in filters with inline verify samples - Removes ANSI control sequences, progress bars, repeated lines, and non-actionable noise - Preserves failures, errors, warnings, changed files, summaries, and the tail of long output diff --git a/docs/compression/RTK_COMPRESSION.md b/docs/compression/RTK_COMPRESSION.md index 4457cda3da..6ac7f0d2d7 100644 --- a/docs/compression/RTK_COMPRESSION.md +++ b/docs/compression/RTK_COMPRESSION.md @@ -52,28 +52,59 @@ class is not enough. RTK loads filters in this order: -1. Project filters from `.rtk/filters.json`, only when trusted. -2. Global filters from `DATA_DIR/rtk/filters.json`. +1. Project filters from `.rtk/filters.toml` and `.rtk/filters.json`, only when trusted. +2. Global filters from `DATA_DIR/rtk/filters.toml` and `DATA_DIR/rtk/filters.json`. 3. Built-in filters from `open-sse/services/compression/engines/rtk/filters/`. +Within the same scope, RTK TOML schema v1 filters take precedence over OmniRoute JSON filters. TOML +`match_command` expressions are checked before command-type matching so an imported command-specific +filter can override a broader filter in that scope. Project scope still takes precedence over global +scope, regardless of file format. + Project filters are intentionally trust-gated because regex filters can change how tool output is shown to agents. A project filter file is accepted when one of these is true: - `rtkConfig.trustProjectFilters` is `true`. - `OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=1` is set. -- `.rtk/trust.json` contains the SHA-256 hash of `.rtk/filters.json`. +- `.rtk/trust.json` contains the matching SHA-256 hash for the project filter file. Trust file example: ```json { - "filtersSha256": "0123456789abcdef..." + "filtersSha256": "0123456789abcdef...", + "filtersTomlSha256": "fedcba9876543210..." } ``` +The hashes are separate: `filtersSha256` trusts `.rtk/filters.json`, while `filtersTomlSha256` +trusts `.rtk/filters.toml`. Editing either file invalidates only its own trust entry. Global files +are administrator-installed and use the existing global-filter trust behavior. + Custom filters can be one filter object or an array of filter objects. Invalid custom filters are skipped and reported by `/api/context/rtk/filters` diagnostics. Invalid built-in filters fail fast. +## RTK TOML schema v1 compatibility + +OmniRoute can parse, validate, test, and install declarative filter files using RTK TOML schema v1. +The supported fields are `description`, `match_command`, `strip_ansi`, `filter_stderr`, +`strip_lines_matching`, `keep_lines_matching`, `replace`, `match_output`, `truncate_lines_at`, +`head_lines`, `tail_lines`, `max_lines`, `on_empty`, and `[[tests.]]` inline tests. +Unknown fields, invalid or unsafe regular expressions, simultaneous strip/keep rules, files over +1 MiB, and references to unknown filters are rejected. A file whose inline tests fail can be +validated for inspection but cannot be installed or loaded. Custom-file load failures remain +fail-open: the invalid file is skipped and the remaining filters continue to work. + +OmniRoute receives tool output after the client has already captured it, so `filter_stderr = true` +cannot change process capture. The field is accepted as a no-op and validation returns a warning. +This is intentionally described as **RTK TOML schema v1 compatibility**, not full compatibility +with the RTK executable, shell hooks, Rust command implementations, or its trust-store layout. + +The dashboard's advanced RTK view accepts pasted or uploaded TOML. Validation is read-only. +Installation writes `DATA_DIR/rtk/filters.toml` atomically with restrictive permissions and refreshes +the live filter catalog without a restart. Replacing an existing file requires explicit `overwrite` +confirmation and creates `DATA_DIR/rtk/filters.toml.bak` first. + ## Filter DSL Filters use the JSON schema described in [Compression Rules Format](./COMPRESSION_RULES_FORMAT.md). @@ -216,6 +247,7 @@ round-trips through the same store and survives a restart. | `/api/context/rtk/config` | GET | Read RTK config | | `/api/context/rtk/config` | PUT | Update RTK config | | `/api/context/rtk/filters` | GET | List filter catalog and load diagnostics | +| `/api/context/rtk/import` | POST | Validate or install RTK TOML schema v1 files | | `/api/context/rtk/test` | POST | Preview RTK compression for one text payload | | `/api/context/rtk/raw-output/[id]` | GET | Read retained redacted raw output | | `/api/compression/preview` | POST | Preview any compression mode | @@ -253,6 +285,18 @@ Compression preview payload: Management routes require dashboard management auth or the matching API-key policy. +RTK TOML validation payload: + +```json +{ + "action": "validate", + "content": "schema_version = 1\n\n[filters.my-tool]\nmatch_command = \"^my-tool\\\\b\"\nmax_lines = 20\n" +} +``` + +Use `"action": "install"` to install the validated file globally. Add `"overwrite": true` only +after reviewing and confirming replacement of an existing global file. + ## Raw Output Recovery RTK normally returns only compressed text. For debugging, `rawOutputRetention` can retain redacted diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 994154d41d..6c9241849b 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -2201,6 +2201,36 @@ paths: "200": description: RTK filter catalog and diagnostics + /api/context/rtk/import: + post: + tags: [Compression] + summary: Validate or install an RTK TOML schema v1 filter file + security: + - ManagementSessionAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [action, content] + additionalProperties: false + properties: + action: + type: string + enum: [validate, install] + content: + type: string + maxLength: 1048576 + overwrite: + type: boolean + description: Replace an existing global file and create a backup + responses: + "200": + description: Filter metadata, inline-test outcomes, warnings, and installation status + "400": + description: Invalid TOML, schema, regular expression, inline test, or install request + /api/context/rtk/test: post: tags: [Compression] diff --git a/open-sse/package.json b/open-sse/package.json index c53d7d28fd..6f38dfd26e 100644 --- a/open-sse/package.json +++ b/open-sse/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@toon-format/toon": "^2.3.0", - "safe-regex": "^2.1.1" + "safe-regex": "^2.1.1", + "smol-toml": "1.6.1" } } diff --git a/open-sse/services/compression/engines/rtk/filterLoader.ts b/open-sse/services/compression/engines/rtk/filterLoader.ts index aa10dd8224..184d94417f 100644 --- a/open-sse/services/compression/engines/rtk/filterLoader.ts +++ b/open-sse/services/compression/engines/rtk/filterLoader.ts @@ -4,6 +4,7 @@ import os from "node:os"; import crypto from "node:crypto"; import { detectCommandType } from "./commandDetector.ts"; import { validateRtkFilter, type RtkFilterDefinition } from "./filterSchema.ts"; +import { parseRtkTomlV1, RtkTomlCompatibilityError } from "./tomlCompatibility.ts"; let cache: RtkFilterDefinition[] | null = null; let cacheKey: string | null = null; @@ -29,6 +30,7 @@ function cachedMatchPattern(pattern: string, value: string): boolean { export interface RtkFilterLoadDiagnostic { source: "project" | "global" | "builtin"; + format?: "omniroute-json" | "rtk-toml-v1"; path?: string; level: "warning" | "error"; message: string; @@ -38,6 +40,7 @@ interface FilterSource { source: "project" | "global" | "builtin"; path: string; trusted: boolean; + format: "omniroute-json" | "rtk-toml-v1"; } interface RtkFilterLoadOptions { @@ -95,8 +98,12 @@ function projectFiltersTrusted( try { const filtersHash = sha256(fs.readFileSync(filtersPath, "utf8")); const trust = JSON.parse(fs.readFileSync(trustPath, "utf8")) as Record; - const trustedHash = - typeof trust.filtersSha256 === "string" + const isToml = filtersPath.endsWith(".toml"); + const trustedHash = isToml + ? typeof trust.filtersTomlSha256 === "string" + ? trust.filtersTomlSha256 + : null + : typeof trust.filtersSha256 === "string" ? trust.filtersSha256 : typeof trust.trustedFiltersSha256 === "string" ? trust.trustedFiltersSha256 @@ -110,29 +117,52 @@ function projectFiltersTrusted( function collectFilterSources(options: RtkFilterLoadOptions = {}): FilterSource[] { const sources: FilterSource[] = []; - const projectPath = path.join(process.cwd(), ".rtk", "filters.json"); - if (options.customFiltersEnabled !== false && fs.existsSync(projectPath)) { - const trusted = projectFiltersTrusted(projectPath, options.trustProjectFilters === true); + if (options.customFiltersEnabled !== false) { + collectProjectFilterSources(sources, options); + collectGlobalFilterSources(sources); + } + collectBuiltinFilterSources(sources); + return sources; +} + +function collectProjectFilterSources(sources: FilterSource[], options: RtkFilterLoadOptions): void { + const projectCandidates = [ + { path: path.join(process.cwd(), ".rtk", "filters.toml"), format: "rtk-toml-v1" as const }, + { path: path.join(process.cwd(), ".rtk", "filters.json"), format: "omniroute-json" as const }, + ]; + for (const candidate of projectCandidates) { + if (!fs.existsSync(candidate.path)) continue; + const trusted = projectFiltersTrusted(candidate.path, options.trustProjectFilters === true); if (trusted === true) { - sources.push({ source: "project", path: projectPath, trusted: true }); - } else { - diagnostics.push({ - source: "project", - path: projectPath, - level: "warning", - message: - trusted === "changed" - ? "Project RTK filters changed after trust and were skipped" - : "Project RTK filters are untrusted and were skipped", - }); + sources.push({ source: "project", ...candidate, trusted: true }); + continue; + } + diagnostics.push({ + source: "project", + format: candidate.format, + path: candidate.path, + level: "warning", + message: + trusted === "changed" + ? "Project RTK filters changed after trust and were skipped" + : "Project RTK filters are untrusted and were skipped", + }); + } +} + +function collectGlobalFilterSources(sources: FilterSource[]): void { + const globalCandidates = [ + { path: path.join(getDataDir(), "rtk", "filters.toml"), format: "rtk-toml-v1" as const }, + { path: path.join(getDataDir(), "rtk", "filters.json"), format: "omniroute-json" as const }, + ]; + for (const candidate of globalCandidates) { + if (fs.existsSync(candidate.path)) { + sources.push({ source: "global", ...candidate, trusted: true }); } } +} - const globalPath = path.join(getDataDir(), "rtk", "filters.json"); - if (options.customFiltersEnabled !== false && fs.existsSync(globalPath)) { - sources.push({ source: "global", path: globalPath, trusted: true }); - } - +function collectBuiltinFilterSources(sources: FilterSource[]): void { const builtinDir = getFiltersDir(); if (fs.existsSync(builtinDir)) { let builtinFiles: string[] = []; @@ -152,25 +182,56 @@ function collectFilterSources(options: RtkFilterLoadOptions = {}): FilterSource[ source: "builtin", path: path.join(builtinDir, file), trusted: true, + format: "omniroute-json", }); } } - - return sources; } function parseFilterFile(source: FilterSource): RtkFilterDefinition[] { try { - const parsed = JSON.parse(fs.readFileSync(source.path, "utf8")); - const entries = Array.isArray(parsed) ? parsed : [parsed]; - return entries.map(validateRtkFilter); + const content = fs.readFileSync(source.path, "utf8"); + const definitions = + source.format === "rtk-toml-v1" + ? (() => { + const result = parseRtkTomlV1(content); + if (!result.passed) { + throw new Error("one or more inline tests failed"); + } + for (const warning of result.warnings) { + diagnostics.push({ + source: source.source, + format: source.format, + path: source.path, + level: "warning", + message: warning, + }); + } + return result.filters; + })() + : (() => { + const parsed = JSON.parse(content); + const entries = Array.isArray(parsed) ? parsed : [parsed]; + return entries.map(validateRtkFilter); + })(); + return definitions.map((definition) => ({ + ...definition, + source: source.source, + sourceFormat: definition.sourceFormat ?? source.format, + })); } catch (error) { - const message = error instanceof Error ? error.message : String(error); + const message = + error instanceof RtkTomlCompatibilityError + ? error.publicMessage + : error instanceof Error + ? error.message + : String(error); if (source.source === "builtin") { throw new Error(`Invalid RTK filter ${path.basename(source.path)}: ${message}`); } diagnostics.push({ source: source.source, + format: source.format, path: source.path, level: "warning", message: `Invalid custom RTK filter skipped: ${message}`, @@ -194,7 +255,16 @@ export function loadRtkFilters(options: RtkFilterLoadOptions = {}): RtkFilterDef filters.push(...parseFilterFile(source)); } - const sorted = filters.sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id)); + const sourceRank = { project: 3, global: 2, builtin: 1 } as const; + const formatRank = { "rtk-toml-v1": 2, "omniroute-json": 1 } as const; + const sorted = filters.sort( + (a, b) => + sourceRank[b.source ?? "builtin"] - sourceRank[a.source ?? "builtin"] || + formatRank[b.sourceFormat ?? "omniroute-json"] - + formatRank[a.sourceFormat ?? "omniroute-json"] || + b.priority - a.priority || + a.id.localeCompare(b.id) + ); cache = sorted; cacheKey = currentCacheKey; return sorted; @@ -208,7 +278,14 @@ export function getRtkFilterLoadDiagnostics(): RtkFilterLoadDiagnostic[] { export function getRtkFilterCatalog(): Array< Pick< RtkFilterDefinition, - "id" | "name" | "description" | "commandTypes" | "category" | "priority" + | "id" + | "name" + | "description" + | "commandTypes" + | "category" + | "priority" + | "source" + | "sourceFormat" > > { return loadRtkFilters().map((filter) => ({ @@ -218,6 +295,8 @@ export function getRtkFilterCatalog(): Array< commandTypes: filter.commandTypes, category: filter.category, priority: filter.priority, + source: filter.source, + sourceFormat: filter.sourceFormat, })); } @@ -229,17 +308,25 @@ export function matchRtkFilter( const detection = detectCommandType(text, command); const detectedCommand = detection.command ?? command ?? ""; const filters = loadRtkFilters(options); - return ( - filters.find((filter) => filter.commandTypes.includes(detection.type)) ?? - filters.find( - (filter) => - detectedCommand && - filter.commandPatterns.some((pattern) => cachedMatchPattern(pattern, detectedCommand)) - ) ?? - filters.find((filter) => - filter.matchPatterns.some((pattern) => cachedMatchPattern(pattern, text)) - ) ?? - filters.find((filter) => filter.commandTypes.includes("generic-output")) ?? - null - ); + for (const source of ["project", "global", "builtin"] as const) { + const scoped = filters.filter((filter) => (filter.source ?? "builtin") === source); + const matched = + scoped.find( + (filter) => + filter.sourceFormat === "rtk-toml-v1" && + detectedCommand && + filter.commandPatterns.some((pattern) => cachedMatchPattern(pattern, detectedCommand)) + ) ?? + scoped.find((filter) => filter.commandTypes.includes(detection.type)) ?? + scoped.find( + (filter) => + detectedCommand && + filter.commandPatterns.some((pattern) => cachedMatchPattern(pattern, detectedCommand)) + ) ?? + scoped.find((filter) => + filter.matchPatterns.some((pattern) => cachedMatchPattern(pattern, text)) + ); + if (matched) return matched; + } + return filters.find((filter) => filter.commandTypes.includes("generic-output")) ?? null; } diff --git a/open-sse/services/compression/engines/rtk/filterSchema.ts b/open-sse/services/compression/engines/rtk/filterSchema.ts index 2acceb7cab..9eb9d5b621 100644 --- a/open-sse/services/compression/engines/rtk/filterSchema.ts +++ b/open-sse/services/compression/engines/rtk/filterSchema.ts @@ -137,6 +137,12 @@ export interface RtkFilterDefinition { maxLines: number; preserveHead: number; preserveTail: number; + /** Exact RTK TOML schema-v1 head/tail stages. Undefined for OmniRoute-native JSON filters. */ + rtkTomlHeadLines?: number; + rtkTomlTailLines?: number; + rtkTomlMaxLines?: number; + sourceFormat?: "omniroute-json" | "rtk-toml-v1"; + source?: "project" | "global" | "builtin"; tests: Array<{ name: string; input: string; expected: string; command?: string }>; } diff --git a/open-sse/services/compression/engines/rtk/lineFilter.ts b/open-sse/services/compression/engines/rtk/lineFilter.ts index f1311858ba..52a6a83870 100644 --- a/open-sse/services/compression/engines/rtk/lineFilter.ts +++ b/open-sse/services/compression/engines/rtk/lineFilter.ts @@ -54,6 +54,41 @@ function normalizeStderrPrefix(line: string): string { return line.replace(/^\s*(?:stderr|err)\s*(?:\||:)\s*/i, ""); } +function applyRtkTomlLineLimits( + lines: string[], + filter: RtkFilterDefinition, + appliedRules: string[] +): string[] { + const head = filter.rtkTomlHeadLines; + const tail = filter.rtkTomlTailLines; + const total = lines.length; + + if (head !== undefined && tail !== undefined) { + if (total > head + tail) { + lines = [ + ...lines.slice(0, head), + `... (${total - head - tail} lines omitted)`, + ...(tail > 0 ? lines.slice(-tail) : []), + ]; + appliedRules.push(`${filter.id}:rtk-head-tail`); + } + } else if (head !== undefined && total > head) { + lines = [...lines.slice(0, head), `... (${total - head} lines omitted)`]; + appliedRules.push(`${filter.id}:rtk-head`); + } else if (tail !== undefined && total > tail) { + lines = [`... (${total - tail} lines omitted)`, ...(tail > 0 ? lines.slice(-tail) : [])]; + appliedRules.push(`${filter.id}:rtk-tail`); + } + + const maxLines = filter.rtkTomlMaxLines; + if (maxLines !== undefined && lines.length > maxLines) { + const dropped = lines.length - maxLines; + lines = [...lines.slice(0, maxLines), `... (${dropped} lines truncated)`]; + appliedRules.push(`${filter.id}:rtk-max-lines`); + } + return lines; +} + function truncateUnicodeSafe(line: string, maxChars: number): string { if (maxChars <= 0) return line; const chars = Array.from(line); @@ -70,6 +105,7 @@ export function applyLineFilter(text: string, filter: RtkFilterDefinition): Line const appliedRules: string[] = []; let lines = text.split(/\r?\n/); + if (filter.sourceFormat === "rtk-toml-v1" && lines.at(-1) === "") lines.pop(); const originalLineCount = lines.length; if (filter.stripAnsi) { @@ -159,6 +195,18 @@ export function applyLineFilter(text: string, filter: RtkFilterDefinition): Line } } + if (filter.sourceFormat === "rtk-toml-v1") { + lines = applyRtkTomlLineLimits(lines, filter, appliedRules); + const output = lines.join("\n"); + const finalOutput = output.trim().length === 0 && filter.onEmpty ? filter.onEmpty : output; + return { + text: finalOutput, + strippedLines: Math.max(0, originalLineCount - finalOutput.split(/\r?\n/).length), + keptByRule: keepPatterns.length > 0, + appliedRules, + }; + } + const truncated = smartTruncate(lines.join("\n"), { maxLines: filter.maxLines, preserveHead: filter.preserveHead, diff --git a/open-sse/services/compression/engines/rtk/tomlCompatibility.ts b/open-sse/services/compression/engines/rtk/tomlCompatibility.ts new file mode 100644 index 0000000000..1c45c04549 --- /dev/null +++ b/open-sse/services/compression/engines/rtk/tomlCompatibility.ts @@ -0,0 +1,334 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { parse as parseToml } from "smol-toml"; +import { z } from "zod"; +import { isReDoSProne, type RtkFilterDefinition } from "./filterSchema.ts"; +import { applyLineFilter } from "./lineFilter.ts"; + +const MAX_TOML_BYTES = 1024 * 1024; + +const replaceRuleSchema = z + .object({ + pattern: z.string().min(1), + replacement: z.string(), + }) + .strict(); + +const matchOutputRuleSchema = z + .object({ + pattern: z.string().min(1), + message: z.string(), + unless: z.string().min(1).optional(), + }) + .strict(); + +const filterSchema = z + .object({ + description: z.string().optional(), + match_command: z.string().min(1), + strip_ansi: z.boolean().optional(), + filter_stderr: z.boolean().optional(), + strip_lines_matching: z.array(z.string()).optional(), + keep_lines_matching: z.array(z.string()).optional(), + replace: z.array(replaceRuleSchema).optional(), + match_output: z.array(matchOutputRuleSchema).optional(), + truncate_lines_at: z.number().int().min(0).optional(), + head_lines: z.number().int().min(0).optional(), + tail_lines: z.number().int().min(0).optional(), + max_lines: z.number().int().min(0).optional(), + on_empty: z.string().optional(), + }) + .strict(); + +const inlineTestSchema = z + .object({ + name: z.string().min(1), + input: z.string(), + expected: z.string(), + }) + .strict(); + +const fileSchema = z + .object({ + schema_version: z.literal(1), + filters: z.record(z.string().min(1), filterSchema).default({}), + tests: z.record(z.string().min(1), z.array(inlineTestSchema)).default({}), + }) + .strict(); + +type ParsedFilter = z.infer; + +function arrayOrEmpty(value: T[] | undefined): T[] { + return value ?? []; +} + +function numberOrZero(value: number | undefined): number { + return value ?? 0; +} + +export interface RtkTomlTestOutcome { + filterId: string; + testName: string; + passed: boolean; + actual: string; + expected: string; +} + +export interface RtkTomlCompatibilityResult { + schemaVersion: 1; + sha256: string; + filters: RtkFilterDefinition[]; + outcomes: RtkTomlTestOutcome[]; + filtersWithoutTests: string[]; + warnings: string[]; + passed: boolean; +} + +export class RtkTomlCompatibilityError extends Error { + readonly publicMessage: string; + + constructor(message: string) { + super("RTK TOML schema v1 compatibility error"); + this.name = "RtkTomlCompatibilityError"; + this.publicMessage = message; + } +} + +function compatibilityError(message: string): RtkTomlCompatibilityError { + return new RtkTomlCompatibilityError(message); +} + +function categoryFor(name: string, commandPattern: string): RtkFilterDefinition["category"] { + const value = `${name} ${commandPattern}`.toLowerCase(); + if (/\b(?:git|gh)\b/.test(value)) return "git"; + if (/\b(?:test|jest|vitest|pytest|cargo test|go test|rspec|playwright)\b/.test(value)) { + return "test"; + } + if (/\b(?:build|tsc|eslint|ruff|clippy|gradle|make|next|vite|webpack)\b/.test(value)) { + return "build"; + } + if (/\b(?:docker|kubectl|podman|compose)\b/.test(value)) return "docker"; + if (/\b(?:npm|pnpm|yarn|bun|pip|poetry|uv|bundle|composer)\b/.test(value)) { + return "package"; + } + if (/\b(?:terraform|tofu|ansible|helm|pulumi)\b/.test(value)) return "infra"; + if (/\b(?:aws|gcloud|az|cloudflare)\b/.test(value)) return "cloud"; + if (/\b(?:ls|find|grep|rg|df|du|ps|systemctl|ssh|rsync)\b/.test(value)) return "shell"; + return "generic"; +} + +function regexFields(filter: ParsedFilter): Array<{ field: string; pattern: string }> { + return [ + { field: "match_command", pattern: filter.match_command }, + ...(filter.strip_lines_matching ?? []).map((pattern) => ({ + field: "strip_lines_matching", + pattern, + })), + ...(filter.keep_lines_matching ?? []).map((pattern) => ({ + field: "keep_lines_matching", + pattern, + })), + ...(filter.replace ?? []).map(({ pattern }) => ({ field: "replace.pattern", pattern })), + ...(filter.match_output ?? []).flatMap(({ pattern, unless }) => [ + { field: "match_output.pattern", pattern }, + ...(unless ? [{ field: "match_output.unless", pattern: unless }] : []), + ]), + ]; +} + +function validateRegexes(name: string, filter: ParsedFilter): void { + for (const { field, pattern } of regexFields(filter)) { + if (isReDoSProne(pattern)) { + throw compatibilityError(`filter '${name}' has an unsafe regex in ${field}`); + } + try { + new RegExp(pattern); + } catch { + throw compatibilityError(`filter '${name}' has an invalid regex in ${field}`); + } + } +} + +function toDefinition( + name: string, + filter: ParsedFilter, + tests: z.infer[] +): RtkFilterDefinition { + if ( + (filter.strip_lines_matching?.length ?? 0) > 0 && + (filter.keep_lines_matching?.length ?? 0) > 0 + ) { + throw compatibilityError( + `filter '${name}' cannot combine strip_lines_matching with keep_lines_matching` + ); + } + validateRegexes(name, filter); + return { + id: name, + name, + description: filter.description ?? "", + commandTypes: [], + commandPatterns: [filter.match_command], + matchPatterns: [], + category: categoryFor(name, filter.match_command), + priority: 50, + stripPatterns: arrayOrEmpty(filter.strip_lines_matching), + keepPatterns: arrayOrEmpty(filter.keep_lines_matching), + priorityPatterns: [], + collapsePatterns: [], + stripAnsi: filter.strip_ansi ?? false, + replace: arrayOrEmpty(filter.replace), + matchOutput: arrayOrEmpty(filter.match_output), + truncateLineAt: numberOrZero(filter.truncate_lines_at), + onEmpty: filter.on_empty ?? "", + filterStderr: false, + deduplicate: false, + maxLines: numberOrZero(filter.max_lines), + preserveHead: 0, + preserveTail: 0, + rtkTomlHeadLines: filter.head_lines, + rtkTomlTailLines: filter.tail_lines, + rtkTomlMaxLines: filter.max_lines, + sourceFormat: "rtk-toml-v1", + tests, + }; +} + +function comparable(value: string): string { + return value.replace(/\n+$/g, ""); +} + +function tomlSyntaxLocation(error: unknown): string { + if (typeof error !== "object" || error === null) return ""; + const { line, column } = error as { line?: unknown; column?: unknown }; + if (!Number.isSafeInteger(line) || !Number.isSafeInteger(column)) return ""; + return ` (line ${line}, column ${column})`; +} + +export function parseRtkTomlV1(content: string): RtkTomlCompatibilityResult { + if (Buffer.byteLength(content, "utf8") > MAX_TOML_BYTES) { + throw compatibilityError(`file exceeds the ${MAX_TOML_BYTES}-byte limit`); + } + + let raw: unknown; + try { + raw = parseToml(content); + } catch (error) { + throw compatibilityError(`invalid TOML syntax${tomlSyntaxLocation(error)}`); + } + + const parsed = fileSchema.safeParse(raw); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + const field = issue?.path.length ? issue.path.join(".") : "document"; + throw compatibilityError(`${field}: ${issue?.message ?? "invalid document"}`); + } + if (Object.keys(parsed.data.filters).length === 0) { + throw compatibilityError("document contains no filters"); + } + + for (const testName of Object.keys(parsed.data.tests)) { + if (!(testName in parsed.data.filters)) { + throw compatibilityError(`tests reference unknown filter '${testName}'`); + } + } + + const filters = Object.entries(parsed.data.filters).map(([name, filter]) => + toDefinition(name, filter, parsed.data.tests[name] ?? []) + ); + const outcomes = filters.flatMap((filter) => + filter.tests.map((test) => { + const actual = comparable(applyLineFilter(test.input, filter).text); + const expected = comparable(test.expected); + return { + filterId: filter.id, + testName: test.name, + passed: actual === expected, + actual, + expected, + }; + }) + ); + const filtersWithoutTests = filters + .filter((filter) => filter.tests.length === 0) + .map((filter) => filter.id); + const warnings = filtersWithoutTests.map( + (id) => `Filter '${id}' has no inline tests and should be reviewed before installation` + ); + for (const [id, filter] of Object.entries(parsed.data.filters)) { + if (filter.filter_stderr) { + warnings.push( + `Filter '${id}': filter_stderr is accepted as a no-op because OmniRoute receives already-captured tool output` + ); + } + } + + return { + schemaVersion: 1, + sha256: crypto.createHash("sha256").update(content).digest("hex"), + filters, + outcomes, + filtersWithoutTests, + warnings, + passed: outcomes.every((outcome) => outcome.passed), + }; +} + +function getDataDir(): string { + return process.env.DATA_DIR || path.join(os.homedir(), ".omniroute"); +} + +export function getGlobalRtkTomlPath(): string { + return path.join(getDataDir(), "rtk", "filters.toml"); +} + +export function installGlobalRtkTomlV1( + content: string, + options: { overwrite?: boolean } = {} +): RtkTomlCompatibilityResult & { installedPath: string; backupCreated: boolean } { + const result = parseRtkTomlV1(content); + if (!result.passed) { + throw compatibilityError("one or more inline tests failed"); + } + + const target = getGlobalRtkTomlPath(); + const directory = path.dirname(target); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + try { + fs.chmodSync(directory, 0o700); + } catch { + // Best effort on filesystems that do not support POSIX permissions. + } + if (fs.existsSync(target) && !options.overwrite) { + throw compatibilityError("filters.toml already exists; confirm overwrite to replace it"); + } + + let backupCreated = false; + if (fs.existsSync(target)) { + fs.copyFileSync(target, `${target}.bak`); + try { + fs.chmodSync(`${target}.bak`, 0o600); + } catch { + // Best effort on filesystems that do not support POSIX permissions. + } + backupCreated = true; + } + const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp`; + try { + fs.writeFileSync(temporary, content, { encoding: "utf8", mode: 0o600, flag: "wx" }); + fs.renameSync(temporary, target); + try { + fs.chmodSync(target, 0o600); + } catch { + // Best effort on filesystems that do not support POSIX permissions. + } + } finally { + fs.rmSync(temporary, { force: true }); + } + + return { ...result, installedPath: "rtk/filters.toml", backupCreated }; +} + +export const RTK_TOML_MAX_BYTES = MAX_TOML_BYTES; diff --git a/package-lock.json b/package-lock.json index b3f2179b5f..d5eb7bd8fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -74,6 +74,7 @@ "recharts": "^3.8.1", "safe-regex": "^2.1.1", "selfsigned": "^5.5.0", + "smol-toml": "1.6.1", "socks": "^2.8.7", "sql.js": "^1.14.1", "sqlite-vec": "^0.1.9", @@ -33779,7 +33780,6 @@ "version": "1.6.1", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", - "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">= 18" @@ -37620,7 +37620,8 @@ "version": "3.8.49", "dependencies": { "@toon-format/toon": "^2.3.0", - "safe-regex": "^2.1.1" + "safe-regex": "^2.1.1", + "smol-toml": "1.6.1" } } } diff --git a/package.json b/package.json index 93f77f6e5a..849aafdb5f 100644 --- a/package.json +++ b/package.json @@ -285,6 +285,7 @@ "recharts": "^3.8.1", "safe-regex": "^2.1.1", "selfsigned": "^5.5.0", + "smol-toml": "1.6.1", "socks": "^2.8.7", "sql.js": "^1.14.1", "sqlite-vec": "^0.1.9", diff --git a/skills/omni-context-rtk/SKILL.md b/skills/omni-context-rtk/SKILL.md index 8cb4d8193f..8bad8466e5 100644 --- a/skills/omni-context-rtk/SKILL.md +++ b/skills/omni-context-rtk/SKILL.md @@ -43,6 +43,17 @@ curl https://localhost:20128/api/context/rtk/filters \ -H "Authorization: Bearer $OMNIROUTE_TOKEN" ``` +### POST /api/context/rtk/import + +Validate or install an RTK TOML schema v1 filter file + +```bash +curl -X POST https://localhost:20128/api/context/rtk/import \ + -H "Authorization: Bearer $OMNIROUTE_TOKEN" + -H "Content-Type: application/json" \ + -d '{}' +``` + ### POST /api/context/rtk/test Run RTK compression preview for text diff --git a/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx index 13df110d24..c49f89b70f 100644 --- a/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx +++ b/src/app/(dashboard)/dashboard/context/rtk/RtkContextPageClient.tsx @@ -4,6 +4,7 @@ import { useEffect, useMemo, useState } from "react"; import { useTranslations } from "next-intl"; import { SegmentedControl, Collapsible } from "@/shared/components"; import RtkLearnDiscoverCard from "./RtkLearnDiscoverCard"; +import RtkTomlImportCard from "./RtkTomlImportCard"; type RtkFilter = { id: string; @@ -78,11 +79,14 @@ export default function RtkContextPageClient() { .catch(() => {}); }, []); - useEffect(() => { + const loadFilters = () => fetch("/api/context/rtk/filters") .then((res) => (res.ok ? res.json() : null)) .then((data) => setFilters(Array.isArray(data?.filters) ? data.filters : [])) .catch(() => {}); + + useEffect(() => { + void loadFilters(); fetch("/api/context/rtk/config") .then((res) => (res.ok ? res.json() : null)) .then((data) => setConfig(data)) @@ -363,6 +367,8 @@ export default function RtkContextPageClient() { + {viewMode === "advanced" && } + ); diff --git a/src/app/(dashboard)/dashboard/context/rtk/RtkTomlImportCard.tsx b/src/app/(dashboard)/dashboard/context/rtk/RtkTomlImportCard.tsx new file mode 100644 index 0000000000..f6a3718bbf --- /dev/null +++ b/src/app/(dashboard)/dashboard/context/rtk/RtkTomlImportCard.tsx @@ -0,0 +1,255 @@ +"use client"; + +import { useState } from "react"; +import { useTranslations } from "next-intl"; + +const RTK_TOML_MAX_BYTES = 1024 * 1024; + +interface ImportFilterSummary { + id: string; + description: string; + category: string; + commandPatterns: string[]; + testCount: number; +} + +interface ImportTestOutcome { + filterId: string; + testName: string; + passed: boolean; +} + +interface ImportResult { + sha256: string; + passed: boolean; + filters: ImportFilterSummary[]; + outcomes: ImportTestOutcome[]; + warnings: string[]; + installedPath?: string; + backupCreated?: boolean; +} + +interface RtkTomlImportCardProps { + onInstalled?: () => void | Promise; +} + +interface RtkTomlEditorProps { + content: string; + processing: "validate" | "install" | null; + overwrite: boolean; + onContentChange: (content: string) => void; + onFileChange: (file: File | undefined) => void; + onProcess: (action: "validate" | "install") => void; + onOverwriteChange: (overwrite: boolean) => void; +} + +async function readErrorMessage(response: Response): Promise { + try { + const body = (await response.json()) as { error?: { message?: unknown } }; + return typeof body.error?.message === "string" ? body.error.message : null; + } catch { + return null; + } +} + +function RtkTomlEditor({ + content, + processing, + overwrite, + onContentChange, + onFileChange, + onProcess, + onOverwriteChange, +}: RtkTomlEditorProps) { + const t = useTranslations("contextRtk"); + return ( + <> +
+ +