diff --git a/CHANGELOG.md b/CHANGELOG.md index 3837e60421..cc150af447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ - **db (import):** fix `EBUSY: resource busy or locked` when importing a database on **Windows**. The import route deleted the live `storage.sqlite` + WAL/`-shm`/`-journal` sidecars with a plain `fs.unlinkSync` immediately after `resetDbInstance()`, but Windows releases the SQLite file handle asynchronously after `close()` (mmap / antivirus), so the unlink raced and threw `EBUSY`. The route now deletes via `unlinkFileWithRetry` (EBUSY/EPERM backoff) — the same helper the restore path already uses. Regression guard in `tests/unit/db-import-ebusy-5406.test.ts`. ([#5406](https://github.com/diegosouzapw/OmniRoute/issues/5406), consolidated under [#5161](https://github.com/diegosouzapw/OmniRoute/issues/5161)) - **compression (pipeline):** add an honest default-on **inflation guard** to the stacked compression pipeline (T02 / Headroom H1). If the fully-stacked engines produce a body that did not actually shrink — its token count is `>=` the original — the compressed body is discarded and the verbatim original request is sent upstream instead, with a `pipeline-inflation-guard` warning recorded in the compression stats. This is safe by construction (the only fallback is the unmodified original, always a valid payload) and complements the existing opt-in per-step TV1 bail-out, which governs step-to-step advancement rather than the final output. New `open-sse/services/compression/pipelineGuards.ts`; wired at the single `finalizeStackedResult` choke point shared by the sync and async stacked paths. Regression guards (incl. an inflating-engine integration test) in `tests/unit/compression-pipeline-inflation-guard.test.ts`. +### ✨ New Features + +- **compression (caveman):** complete the German, French, and Japanese rule packs with the `dedup` (repeated-context collapsing) and `ultra` (abbreviation / terse) categories they were missing — these three languages previously shipped only `context`/`filler`/`structural`, while `en`/`es`/`id`/`pt-BR` had all five. So a de/fr/ja conversation compressed at higher intensities now collapses repeated boilerplate ("wie bereits besprochen" → "Siehe oben.", "comme mentionné précédemment" → "Voir ci-dessus.", "前述のとおり" → "(上記参照)") and abbreviates dense technical vocabulary (`Datenbank`→`DB`, `Authentifizierung`→`Auth`; `base de données`→`BD`, `authentification`→`auth`; `データベース`→`DB`, `アプリケーション`→`app`). Patterns mirror the existing `es` pack and stay ReDoS-safe (bounded literal alternations; the CJK pack uses no `\b` since Japanese has no word boundaries). Regression guard: `tests/unit/caveman-packs-de-fr-ja.test.ts` (packs load + validate + shrink a representative sample). gaps v3.8.42 — T05/C2. + --- ## [3.8.41] — 2026-06-29 diff --git a/open-sse/services/compression/rules/de/dedup.json b/open-sse/services/compression/rules/de/dedup.json new file mode 100644 index 0000000000..057c376e73 --- /dev/null +++ b/open-sse/services/compression/rules/de/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "de", + "category": "dedup", + "rules": [ + { + "name": "de_repeated_context", + "pattern": "\\b(?:wie bereits besprochen|wie zuvor erwähnt|wie bereits erwähnt|wie ich bereits sagte)\\b[,.]?\\s*", + "replacement": "Siehe oben. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "de_repeated_question", + "pattern": "\\b(?:dieselbe Frage wie zuvor|das habe ich bereits gefragt|das ist dieselbe Frage)\\b[,.]?\\s*", + "replacement": "[gleiche Frage] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "de_reestablished_context", + "pattern": "\\b(?:zurück zum obigen Code|bezugnehmend auf|zurückkommend auf)\\b\\s*", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "de_summary_replacement", + "pattern": "\\b(?:um zusammenzufassen was wir besprochen haben|zusammenfassend unser Gespräch|um es zusammenzufassen)\\b[,.]?\\s*", + "replacement": "Zusammenfassung: ", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/de/ultra.json b/open-sse/services/compression/rules/de/ultra.json new file mode 100644 index 0000000000..4b5a140bc9 --- /dev/null +++ b/open-sse/services/compression/rules/de/ultra.json @@ -0,0 +1,107 @@ +{ + "language": "de", + "category": "ultra", + "rules": [ + { + "name": "de_articles", + "pattern": "\\b(?:[Dd]er|[Dd]ie|[Dd]as|[Dd]en|[Dd]em|[Dd]es|[Ee]in|[Ee]ine|[Ee]inen|[Ee]inem|[Ee]iner|[Ee]ines)\\s+(?=[a-zäöüß])", + "flags": "g", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "de_leader_phrases", + "pattern": "^(?:ich werde|ich kann|ich würde|lass mich|wir werden|wir können|lass uns)\\s+(?=[a-zäöüß])", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "de_ultra_database_abbreviation", + "pattern": "\\bDatenbank\\b", + "replacement": "DB", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_config_abbreviation", + "pattern": "\\bKonfiguration\\b", + "replacement": "Konfig", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_function_abbreviation", + "pattern": "\\bFunktion\\b", + "replacement": "Fn", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_request_abbreviation", + "pattern": "\\b(?:Anfrage|Anforderung)\\b", + "replacement": "Req", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_response_abbreviation", + "pattern": "\\bAntwort\\b", + "replacement": "Res", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_implementation_abbreviation", + "pattern": "\\bImplementierung\\b", + "replacement": "Impl", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_authentication_abbreviation", + "pattern": "\\bAuthentifizierung\\b", + "replacement": "Auth", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_authorization_abbreviation", + "pattern": "\\bAutorisierung\\b", + "replacement": "Authz", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_application_abbreviation", + "pattern": "\\b(?:Anwendung|Applikation)\\b", + "replacement": "App", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "de_ultra_dependency_abbreviation", + "pattern": "\\b(?:Abhängigkeit|Abhängigkeiten)\\b", + "replacement": "Dep", + "replacementMap": { + "Abhängigkeit": "Dep", + "Abhängigkeiten": "Deps" + }, + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + } + ] +} diff --git a/open-sse/services/compression/rules/fr/dedup.json b/open-sse/services/compression/rules/fr/dedup.json new file mode 100644 index 0000000000..75222cfa01 --- /dev/null +++ b/open-sse/services/compression/rules/fr/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "fr", + "category": "dedup", + "rules": [ + { + "name": "fr_repeated_context", + "pattern": "\\b(?:comme discuté précédemment|comme mentionné précédemment|comme indiqué précédemment|comme je l'ai dit avant)\\b[,.]?\\s*", + "replacement": "Voir ci-dessus. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "fr_repeated_question", + "pattern": "\\b(?:même question qu'avant|j'ai déjà posé cette question|c'est la même question)\\b[,.]?\\s*", + "replacement": "[même question] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "fr_reestablished_context", + "pattern": "\\b(?:revenons au code ci-dessus|en référence à|revenant à)\\b\\s*", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "fr_summary_replacement", + "pattern": "\\b(?:pour résumer ce que nous avons discuté|en résumé de notre conversation|pour récapituler)\\b[,.]?\\s*", + "replacement": "Résumé: ", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/fr/ultra.json b/open-sse/services/compression/rules/fr/ultra.json new file mode 100644 index 0000000000..3c1f88562c --- /dev/null +++ b/open-sse/services/compression/rules/fr/ultra.json @@ -0,0 +1,107 @@ +{ + "language": "fr", + "category": "ultra", + "rules": [ + { + "name": "fr_articles", + "pattern": "\\b(?:[Ll]e|[Ll]a|[Ll]es|[Uu]n|[Uu]ne|[Dd]es|[Dd]u)\\s+(?=[a-zàâçéèêëîïôûùüÿœ])", + "flags": "g", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "fr_leader_phrases", + "pattern": "^(?:je vais|je peux|nous allons|nous pouvons|laisse-moi|on peut)\\s+(?=[a-zàâçéèêëîïôûùüÿœ])", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "fr_ultra_database_abbreviation", + "pattern": "\\bbase de données\\b", + "replacement": "BD", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_config_abbreviation", + "pattern": "\\bconfiguration\\b", + "replacement": "config", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_function_abbreviation", + "pattern": "\\bfonction\\b", + "replacement": "fn", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_request_abbreviation", + "pattern": "\\b(?:requête|demande)\\b", + "replacement": "req", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_response_abbreviation", + "pattern": "\\bréponse\\b", + "replacement": "rép", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_implementation_abbreviation", + "pattern": "\\bimplémentation\\b", + "replacement": "impl", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_authentication_abbreviation", + "pattern": "\\bauthentification\\b", + "replacement": "auth", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_authorization_abbreviation", + "pattern": "\\bautorisation\\b", + "replacement": "authz", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_application_abbreviation", + "pattern": "\\bapplication\\b", + "replacement": "app", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "fr_ultra_dependency_abbreviation", + "pattern": "\\b(?:dépendance|dépendances)\\b", + "replacement": "dép", + "replacementMap": { + "dépendance": "dép", + "dépendances": "déps" + }, + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + } + ] +} diff --git a/open-sse/services/compression/rules/ja/dedup.json b/open-sse/services/compression/rules/ja/dedup.json new file mode 100644 index 0000000000..e9674c2e44 --- /dev/null +++ b/open-sse/services/compression/rules/ja/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "ja", + "category": "dedup", + "rules": [ + { + "name": "ja_repeated_context", + "pattern": "(?:前述のとおり|前に述べたように|以前述べたように|先ほど言ったように)", + "replacement": "(上記参照)", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "ja_repeated_question", + "pattern": "(?:前と同じ質問ですが|これは前と同じ質問です|先ほどと同じ質問ですが)", + "replacement": "[同じ質問]", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "ja_reestablished_context", + "pattern": "(?:上記のコードに戻ると|先ほどの話に戻ると)", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "ja_summary_replacement", + "pattern": "(?:これまでの議論を要約すると|会話を要約すると|まとめると)", + "replacement": "要約:", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/ja/ultra.json b/open-sse/services/compression/rules/ja/ultra.json new file mode 100644 index 0000000000..1deea66448 --- /dev/null +++ b/open-sse/services/compression/rules/ja/ultra.json @@ -0,0 +1,86 @@ +{ + "language": "ja", + "category": "ultra", + "rules": [ + { + "name": "ja_ultra_database_abbreviation", + "pattern": "データベース", + "replacement": "DB", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_config_abbreviation", + "pattern": "(?:コンフィギュレーション|設定)", + "replacement": "config", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_function_abbreviation", + "pattern": "関数", + "replacement": "fn", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_request_abbreviation", + "pattern": "リクエスト", + "replacement": "req", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_response_abbreviation", + "pattern": "レスポンス", + "replacement": "res", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_implementation_abbreviation", + "pattern": "実装", + "replacement": "impl", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_authentication_abbreviation", + "pattern": "認証", + "replacement": "auth", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_authorization_abbreviation", + "pattern": "認可", + "replacement": "authz", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_application_abbreviation", + "pattern": "アプリケーション", + "replacement": "app", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ja_ultra_dependency_abbreviation", + "pattern": "依存関係", + "replacement": "dep", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + } + ] +} diff --git a/tests/unit/caveman-packs-de-fr-ja.test.ts b/tests/unit/caveman-packs-de-fr-ja.test.ts new file mode 100644 index 0000000000..d0c08badb8 --- /dev/null +++ b/tests/unit/caveman-packs-de-fr-ja.test.ts @@ -0,0 +1,57 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + loadRulePack, + loadAllRulesForLanguage, +} from "../../open-sse/services/compression/ruleLoader.ts"; + +// T05 / C2 — caveman dedup + ultra packs for de, fr, ja (previously only context/filler/structural). +const LANGS = ["de", "fr", "ja"] as const; + +// Samples chosen to contain terms the ultra pack abbreviates. For ja the katakana loanwords +// (データベース/アプリケーション/…) are the clear character-length wins; the short-kanji rules +// (設定→config, 認証→auth) are token wins but can be char-neutral, so they stay out of this sample. +const ULTRA_SAMPLES: Record<(typeof LANGS)[number], string> = { + de: "Die Datenbank-Konfiguration der Anwendung benötigt eine Authentifizierung.", + fr: "La configuration de la base de données de l'application nécessite une authentification.", + ja: "データベースとアプリケーション、レスポンスとリクエストとコンフィギュレーション", +}; + +type LoadedRule = ReturnType[number]; + +function applyAll(text: string, rules: LoadedRule[]): string { + return rules.reduce((acc, rule) => { + return typeof rule.replacement === "function" + ? acc.replace(rule.pattern, rule.replacement) + : acc.replace(rule.pattern, rule.replacement); + }, text); +} + +for (const lang of LANGS) { + test(`caveman ${lang}: dedup + ultra packs load and validate`, () => { + // loadRulePack throws if the pack fails validateRulePack, so a non-throwing non-empty load + // proves both presence and schema validity. + const dedup = loadRulePack(lang, "dedup", { refresh: true }); + const ultra = loadRulePack(lang, "ultra", { refresh: true }); + assert.ok(dedup.length > 0, `${lang}/dedup should have rules`); + assert.ok(ultra.length > 0, `${lang}/ultra should have rules`); + }); + + test(`caveman ${lang}: ultra pack shrinks a representative sample`, () => { + const ultra = loadRulePack(lang, "ultra", { refresh: true }); + const sample = ULTRA_SAMPLES[lang]; + const out = applyAll(sample, ultra); + assert.ok( + out.length < sample.length, + `${lang} ultra should shrink the sample (${sample.length} -> ${out.length})` + ); + }); +} + +test("de/fr/ja expose dedup + ultra categories via loadAllRulesForLanguage", () => { + for (const lang of LANGS) { + const cats = new Set(loadAllRulesForLanguage(lang, { refresh: true }).map((r) => r.category)); + assert.ok(cats.has("dedup"), `${lang} should expose the dedup category`); + assert.ok(cats.has("ultra"), `${lang} should expose the ultra category`); + } +});