mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-06 15:22:12 +03:00
feat(compression): T05/C2 — caveman dedup + ultra packs for de, fr, ja (#5529)
Integrated into release/v3.8.42 (round 3). T05/C2 caveman packs de/fr/ja
This commit is contained in:
Diego Rodrigues de Sa e Souza
committed by
GitHub
GitHub
parent
47de0a12ef
commit
1c882ed7f9
@@ -19,6 +19,10 @@
|
||||
- **db (import):** fix `EBUSY: resource busy or locked` when importing a database on **Windows**. The import route deleted the live `storage.sqlite` + WAL/`-shm`/`-journal` sidecars with a plain `fs.unlinkSync` immediately after `resetDbInstance()`, but Windows releases the SQLite file handle asynchronously after `close()` (mmap / antivirus), so the unlink raced and threw `EBUSY`. The route now deletes via `unlinkFileWithRetry` (EBUSY/EPERM backoff) — the same helper the restore path already uses. Regression guard in `tests/unit/db-import-ebusy-5406.test.ts`. ([#5406](https://github.com/diegosouzapw/OmniRoute/issues/5406), consolidated under [#5161](https://github.com/diegosouzapw/OmniRoute/issues/5161))
|
||||
- **compression (pipeline):** add an honest default-on **inflation guard** to the stacked compression pipeline (T02 / Headroom H1). If the fully-stacked engines produce a body that did not actually shrink — its token count is `>=` the original — the compressed body is discarded and the verbatim original request is sent upstream instead, with a `pipeline-inflation-guard` warning recorded in the compression stats. This is safe by construction (the only fallback is the unmodified original, always a valid payload) and complements the existing opt-in per-step TV1 bail-out, which governs step-to-step advancement rather than the final output. New `open-sse/services/compression/pipelineGuards.ts`; wired at the single `finalizeStackedResult` choke point shared by the sync and async stacked paths. Regression guards (incl. an inflating-engine integration test) in `tests/unit/compression-pipeline-inflation-guard.test.ts`.
|
||||
|
||||
### ✨ New Features
|
||||
|
||||
- **compression (caveman):** complete the German, French, and Japanese rule packs with the `dedup` (repeated-context collapsing) and `ultra` (abbreviation / terse) categories they were missing — these three languages previously shipped only `context`/`filler`/`structural`, while `en`/`es`/`id`/`pt-BR` had all five. So a de/fr/ja conversation compressed at higher intensities now collapses repeated boilerplate ("wie bereits besprochen" → "Siehe oben.", "comme mentionné précédemment" → "Voir ci-dessus.", "前述のとおり" → "（上記参照）") and abbreviates dense technical vocabulary (`Datenbank`→`DB`, `Authentifizierung`→`Auth`; `base de données`→`BD`, `authentification`→`auth`; `データベース`→`DB`, `アプリケーション`→`app`). Patterns mirror the existing `es` pack and stay ReDoS-safe (bounded literal alternations; the CJK pack uses no `\b` since Japanese has no word boundaries). Regression guard: `tests/unit/caveman-packs-de-fr-ja.test.ts` (packs load + validate + shrink a representative sample). gaps v3.8.42 — T05/C2.
|
||||
|
||||
---
|
||||
|
||||
## [3.8.41] — 2026-06-29
|
||||
|
||||
38
open-sse/services/compression/rules/de/dedup.json
Normal file
38
open-sse/services/compression/rules/de/dedup.json
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"language": "de",
|
||||
"category": "dedup",
|
||||
"rules": [
|
||||
{
|
||||
"name": "de_repeated_context",
|
||||
"pattern": "\\b(?:wie bereits besprochen|wie zuvor erwähnt|wie bereits erwähnt|wie ich bereits sagte)\\b[,.]?\\s*",
|
||||
"replacement": "Siehe oben. ",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "de_repeated_question",
|
||||
"pattern": "\\b(?:dieselbe Frage wie zuvor|das habe ich bereits gefragt|das ist dieselbe Frage)\\b[,.]?\\s*",
|
||||
"replacement": "[gleiche Frage] ",
|
||||
"context": "user",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "de_reestablished_context",
|
||||
"pattern": "\\b(?:zurück zum obigen Code|bezugnehmend auf|zurückkommend auf)\\b\\s*",
|
||||
"replacement": "Re: ",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "de_summary_replacement",
|
||||
"pattern": "\\b(?:um zusammenzufassen was wir besprochen haben|zusammenfassend unser Gespräch|um es zusammenzufassen)\\b[,.]?\\s*",
|
||||
"replacement": "Zusammenfassung: ",
|
||||
"context": "assistant",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
}
|
||||
]
|
||||
}
|
||||
107
open-sse/services/compression/rules/de/ultra.json
Normal file
107
open-sse/services/compression/rules/de/ultra.json
Normal file
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"language": "de",
|
||||
"category": "ultra",
|
||||
"rules": [
|
||||
{
|
||||
"name": "de_articles",
|
||||
"pattern": "\\b(?:[Dd]er|[Dd]ie|[Dd]as|[Dd]en|[Dd]em|[Dd]es|[Ee]in|[Ee]ine|[Ee]inen|[Ee]inem|[Ee]iner|[Ee]ines)\\s+(?=[a-zäöüß])",
|
||||
"flags": "g",
|
||||
"replacement": "",
|
||||
"context": "all",
|
||||
"category": "terse",
|
||||
"minIntensity": "full"
|
||||
},
|
||||
{
|
||||
"name": "de_leader_phrases",
|
||||
"pattern": "^(?:ich werde|ich kann|ich würde|lass mich|wir werden|wir können|lass uns)\\s+(?=[a-zäöüß])",
|
||||
"replacement": "",
|
||||
"context": "all",
|
||||
"category": "terse",
|
||||
"minIntensity": "full"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_database_abbreviation",
|
||||
"pattern": "\\bDatenbank\\b",
|
||||
"replacement": "DB",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_config_abbreviation",
|
||||
"pattern": "\\bKonfiguration\\b",
|
||||
"replacement": "Konfig",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_function_abbreviation",
|
||||
"pattern": "\\bFunktion\\b",
|
||||
"replacement": "Fn",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_request_abbreviation",
|
||||
"pattern": "\\b(?:Anfrage|Anforderung)\\b",
|
||||
"replacement": "Req",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_response_abbreviation",
|
||||
"pattern": "\\bAntwort\\b",
|
||||
"replacement": "Res",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_implementation_abbreviation",
|
||||
"pattern": "\\bImplementierung\\b",
|
||||
"replacement": "Impl",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_authentication_abbreviation",
|
||||
"pattern": "\\bAuthentifizierung\\b",
|
||||
"replacement": "Auth",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_authorization_abbreviation",
|
||||
"pattern": "\\bAutorisierung\\b",
|
||||
"replacement": "Authz",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_application_abbreviation",
|
||||
"pattern": "\\b(?:Anwendung|Applikation)\\b",
|
||||
"replacement": "App",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "de_ultra_dependency_abbreviation",
|
||||
"pattern": "\\b(?:Abhängigkeit|Abhängigkeiten)\\b",
|
||||
"replacement": "Dep",
|
||||
"replacementMap": {
|
||||
"Abhängigkeit": "Dep",
|
||||
"Abhängigkeiten": "Deps"
|
||||
},
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
}
|
||||
]
|
||||
}
|
||||
38
open-sse/services/compression/rules/fr/dedup.json
Normal file
38
open-sse/services/compression/rules/fr/dedup.json
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"language": "fr",
|
||||
"category": "dedup",
|
||||
"rules": [
|
||||
{
|
||||
"name": "fr_repeated_context",
|
||||
"pattern": "\\b(?:comme discuté précédemment|comme mentionné précédemment|comme indiqué précédemment|comme je l'ai dit avant)\\b[,.]?\\s*",
|
||||
"replacement": "Voir ci-dessus. ",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "fr_repeated_question",
|
||||
"pattern": "\\b(?:même question qu'avant|j'ai déjà posé cette question|c'est la même question)\\b[,.]?\\s*",
|
||||
"replacement": "[même question] ",
|
||||
"context": "user",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "fr_reestablished_context",
|
||||
"pattern": "\\b(?:revenons au code ci-dessus|en référence à|revenant à)\\b\\s*",
|
||||
"replacement": "Re: ",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "fr_summary_replacement",
|
||||
"pattern": "\\b(?:pour résumer ce que nous avons discuté|en résumé de notre conversation|pour récapituler)\\b[,.]?\\s*",
|
||||
"replacement": "Résumé: ",
|
||||
"context": "assistant",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
}
|
||||
]
|
||||
}
|
||||
107
open-sse/services/compression/rules/fr/ultra.json
Normal file
107
open-sse/services/compression/rules/fr/ultra.json
Normal file
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"language": "fr",
|
||||
"category": "ultra",
|
||||
"rules": [
|
||||
{
|
||||
"name": "fr_articles",
|
||||
"pattern": "\\b(?:[Ll]e|[Ll]a|[Ll]es|[Uu]n|[Uu]ne|[Dd]es|[Dd]u)\\s+(?=[a-zàâçéèêëîïôûùüÿœ])",
|
||||
"flags": "g",
|
||||
"replacement": "",
|
||||
"context": "all",
|
||||
"category": "terse",
|
||||
"minIntensity": "full"
|
||||
},
|
||||
{
|
||||
"name": "fr_leader_phrases",
|
||||
"pattern": "^(?:je vais|je peux|nous allons|nous pouvons|laisse-moi|on peut)\\s+(?=[a-zàâçéèêëîïôûùüÿœ])",
|
||||
"replacement": "",
|
||||
"context": "all",
|
||||
"category": "terse",
|
||||
"minIntensity": "full"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_database_abbreviation",
|
||||
"pattern": "\\bbase de données\\b",
|
||||
"replacement": "BD",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_config_abbreviation",
|
||||
"pattern": "\\bconfiguration\\b",
|
||||
"replacement": "config",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_function_abbreviation",
|
||||
"pattern": "\\bfonction\\b",
|
||||
"replacement": "fn",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_request_abbreviation",
|
||||
"pattern": "\\b(?:requête|demande)\\b",
|
||||
"replacement": "req",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_response_abbreviation",
|
||||
"pattern": "\\bréponse\\b",
|
||||
"replacement": "rép",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_implementation_abbreviation",
|
||||
"pattern": "\\bimplémentation\\b",
|
||||
"replacement": "impl",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_authentication_abbreviation",
|
||||
"pattern": "\\bauthentification\\b",
|
||||
"replacement": "auth",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_authorization_abbreviation",
|
||||
"pattern": "\\bautorisation\\b",
|
||||
"replacement": "authz",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_application_abbreviation",
|
||||
"pattern": "\\bapplication\\b",
|
||||
"replacement": "app",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "fr_ultra_dependency_abbreviation",
|
||||
"pattern": "\\b(?:dépendance|dépendances)\\b",
|
||||
"replacement": "dép",
|
||||
"replacementMap": {
|
||||
"dépendance": "dép",
|
||||
"dépendances": "déps"
|
||||
},
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
}
|
||||
]
|
||||
}
|
||||
38
open-sse/services/compression/rules/ja/dedup.json
Normal file
38
open-sse/services/compression/rules/ja/dedup.json
Normal file
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"language": "ja",
|
||||
"category": "dedup",
|
||||
"rules": [
|
||||
{
|
||||
"name": "ja_repeated_context",
|
||||
"pattern": "(?:前述のとおり|前に述べたように|以前述べたように|先ほど言ったように)",
|
||||
"replacement": "（上記参照）",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "ja_repeated_question",
|
||||
"pattern": "(?:前と同じ質問ですが|これは前と同じ質問です|先ほどと同じ質問ですが)",
|
||||
"replacement": "［同じ質問］",
|
||||
"context": "user",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "ja_reestablished_context",
|
||||
"pattern": "(?:上記のコードに戻ると|先ほどの話に戻ると)",
|
||||
"replacement": "Re: ",
|
||||
"context": "all",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
},
|
||||
{
|
||||
"name": "ja_summary_replacement",
|
||||
"pattern": "(?:これまでの議論を要約すると|会話を要約すると|まとめると)",
|
||||
"replacement": "要約：",
|
||||
"context": "assistant",
|
||||
"category": "dedup",
|
||||
"minIntensity": "lite"
|
||||
}
|
||||
]
|
||||
}
|
||||
86
open-sse/services/compression/rules/ja/ultra.json
Normal file
86
open-sse/services/compression/rules/ja/ultra.json
Normal file
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"language": "ja",
|
||||
"category": "ultra",
|
||||
"rules": [
|
||||
{
|
||||
"name": "ja_ultra_database_abbreviation",
|
||||
"pattern": "データベース",
|
||||
"replacement": "DB",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_config_abbreviation",
|
||||
"pattern": "(?:コンフィギュレーション|設定)",
|
||||
"replacement": "config",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_function_abbreviation",
|
||||
"pattern": "関数",
|
||||
"replacement": "fn",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_request_abbreviation",
|
||||
"pattern": "リクエスト",
|
||||
"replacement": "req",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_response_abbreviation",
|
||||
"pattern": "レスポンス",
|
||||
"replacement": "res",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_implementation_abbreviation",
|
||||
"pattern": "実装",
|
||||
"replacement": "impl",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_authentication_abbreviation",
|
||||
"pattern": "認証",
|
||||
"replacement": "auth",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_authorization_abbreviation",
|
||||
"pattern": "認可",
|
||||
"replacement": "authz",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_application_abbreviation",
|
||||
"pattern": "アプリケーション",
|
||||
"replacement": "app",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
},
|
||||
{
|
||||
"name": "ja_ultra_dependency_abbreviation",
|
||||
"pattern": "依存関係",
|
||||
"replacement": "dep",
|
||||
"context": "all",
|
||||
"category": "ultra",
|
||||
"minIntensity": "ultra"
|
||||
}
|
||||
]
|
||||
}
|
||||
57
tests/unit/caveman-packs-de-fr-ja.test.ts
Normal file
57
tests/unit/caveman-packs-de-fr-ja.test.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
loadRulePack,
|
||||
loadAllRulesForLanguage,
|
||||
} from "../../open-sse/services/compression/ruleLoader.ts";
|
||||
|
||||
// T05 / C2 — caveman dedup + ultra packs for de, fr, ja (previously only context/filler/structural).
|
||||
const LANGS = ["de", "fr", "ja"] as const;
|
||||
|
||||
// Samples chosen to contain terms the ultra pack abbreviates. For ja the katakana loanwords
|
||||
// (データベース/アプリケーション/…) are the clear character-length wins; the short-kanji rules
|
||||
// (設定→config, 認証→auth) are token wins but can be char-neutral, so they stay out of this sample.
|
||||
const ULTRA_SAMPLES: Record<(typeof LANGS)[number], string> = {
|
||||
de: "Die Datenbank-Konfiguration der Anwendung benötigt eine Authentifizierung.",
|
||||
fr: "La configuration de la base de données de l'application nécessite une authentification.",
|
||||
ja: "データベースとアプリケーション、レスポンスとリクエストとコンフィギュレーション",
|
||||
};
|
||||
|
||||
type LoadedRule = ReturnType<typeof loadRulePack>[number];
|
||||
|
||||
function applyAll(text: string, rules: LoadedRule[]): string {
|
||||
return rules.reduce((acc, rule) => {
|
||||
return typeof rule.replacement === "function"
|
||||
? acc.replace(rule.pattern, rule.replacement)
|
||||
: acc.replace(rule.pattern, rule.replacement);
|
||||
}, text);
|
||||
}
|
||||
|
||||
for (const lang of LANGS) {
|
||||
test(`caveman ${lang}: dedup + ultra packs load and validate`, () => {
|
||||
// loadRulePack throws if the pack fails validateRulePack, so a non-throwing non-empty load
|
||||
// proves both presence and schema validity.
|
||||
const dedup = loadRulePack(lang, "dedup", { refresh: true });
|
||||
const ultra = loadRulePack(lang, "ultra", { refresh: true });
|
||||
assert.ok(dedup.length > 0, `${lang}/dedup should have rules`);
|
||||
assert.ok(ultra.length > 0, `${lang}/ultra should have rules`);
|
||||
});
|
||||
|
||||
test(`caveman ${lang}: ultra pack shrinks a representative sample`, () => {
|
||||
const ultra = loadRulePack(lang, "ultra", { refresh: true });
|
||||
const sample = ULTRA_SAMPLES[lang];
|
||||
const out = applyAll(sample, ultra);
|
||||
assert.ok(
|
||||
out.length < sample.length,
|
||||
`${lang} ultra should shrink the sample (${sample.length} -> ${out.length})`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test("de/fr/ja expose dedup + ultra categories via loadAllRulesForLanguage", () => {
|
||||
for (const lang of LANGS) {
|
||||
const cats = new Set(loadAllRulesForLanguage(lang, { refresh: true }).map((r) => r.category));
|
||||
assert.ok(cats.has("dedup"), `${lang} should expose the dedup category`);
|
||||
assert.ok(cats.has("ultra"), `${lang} should expose the ultra category`);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user