From 89468608854fdc5b741fc4cfd79009d5ecb093ea Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 30 Jun 2026 02:08:08 -0300 Subject: [PATCH] =?UTF-8?q?feat(compression):=20T05/C6=20=E2=80=94=20Chine?= =?UTF-8?q?se=20(zh=20/=20wenyan)=20caveman=20pack=20+=20detection=20(#553?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.42 (round 3). T05/C6 zh/wenyan pack + detection --- CHANGELOG.md | 4 ++ .../services/compression/languageDetector.ts | 12 +++- .../services/compression/rules/zh/dedup.json | 38 ++++++++++ .../services/compression/rules/zh/filler.json | 54 +++++++++++++++ .../services/compression/rules/zh/ultra.json | 55 +++++++++++++++ tests/unit/caveman-packs-zh-wenyan.test.ts | 69 +++++++++++++++++++ 6 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 open-sse/services/compression/rules/zh/dedup.json create mode 100644 open-sse/services/compression/rules/zh/filler.json create mode 100644 open-sse/services/compression/rules/zh/ultra.json create mode 100644 tests/unit/caveman-packs-zh-wenyan.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cc150af447..e60c6cc065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ - **compression (caveman):** complete the German, French, and Japanese rule packs with the `dedup` (repeated-context collapsing) and `ultra` (abbreviation / terse) categories they were missing — these three languages previously shipped only `context`/`filler`/`structural`, while `en`/`es`/`id`/`pt-BR` had all five. So a de/fr/ja conversation compressed at higher intensities now collapses repeated boilerplate ("wie bereits besprochen" → "Siehe oben.", "comme mentionné précédemment" → "Voir ci-dessus.", "前述のとおり" → "（上記参照）") and abbreviates dense technical vocabulary (`Datenbank`→`DB`, `Authentifizierung`→`Auth`; `base de données`→`BD`, `authentification`→`auth`; `データベース`→`DB`, `アプリケーション`→`app`). Patterns mirror the existing `es` pack and stay ReDoS-safe (bounded literal alternations; the CJK pack uses no `\b` since Japanese has no word boundaries). Regression guard: `tests/unit/caveman-packs-de-fr-ja.test.ts` (packs load + validate + shrink a representative sample). gaps v3.8.42 — T05/C2. +### ✨ New Features + +- **compression (caveman):** add a **Chinese (zh / wenyan 文言) input-side rule pack** — the counterpart of the existing output-side `terse-cjk` style. New `rules/zh/{dedup,filler,ultra}.json` collapse repeated context ("如前所述" → "见上。"), drop pleasantries/hedging ("请帮我…/谢谢/我觉得"), strip sentence-final modal particles ("吗/呢/吧"), and abbreviate dense technical terms ("数据库"→"DB", "应用程序"→"app"). Chinese is now auto-detected: `detectCompressionLanguage` distinguishes zh from ja by Han-without-kana (kana is Japanese-exclusive, so a Han-heavy Japanese sentence still resolves to `ja`), and `zh` is listed in `listSupportedCompressionLanguages`. Patterns are ReDoS-safe (bounded literal alternations, no `\b` since CJK has no word boundaries). Regression guard: `tests/unit/caveman-packs-zh-wenyan.test.ts` (packs load + validate + shrink; zh/ja/non-CJK detection). gaps v3.8.42 — T05/C6. + --- ## [3.8.41] — 2026-06-29 diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts index 230d6086f8..bbd4a79f83 100644 --- a/open-sse/services/compression/languageDetector.ts +++ b/open-sse/services/compression/languageDetector.ts @@ -17,6 +17,14 @@ const LANGUAGE_HINTS: Record = { * zero hits → English. (B-LANG-DETECTOR) */ export function detectCompressionLanguage(text: string): string { + // CJK disambiguation: Han ideographs (U+4E00–U+9FFF) are shared by Chinese and Japanese, but + // kana (U+3040–U+30FF) is Japanese-exclusive. Text with Han and no kana is Chinese (zh); text + // with kana falls through to the scorer below, where the `ja` kana hint catches it. Keeping zh + // out of the additive scorer means a Han-heavy Japanese sentence is never misread as Chinese. + if (/[一-鿿]/.test(text) && !/[぀-ヿ]/.test(text)) { + return "zh"; + } + let best = "en"; let bestScore = 0; for (const [language, patterns] of Object.entries(LANGUAGE_HINTS)) { @@ -37,5 +45,7 @@ export function detectCompressionLanguage(text: string): string { } export function listSupportedCompressionLanguages(): string[] { - return ["en", ...Object.keys(LANGUAGE_HINTS)]; + // zh is detected via the CJK Han/kana disambiguation in detectCompressionLanguage rather than a + // keyword hint (so it stays out of the additive scorer), hence it is listed explicitly here. + return ["en", "zh", ...Object.keys(LANGUAGE_HINTS)]; } diff --git a/open-sse/services/compression/rules/zh/dedup.json b/open-sse/services/compression/rules/zh/dedup.json new file mode 100644 index 0000000000..1f5669adea --- /dev/null +++ b/open-sse/services/compression/rules/zh/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "zh", + "category": "dedup", + "rules": [ + { + "name": "zh_repeated_context", + "pattern": "(?:如前所述|如之前所说|如先前提到|正如我之前所说)", + "replacement": "见上。", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "zh_repeated_question", + "pattern": "(?:和之前一样的问题|我之前问过这个问题|这是同一个问题)", + "replacement": "［同问］", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "zh_reestablished_context", + "pattern": "(?:回到上面的代码|关于之前提到的内容|回到之前的话题)", + "replacement": "Re: ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "zh_summary_replacement", + "pattern": "(?:总结一下我们讨论的内容|总结我们的对话|概括来说)", + "replacement": "总结：", + "context": "assistant", + "category": "dedup", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/zh/filler.json b/open-sse/services/compression/rules/zh/filler.json new file mode 100644 index 0000000000..33b789f7f8 --- /dev/null +++ b/open-sse/services/compression/rules/zh/filler.json @@ -0,0 +1,54 @@ +{ + "language": "zh", + "category": "filler", + "rules": [ + { + "name": "zh_filler_please", + "pattern": "请(?:你|您)?(?:帮我|帮忙)?", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "zh_filler_thanks", + "pattern": "(?:谢谢|多谢|感谢)(?:你|您|大家)?[。，！]?", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "zh_filler_trouble", + "pattern": "(?:麻烦你|麻烦您|劳驾)[，]?", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "zh_filler_greeting", + "pattern": "(?:你好|您好|大家好)[，,]?", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "zh_filler_hedge_think", + "pattern": "(?:我觉得|我认为|我想说)", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "full" + }, + { + "name": "zh_filler_hedge_actually", + "pattern": "(?:其实|说实话|基本上)[，,]?", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "full" + } + ] +} diff --git a/open-sse/services/compression/rules/zh/ultra.json b/open-sse/services/compression/rules/zh/ultra.json new file mode 100644 index 0000000000..0c15a6fca1 --- /dev/null +++ b/open-sse/services/compression/rules/zh/ultra.json @@ -0,0 +1,55 @@ +{ + "language": "zh", + "category": "ultra", + "rules": [ + { + "name": "zh_ultra_modal_particles", + "pattern": "(?:吗|呢|吧|啊|呀|嘛)(?=[。，！？、\\s]|$)", + "flags": "g", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "zh_ultra_database_abbreviation", + "pattern": "数据库", + "replacement": "DB", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "zh_ultra_application_abbreviation", + "pattern": "应用程序", + "replacement": "app", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "zh_ultra_dependency_abbreviation", + "pattern": "依赖关系", + "replacement": "dep", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "zh_ultra_config_file_abbreviation", + "pattern": "配置文件", + "replacement": "cfg", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "zh_ultra_function_abbreviation", + "pattern": "函数", + "replacement": "fn", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + } + ] +} diff --git a/tests/unit/caveman-packs-zh-wenyan.test.ts b/tests/unit/caveman-packs-zh-wenyan.test.ts new file mode 100644 index 0000000000..cf22be97b9 --- /dev/null +++ b/tests/unit/caveman-packs-zh-wenyan.test.ts @@ -0,0 +1,69 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + loadRulePack, + loadAllRulesForLanguage, +} from "../../open-sse/services/compression/ruleLoader.ts"; +import { + detectCompressionLanguage, + listSupportedCompressionLanguages, +} from "../../open-sse/services/compression/languageDetector.ts"; + +// T05 / C6 — Chinese (zh / wenyan) caveman pack: the input-side counterpart of the existing +// output-side "terse-cjk" (文言) style. Adds dedup + filler + ultra rule packs and zh detection. + +type LoadedRule = ReturnType[number]; + +function applyAll(text: string, rules: LoadedRule[]): string { + return rules.reduce((acc, rule) => { + return typeof rule.replacement === "function" + ? acc.replace(rule.pattern, rule.replacement) + : acc.replace(rule.pattern, rule.replacement); + }, text); +} + +test("zh: dedup + filler + ultra packs load and validate", () => { + // loadRulePack throws on validateRulePack failure, so a non-throwing non-empty load proves + // both presence and schema validity. + for (const category of ["dedup", "filler", "ultra"]) { + const rules = loadRulePack("zh", category, { refresh: true }); + assert.ok(rules.length > 0, `zh/${category} should have rules`); + } +}); + +test("zh: the pack shrinks a representative sample (classical terseness)", () => { + const all = loadAllRulesForLanguage("zh", { refresh: true }); + const sample = "请帮我修复数据库的错误，谢谢。如前所述，应用程序需要修复。"; + const out = applyAll(sample, all); + assert.ok( + out.length < sample.length, + `zh pack should shrink the sample (${sample.length} -> ${out.length})` + ); +}); + +test("zh: loadAllRulesForLanguage exposes dedup, filler and ultra categories", () => { + const cats = new Set(loadAllRulesForLanguage("zh", { refresh: true }).map((r) => r.category)); + assert.ok(cats.has("dedup"), "zh should expose dedup"); + assert.ok(cats.has("filler"), "zh should expose filler"); + assert.ok(cats.has("ultra"), "zh should expose ultra"); +}); + +test("detectCompressionLanguage: Han without kana → zh", () => { + assert.equal(detectCompressionLanguage("请帮我修复这个文件的错误"), "zh"); + assert.equal(detectCompressionLanguage("这个应用程序的数据库配置有问题"), "zh"); +}); + +test("detectCompressionLanguage: kana still wins for Japanese (no zh regression)", () => { + // Han-heavy Japanese with kana must stay ja, not flip to zh. + assert.equal(detectCompressionLanguage("このコードを修正してください"), "ja"); + assert.equal(detectCompressionLanguage("実装の認証を修正する必要があります"), "ja"); +}); + +test("detectCompressionLanguage: non-CJK languages are unaffected", () => { + assert.equal(detectCompressionLanguage("necesito corregir este archivo con error"), "es"); + assert.equal(detectCompressionLanguage("ich brauche diese konfiguration"), "de"); +}); + +test("zh is reported as a supported compression language", () => { + assert.ok(listSupportedCompressionLanguages().includes("zh")); +});