feat(compression): T05/C6 — Chinese (zh / wenyan) caveman pack + detection (#5532)

Integrated into release/v3.8.42 (round 3). T05/C6 zh/wenyan pack + detection

CHANGELOG.md

@@ -23,6 +23,10 @@
 
 - **compression (caveman):** complete the German, French, and Japanese rule packs with the `dedup` (repeated-context collapsing) and `ultra` (abbreviation / terse) categories they were missing — these three languages previously shipped only `context`/`filler`/`structural`, while `en`/`es`/`id`/`pt-BR` had all five. So a de/fr/ja conversation compressed at higher intensities now collapses repeated boilerplate ("wie bereits besprochen" → "Siehe oben.", "comme mentionné précédemment" → "Voir ci-dessus.", "前述のとおり" → "（上記参照）") and abbreviates dense technical vocabulary (`Datenbank`→`DB`, `Authentifizierung`→`Auth`; `base de données`→`BD`, `authentification`→`auth`; `データベース`→`DB`, `アプリケーション`→`app`). Patterns mirror the existing `es` pack and stay ReDoS-safe (bounded literal alternations; the CJK pack uses no `\b` since Japanese has no word boundaries). Regression guard: `tests/unit/caveman-packs-de-fr-ja.test.ts` (packs load + validate + shrink a representative sample). gaps v3.8.42 — T05/C2.
 
+### ✨ New Features
+
+- **compression (caveman):** add a **Chinese (zh / wenyan 文言) input-side rule pack** — the counterpart of the existing output-side `terse-cjk` style. New `rules/zh/{dedup,filler,ultra}.json` collapse repeated context ("如前所述" → "见上。"), drop pleasantries/hedging ("请帮我…/谢谢/我觉得"), strip sentence-final modal particles ("吗/呢/吧"), and abbreviate dense technical terms ("数据库"→"DB", "应用程序"→"app"). Chinese is now auto-detected: `detectCompressionLanguage` distinguishes zh from ja by Han-without-kana (kana is Japanese-exclusive, so a Han-heavy Japanese sentence still resolves to `ja`), and `zh` is listed in `listSupportedCompressionLanguages`. Patterns are ReDoS-safe (bounded literal alternations, no `\b` since CJK has no word boundaries). Regression guard: `tests/unit/caveman-packs-zh-wenyan.test.ts` (packs load + validate + shrink; zh/ja/non-CJK detection). gaps v3.8.42 — T05/C6.
+
 ---
 
 ## [3.8.41] — 2026-06-29

open-sse/services/compression/languageDetector.ts

@@ -17,6 +17,14 @@ const LANGUAGE_HINTS: Record<string, RegExp[]> = {
  * zero hits → English. (B-LANG-DETECTOR)
  */
 export function detectCompressionLanguage(text: string): string {
+  // CJK disambiguation: Han ideographs (U+4E00–U+9FFF) are shared by Chinese and Japanese, but
+  // kana (U+3040–U+30FF) is Japanese-exclusive. Text with Han and no kana is Chinese (zh); text
+  // with kana falls through to the scorer below, where the `ja` kana hint catches it. Keeping zh
+  // out of the additive scorer means a Han-heavy Japanese sentence is never misread as Chinese.
+  if (/[一-鿿]/.test(text) && !/[぀-ヿ]/.test(text)) {
+    return "zh";
+  }
+
   let best = "en";
   let bestScore = 0;
   for (const [language, patterns] of Object.entries(LANGUAGE_HINTS)) {
@@ -37,5 +45,7 @@ export function detectCompressionLanguage(text: string): string {
 }
 
 export function listSupportedCompressionLanguages(): string[] {
-  return ["en", ...Object.keys(LANGUAGE_HINTS)];
+  // zh is detected via the CJK Han/kana disambiguation in detectCompressionLanguage rather than a
+  // keyword hint (so it stays out of the additive scorer), hence it is listed explicitly here.
+  return ["en", "zh", ...Object.keys(LANGUAGE_HINTS)];
 }

open-sse/services/compression/rules/zh/dedup.json

@@ -0,0 +1,38 @@
+{
+  "language": "zh",
+  "category": "dedup",
+  "rules": [
+    {
+      "name": "zh_repeated_context",
+      "pattern": "(?:如前所述|如之前所说|如先前提到|正如我之前所说)",
+      "replacement": "见上。",
+      "context": "all",
+      "category": "dedup",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_repeated_question",
+      "pattern": "(?:和之前一样的问题|我之前问过这个问题|这是同一个问题)",
+      "replacement": "［同问］",
+      "context": "user",
+      "category": "dedup",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_reestablished_context",
+      "pattern": "(?:回到上面的代码|关于之前提到的内容|回到之前的话题)",
+      "replacement": "Re: ",
+      "context": "all",
+      "category": "dedup",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_summary_replacement",
+      "pattern": "(?:总结一下我们讨论的内容|总结我们的对话|概括来说)",
+      "replacement": "总结：",
+      "context": "assistant",
+      "category": "dedup",
+      "minIntensity": "lite"
+    }
+  ]
+}

open-sse/services/compression/rules/zh/filler.json

@@ -0,0 +1,54 @@
+{
+  "language": "zh",
+  "category": "filler",
+  "rules": [
+    {
+      "name": "zh_filler_please",
+      "pattern": "请(?:你|您)?(?:帮我|帮忙)?",
+      "replacement": "",
+      "context": "user",
+      "category": "filler",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_filler_thanks",
+      "pattern": "(?:谢谢|多谢|感谢)(?:你|您|大家)?[。，！]?",
+      "replacement": "",
+      "context": "all",
+      "category": "filler",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_filler_trouble",
+      "pattern": "(?:麻烦你|麻烦您|劳驾)[，]?",
+      "replacement": "",
+      "context": "user",
+      "category": "filler",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_filler_greeting",
+      "pattern": "(?:你好|您好|大家好)[，,]?",
+      "replacement": "",
+      "context": "all",
+      "category": "filler",
+      "minIntensity": "lite"
+    },
+    {
+      "name": "zh_filler_hedge_think",
+      "pattern": "(?:我觉得|我认为|我想说)",
+      "replacement": "",
+      "context": "all",
+      "category": "filler",
+      "minIntensity": "full"
+    },
+    {
+      "name": "zh_filler_hedge_actually",
+      "pattern": "(?:其实|说实话|基本上)[，,]?",
+      "replacement": "",
+      "context": "all",
+      "category": "filler",
+      "minIntensity": "full"
+    }
+  ]
+}

open-sse/services/compression/rules/zh/ultra.json

@@ -0,0 +1,55 @@
+{
+  "language": "zh",
+  "category": "ultra",
+  "rules": [
+    {
+      "name": "zh_ultra_modal_particles",
+      "pattern": "(?:吗|呢|吧|啊|呀|嘛)(?=[。，！？、\\s]|$)",
+      "flags": "g",
+      "replacement": "",
+      "context": "all",
+      "category": "terse",
+      "minIntensity": "full"
+    },
+    {
+      "name": "zh_ultra_database_abbreviation",
+      "pattern": "数据库",
+      "replacement": "DB",
+      "context": "all",
+      "category": "ultra",
+      "minIntensity": "ultra"
+    },
+    {
+      "name": "zh_ultra_application_abbreviation",
+      "pattern": "应用程序",
+      "replacement": "app",
+      "context": "all",
+      "category": "ultra",
+      "minIntensity": "ultra"
+    },
+    {
+      "name": "zh_ultra_dependency_abbreviation",
+      "pattern": "依赖关系",
+      "replacement": "dep",
+      "context": "all",
+      "category": "ultra",
+      "minIntensity": "ultra"
+    },
+    {
+      "name": "zh_ultra_config_file_abbreviation",
+      "pattern": "配置文件",
+      "replacement": "cfg",
+      "context": "all",
+      "category": "ultra",
+      "minIntensity": "ultra"
+    },
+    {
+      "name": "zh_ultra_function_abbreviation",
+      "pattern": "函数",
+      "replacement": "fn",
+      "context": "all",
+      "category": "ultra",
+      "minIntensity": "ultra"
+    }
+  ]
+}

tests/unit/caveman-packs-zh-wenyan.test.ts

@@ -0,0 +1,69 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  loadRulePack,
+  loadAllRulesForLanguage,
+} from "../../open-sse/services/compression/ruleLoader.ts";
+import {
+  detectCompressionLanguage,
+  listSupportedCompressionLanguages,
+} from "../../open-sse/services/compression/languageDetector.ts";
+
+// T05 / C6 — Chinese (zh / wenyan) caveman pack: the input-side counterpart of the existing
+// output-side "terse-cjk" (文言) style. Adds dedup + filler + ultra rule packs and zh detection.
+
+type LoadedRule = ReturnType<typeof loadRulePack>[number];
+
+function applyAll(text: string, rules: LoadedRule[]): string {
+  return rules.reduce((acc, rule) => {
+    return typeof rule.replacement === "function"
+      ? acc.replace(rule.pattern, rule.replacement)
+      : acc.replace(rule.pattern, rule.replacement);
+  }, text);
+}
+
+test("zh: dedup + filler + ultra packs load and validate", () => {
+  // loadRulePack throws on validateRulePack failure, so a non-throwing non-empty load proves
+  // both presence and schema validity.
+  for (const category of ["dedup", "filler", "ultra"]) {
+    const rules = loadRulePack("zh", category, { refresh: true });
+    assert.ok(rules.length > 0, `zh/${category} should have rules`);
+  }
+});
+
+test("zh: the pack shrinks a representative sample (classical terseness)", () => {
+  const all = loadAllRulesForLanguage("zh", { refresh: true });
+  const sample = "请帮我修复数据库的错误，谢谢。如前所述，应用程序需要修复。";
+  const out = applyAll(sample, all);
+  assert.ok(
+    out.length < sample.length,
+    `zh pack should shrink the sample (${sample.length} -> ${out.length})`
+  );
+});
+
+test("zh: loadAllRulesForLanguage exposes dedup, filler and ultra categories", () => {
+  const cats = new Set(loadAllRulesForLanguage("zh", { refresh: true }).map((r) => r.category));
+  assert.ok(cats.has("dedup"), "zh should expose dedup");
+  assert.ok(cats.has("filler"), "zh should expose filler");
+  assert.ok(cats.has("ultra"), "zh should expose ultra");
+});
+
+test("detectCompressionLanguage: Han without kana → zh", () => {
+  assert.equal(detectCompressionLanguage("请帮我修复这个文件的错误"), "zh");
+  assert.equal(detectCompressionLanguage("这个应用程序的数据库配置有问题"), "zh");
+});
+
+test("detectCompressionLanguage: kana still wins for Japanese (no zh regression)", () => {
+  // Han-heavy Japanese with kana must stay ja, not flip to zh.
+  assert.equal(detectCompressionLanguage("このコードを修正してください"), "ja");
+  assert.equal(detectCompressionLanguage("実装の認証を修正する必要があります"), "ja");
+});
+
+test("detectCompressionLanguage: non-CJK languages are unaffected", () => {
+  assert.equal(detectCompressionLanguage("necesito corregir este archivo con error"), "es");
+  assert.equal(detectCompressionLanguage("ich brauche diese konfiguration"), "de");
+});
+
+test("zh is reported as a supported compression language", () => {
+  assert.ok(listSupportedCompressionLanguages().includes("zh"));
+});