From 9eb7c2f17ea383eb31d51030e325e622aff23f07 Mon Sep 17 00:00:00 2001 From: botea <74217394+botii16@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:22:34 +0200 Subject: [PATCH] feat(compression): add Hungarian Caveman language pack (#12825) * feat(compression): add Hungarian Caveman language pack * docs(changelog): add Hungarian Caveman entry * chore(changelog): make the fragment a well-formed bullet changelog.d/ fragments must start with "- " (scripts/release/aggregate-changelog.mjs). Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: botii16 Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../features/12825-hungarian-caveman.md | 1 + open-sse/services/compression/caveman.ts | 22 ++- .../services/compression/languageDetector.ts | 1 + open-sse/services/compression/outputMode.ts | 5 + .../compression/rules/hu/context.json | 108 +++++++++++ .../services/compression/rules/hu/dedup.json | 56 ++++++ .../services/compression/rules/hu/filler.json | 173 ++++++++++++++++++ .../compression/rules/hu/structural.json | 126 +++++++++++++ .../services/compression/rules/hu/ultra.json | 130 +++++++++++++ tests/unit/compression/language-packs.test.ts | 20 +- 10 files changed, 635 insertions(+), 7 deletions(-) create mode 100644 changelog.d/features/12825-hungarian-caveman.md create mode 100644 open-sse/services/compression/rules/hu/context.json create mode 100644 open-sse/services/compression/rules/hu/dedup.json create mode 100644 open-sse/services/compression/rules/hu/filler.json create mode 100644 open-sse/services/compression/rules/hu/structural.json create mode 100644 open-sse/services/compression/rules/hu/ultra.json diff --git a/changelog.d/features/12825-hungarian-caveman.md b/changelog.d/features/12825-hungarian-caveman.md new file mode 100644 index 0000000000..d1f51e6f27 --- /dev/null +++ b/changelog.d/features/12825-hungarian-caveman.md @@ -0,0 +1 @@ +- **compression:** add Hungarian Caveman language pack with Hungarian-specific rules, language detection, localized output instructions, and language-pack tests. (#12825 - thanks @botii16) diff --git a/open-sse/services/compression/caveman.ts b/open-sse/services/compression/caveman.ts index c629cb125e..4b05d85aab 100644 --- a/open-sse/services/compression/caveman.ts +++ b/open-sse/services/compression/caveman.ts @@ -163,14 +163,26 @@ const RULE_KEYWORDS: Record = { const ARTICLE_HINT_RE = /\b(?:a|an|the)\b/; -function shouldAttemptRule(ruleName: string, lowerText: string): boolean { - if (ruleName === "articles") { +function shouldAttemptRule(rule: CavemanRule, lowerText: string): boolean { + if (rule.name === "articles") { ARTICLE_HINT_RE.lastIndex = 0; return ARTICLE_HINT_RE.test(lowerText); } - const keywords = RULE_KEYWORDS[ruleName]; - return !keywords || keywords.some((keyword) => lowerText.includes(keyword)); + const keywords = RULE_KEYWORDS[rule.name]; + + if (!keywords) return true; + + // File-based language packs have their own localized regexes. + // Do not block them using the English-only keyword prefilter. + if (!CAVEMAN_RULES.includes(rule)) { + rule.pattern.lastIndex = 0; + const matches = rule.pattern.test(lowerText); + rule.pattern.lastIndex = 0; + return matches; + } + + return keywords.some((keyword) => lowerText.includes(keyword)); } export function applyRulesToText( @@ -182,7 +194,7 @@ export function applyRulesToText( const appliedRules: string[] = []; for (const rule of rules) { - if (!shouldAttemptRule(rule.name, lowerResult)) continue; + if (!shouldAttemptRule(rule, lowerResult)) continue; const before = result; const { pattern, replacement } = rule; diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts index 9e1851443c..944c66c1f9 100644 --- a/open-sse/services/compression/languageDetector.ts +++ b/open-sse/services/compression/languageDetector.ts @@ -9,6 +9,7 @@ const LANGUAGE_HINTS: Record = { fr: [/\b(?:fichier|erreur|merci|peux|besoin)\b/i], ru: [/\b(?:\u044d\u0442\u043e|\u0447\u0442\u043e|\u043a\u0430\u043a|\u0435\u0441\u043b\u0438|\u0447\u0442\u043e\u0431\u044b|\u043a\u043e\u0442\u043e\u0440\u044b\u0439|\u043c\u043e\u0436\u0435\u0442|\u043d\u0443\u0436\u043d\u043e|\u0435\u0441\u0442\u044c|\u0431\u044b\u043b\u043e|\u0431\u0443\u0434\u0435\u0442|\u043c\u043e\u0436\u043d\u043e|\u0434\u043e\u043b\u0436\u0435\u043d|\u0444\u0430\u0439\u043b|\u043e\u0448\u0438\u0431\u043a\u0430|\u043f\u0440\u043e\u0431\u043b\u0435\u043c\u0430|\u0434\u0430\u043d\u043d\u044b\u0435)\b/i, /[\u0430-\u044f\u0451]/i], ja: [/[\u3040-\u30ff]/], + hu: [/\b(?:kérlek|tudnád|szeretném|magyarázd|mutasd|miért|hogyan|függvény|beállítás|adatbázis|következő|problémám|megoldani|hiba|fájl)\b/i], id: [/\b(?:saya|kamu|anda|dengan|untuk|yang|tidak|bisa|terima\s+kasih|dari)\b/i], }; diff --git a/open-sse/services/compression/outputMode.ts b/open-sse/services/compression/outputMode.ts index 0de0095f5a..bc60b2a54a 100644 --- a/open-sse/services/compression/outputMode.ts +++ b/open-sse/services/compression/outputMode.ts @@ -39,6 +39,11 @@ export const CAVEMAN_INSTRUCTION_BY_LANGUAGE = { full: `Responda seco e compacto. Frases curtas OK. Preserve todo conteudo tecnico, codigo, erros, URLs e identificadores exatamente. ${SHARED_BOUNDARIES}`, ultra: `Responda ultra compacto. Use prosa tecnica curta e abreviacoes comuns como DB/auth/config/req/res/fn. Nunca abrevie simbolos de codigo, APIs, erros, URLs ou identificadores. ${SHARED_BOUNDARIES}`, }, + hu: { + lite: `Válaszolj tömören. Hagyd el a tölteléket, udvariaskodást és bizonytalanságot. Tartsd meg a teljes mondatokat, technikai kifejezéseket, kódot, hibákat, URL-eket és azonosítókat pontosan. ${SHARED_BOUNDARIES}`, + full: `Válaszolj tömören, mint egy okos barlanglakó. Hagyd el a tölteléket, udvariaskodást és körülírást. Rövid mondatok és töredékek rendben. Tartsd meg minden technikai tartalmat, kódot, hibát, URL-t és azonosítót pontosan. ${SHARED_BOUNDARIES}`, + ultra: `Válaszolj ultra tömören. Maximális tömörítés, távirati stílus. Használj gyakori rövidítéseket (DB/auth/config/req/res/fn/impl), és hagyd el a felesleges kötőszavakat. Soha ne rövidíts kódszimbólumokat, API-neveket, hibaüzeneteket, URL-eket vagy azonosítókat. ${SHARED_BOUNDARIES}`, + }, es: { lite: `Responde conciso. Quita relleno, cortesias y dudas. Conserva terminos tecnicos, codigo, errores, URLs e identificadores exactos. ${SHARED_BOUNDARIES}`, full: `Responde seco y compacto. Fragmentos OK. Conserva todo el contenido tecnico, codigo, errores, URLs e identificadores exactos. ${SHARED_BOUNDARIES}`, diff --git a/open-sse/services/compression/rules/hu/context.json b/open-sse/services/compression/rules/hu/context.json new file mode 100644 index 0000000000..e39c9351dd --- /dev/null +++ b/open-sse/services/compression/rules/hu/context.json @@ -0,0 +1,108 @@ +{ + "language": "hu", + "category": "context", + "rules": [ + { + "name": "explanatory_prefix", + "pattern": "(? { - it("ships 6 language packs with at least 15 rules each", () => { + it("ships 7 language packs with at least 15 rules each", () => { for (const language of LANGUAGES) { const rules = loadAllRulesForLanguage(language, { refresh: true }); assert.ok(rules.length >= 15, `${language} expected 15+ rules, got ${rules.length}`); @@ -38,6 +38,7 @@ describe("Caveman language packs", () => { assert.equal(detectCompressionLanguage("j'ai besoin de corriger cette erreur fichier"), "fr"); assert.equal(detectCompressionLanguage("このコードを修正してください"), "ja"); assert.equal(detectCompressionLanguage("bisa tolong jelaskan tentang database ini"), "id"); + assert.equal(detectCompressionLanguage("kérlek javítsd ezt a hibát a kódban"), "hu"); }); it("applies non-English rule packs to golden samples", () => { @@ -94,6 +95,20 @@ describe("Caveman language packs", () => { assert.ok(text.includes("src/auth.ts")); }); + it("applies Hungarian rules without touching technical terms", () => { + const huRules = getRulesForContext("user", "ultra", "hu"); + const { text } = applyRulesToText( + "Kérlek, adj részletes magyarázatot az adatbázis és a hitelesítés hibájáról a src/auth.ts fájlban.", + huRules + ); + + assert.ok(!text.toLowerCase().includes("kérlek")); + assert.ok(!text.toLowerCase().includes("részletes magyarázatot")); + assert.ok(text.includes("DB")); + assert.ok(text.includes("auth")); + assert.ok(text.includes("src/auth.ts")); + }); + it("builds localized output mode instructions", () => { const config = { enabled: true, intensity: "full" as const, autoClarity: true }; @@ -103,5 +118,6 @@ describe("Caveman language packs", () => { assert.match(buildCavemanOutputInstruction(config, "fr"), /Reponds/); assert.match(buildCavemanOutputInstruction(config, "ja"), /回答/); assert.match(buildCavemanOutputInstruction(config, "id"), /Jawab/); + assert.match(buildCavemanOutputInstruction(config, "hu"), /Válaszolj/); }); });