From 4b2ac70295b31cc58f633e22a1eadc9f963fd385 Mon Sep 17 00:00:00 2001 From: Anjielon Date: Thu, 6 Aug 2026 05:08:35 +0200 Subject: [PATCH] feat(compression): add Italian (it) Caveman rule pack (#8776) Validated in local merge-train T5 (base49+contributors+pacocartones) --- .../compression/COMPRESSION_LANGUAGE_PACKS.md | 5 +- .../services/compression/languageDetector.ts | 1 + .../compression/rules/it/context.json | 70 ++++++++++++ .../services/compression/rules/it/dedup.json | 38 +++++++ .../services/compression/rules/it/filler.json | 94 ++++++++++++++++ .../compression/rules/it/structural.json | 102 +++++++++++++++++ .../services/compression/rules/it/ultra.json | 106 ++++++++++++++++++ 7 files changed, 414 insertions(+), 2 deletions(-) create mode 100644 open-sse/services/compression/rules/it/context.json create mode 100644 open-sse/services/compression/rules/it/dedup.json create mode 100644 open-sse/services/compression/rules/it/filler.json create mode 100644 open-sse/services/compression/rules/it/structural.json create mode 100644 open-sse/services/compression/rules/it/ultra.json diff --git a/docs/compression/COMPRESSION_LANGUAGE_PACKS.md b/docs/compression/COMPRESSION_LANGUAGE_PACKS.md index eb727dad0b..fad4de2ed0 100644 --- a/docs/compression/COMPRESSION_LANGUAGE_PACKS.md +++ b/docs/compression/COMPRESSION_LANGUAGE_PACKS.md @@ -7,7 +7,7 @@ lastUpdated: 2026-06-28 # Compression Language Packs Caveman compression can load language-specific rule packs in addition to the built-in English rules. -This keeps the core engine stable while allowing Portuguese, Spanish, German, French, Japanese, and +This keeps the core engine stable while allowing Portuguese, Spanish, German, French, Italian, Japanese, and future language packs to evolve independently. ## Location @@ -26,11 +26,12 @@ Current shipped packs (verified against `rules/` directory contents): | Spanish | `rules/es/` | `context`, `dedup`, `filler`, `structural`, `ultra` | | Portuguese (Brazil) | `rules/pt-BR/` | `context`, `dedup`, `filler`, `structural`, `ultra` | | Indonesian | `rules/id/` | `context`, `dedup`, `filler`, `structural`, `ultra` | +| Italian | `rules/it/` | `context`, `dedup`, `filler`, `structural`, `ultra` | | German | `rules/de/` | `context`, `filler`, `structural` | | French | `rules/fr/` | `context`, `filler`, `structural` | | Japanese | `rules/ja/` | `context`, `filler`, `structural` | -> **Parity note:** `en`, `es`, `pt-BR`, and `id` packs have the full 5 categories; `de`, `fr`, `ja` ship 3 categories. The missing `dedup` and `ultra` categories silently fall back to the English built-ins. Contributions welcome to add `dedup.json` and `ultra.json` for the smaller packs. +> **Parity note:** `en`, `es`, `pt-BR`, `id`, and `it` packs have the full 5 categories; `de`, `fr`, `ja` ship 3 categories. The missing `dedup` and `ultra` categories silently fall back to the English built-ins. Contributions welcome to add `dedup.json` and `ultra.json` for the smaller packs. > > The `pt-BR` pack is based on **[Troglodita](https://github.com/leninejunior/troglodita)** by Lenine Júnior — a compression system designed from scratch for Brazilian Portuguese grammar (pleonasm reduction, PT-BR filler removal, technical abbreviations for the dev BR community). > diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts index bbd4a79f83..d44295d225 100644 --- a/open-sse/services/compression/languageDetector.ts +++ b/open-sse/services/compression/languageDetector.ts @@ -1,4 +1,5 @@ const LANGUAGE_HINTS: Record = { + it: [/\b(?:perche|perché|pero|però|cioe|cioè|quindi|potresti|vorrei|adesso|errore|grazie|devo|voglio|questo|quello|anche|sono|molto)\b/i], "pt-BR": [/\b(?:voce|você|preciso|arquivo|codigo|código|erro|falha|obrigado)\b/i], // NOTE: English-ambiguous words are intentionally excluded — "error" (es) and // "configuration" (fr) are identical in English and would misclassify English text. diff --git a/open-sse/services/compression/rules/it/context.json b/open-sse/services/compression/rules/it/context.json new file mode 100644 index 0000000000..745b6d0116 --- /dev/null +++ b/open-sse/services/compression/rules/it/context.json @@ -0,0 +1,70 @@ +{ + "language": "it", + "category": "context", + "rules": [ + { + "name": "it_context_setup", + "pattern": "\\b(?:ecco (?:qui )?il codice|questo è il codice|qui sotto (?:c'è |trovi )?il codice|di seguito il codice|ti allego il codice)\\b\\s*[:.]?\\s*", + "replacement": "Codice:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_intent", + "pattern": "\\b(?:il mio obiettivo è|quello che (?:mi serve|voglio|devo fare) è|quello che sto cercando di fare è|l'idea è (?:quella di |))\\b\\s*", + "replacement": "Obiettivo:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_error_report", + "pattern": "\\b(?:mi (?:dà|da) (?:questo |il seguente |)errore|ricevo (?:questo |il seguente |)errore|ottengo (?:questo |il seguente |)errore|l'errore che (?:mi dà|ricevo) è)\\b\\s*[:.]?\\s*", + "replacement": "Errore:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_expected_behavior", + "pattern": "\\b(?:quello che mi aspetto è|dovrebbe (?:invece |)(?:fare|succedere|restituire)|il comportamento atteso è)\\b\\s*[:.]?\\s*", + "replacement": "Atteso:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_already_tried", + "pattern": "\\b(?:ho (?:già |)provato a|ho tentato di|ho cercato di)\\b\\s*", + "replacement": "Provato:", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_question_directive", + "pattern": "\\b(?:mi (?:sapresti |sai |puoi |)dire (?:se|come|cosa|quando|perché)|(?:sai|sapresti) (?:dirmi )?(?:se|come|cosa|quando|perché))\\b\\s*", + "replacement": "", + "context": "user", + "category": "context", + "minIntensity": "lite" + }, + { + "name": "it_environment_preamble", + "pattern": "\\b(?:sto (?:lavorando|usando|utilizzando)|nel mio (?:progetto|sistema|ambiente))\\b\\s*", + "replacement": "", + "context": "user", + "category": "context", + "minIntensity": "full" + }, + { + "name": "it_scope_note", + "pattern": "\\b(?:tieni (?:presente|conto) che|considera che|nota che|da notare che)\\b\\s*", + "replacement": "NB:", + "context": "all", + "category": "context", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/it/dedup.json b/open-sse/services/compression/rules/it/dedup.json new file mode 100644 index 0000000000..8513e486d1 --- /dev/null +++ b/open-sse/services/compression/rules/it/dedup.json @@ -0,0 +1,38 @@ +{ + "language": "it", + "category": "dedup", + "rules": [ + { + "name": "it_repeated_context", + "pattern": "\\b(?:come (?:ti )?(?:ho )?(?:già |)(?:detto|accennato|scritto|spiegato) (?:prima|sopra|in precedenza)|come dicevo|come sopra|come menzionato)\\b[,.]?\\s*", + "replacement": "Vedi sopra. ", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "it_repeated_question", + "pattern": "\\b(?:stessa domanda di prima|te l'ho già chiesto|è la stessa domanda|come chiedevo prima)\\b[,.]?\\s*", + "replacement": "[stessa domanda] ", + "context": "user", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "it_restating", + "pattern": "\\b(?:in altre parole|detto altrimenti|ovvero|vale a dire|cioè per essere chiari)\\b[,:]?\\s*", + "replacement": "cioè ", + "context": "all", + "category": "dedup", + "minIntensity": "full" + }, + { + "name": "it_recap_preamble", + "pattern": "\\b(?:ricapitolando|per ricapitolare|facciamo il punto|riepilogo)\\b[,:]?\\s*", + "replacement": "", + "context": "assistant", + "category": "dedup", + "minIntensity": "full" + } + ] +} diff --git a/open-sse/services/compression/rules/it/filler.json b/open-sse/services/compression/rules/it/filler.json new file mode 100644 index 0000000000..62649d7663 --- /dev/null +++ b/open-sse/services/compression/rules/it/filler.json @@ -0,0 +1,94 @@ +{ + "language": "it", + "category": "filler", + "rules": [ + { + "name": "it_polite_framing", + "pattern": "\\b(?:per favore|per cortesia|ti (?:pre|)gherei di|potresti|puoi|riusciresti a|ti dispiacerebbe|se puoi|se ti va|quando puoi|gentilmente)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_pleasantries", + "pattern": "\\b(?:ciao|buongiorno|buonasera|buon pomeriggio|salve|grazie mille|grazie tante|ti ringrazio|grazie)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_hedging", + "pattern": "\\b(?:credo che|penso che|mi sembra che|mi pare che|direi che|secondo me|a mio (?:parere|avviso)|forse|magari|probabilmente|presumibilmente|verosimilmente)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_filler_adverbs", + "pattern": "\\b(?:sostanzialmente|essenzialmente|fondamentalmente|praticamente|in realtà|in effetti|letteralmente|semplicemente|diciamo|insomma|comunque|appunto|ovviamente|chiaramente)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_self_reference", + "pattern": "^(?:sto cercando di|vorrei|volevo|avrei bisogno di|ho bisogno di|mi servirebbe|mi serve|vorrei sapere se|volevo sapere se|voglio)\\b\\s*", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_verbose_request", + "pattern": "\\b(?:potresti spiegarmi|puoi spiegarmi|mi spieghi|mi puoi spiegare|potresti dettagliare|puoi dettagliare|mi sapresti dire|sapresti dirmi|mi dici)\\b\\s*", + "replacement": "spiega ", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_assistant_servility", + "pattern": "\\b(?:certamente|assolutamente|volentieri|con piacere|sarei felice di aiutarti|sono felice di aiutarti|ottima domanda|bella domanda|buona domanda)\\b[,.!]?\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_assistant_preamble", + "pattern": "^(?:ecco|ecco qui|ecco a te|allora|dunque|bene)\\b[,:]?\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_closing_offer", + "pattern": "\\b(?:fammi sapere se (?:hai bisogno|ti serve|vuoi)[^.!?]*|se hai (?:altre |ulteriori )?(?:domande|dubbi)[^.!?]*|spero (?:che )?(?:questo )?(?:ti )?(?:sia (?:stato )?d'aiuto|aiuti)[^.!?]*)[.!?]\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "it_apology", + "pattern": "\\b(?:mi scuso per|scusa per|chiedo scusa per|mi dispiace per)\\b[^.!?]*[.!?]\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "full" + }, + { + "name": "it_softeners", + "pattern": "\\b(?:un attimo|un momento|se non ti dispiace|se non è troppo disturbo|se possibile)\\b[,.]?\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/it/structural.json b/open-sse/services/compression/rules/it/structural.json new file mode 100644 index 0000000000..1c87e76d3d --- /dev/null +++ b/open-sse/services/compression/rules/it/structural.json @@ -0,0 +1,102 @@ +{ + "language": "it", + "category": "structural", + "rules": [ + { + "name": "it_purpose", + "pattern": "\\b(?:al fine di|allo scopo di|con l'obiettivo di|in modo da poter|in modo da|così da|in maniera tale da)\\b\\s*", + "replacement": "per ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_causal", + "pattern": "\\b(?:a causa del fatto che|per il fatto che|dal momento che|visto che|dato che|in quanto)\\b\\s*", + "replacement": "perché ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_connectors", + "pattern": "\\b(?:inoltre|in aggiunta|per di più|d'altra parte|d'altro canto|oltre a ciò)\\b[,]?\\s*", + "replacement": "anche ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_emphasis", + "pattern": "\\b(?:molto|davvero|veramente|estremamente|parecchio|piuttosto|abbastanza|super)\\s+(?=[a-zàèéìòùA-Z])", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "it_redundant_phrasing", + "pattern": "\\bnel caso in cui\\b", + "replacement": "se", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_time_redundancy", + "pattern": "\\b(?:nel momento in cui|nell'istante in cui|nel periodo in cui)\\b", + "replacement": "quando", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_place_redundancy", + "pattern": "\\b(?:all'interno di|nell'ambito di|nel contesto di)\\b", + "replacement": "in", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_recommendation", + "pattern": "\\b(?:ti (?:consiglierei|consiglio|suggerirei|suggerisco) di|sarebbe (?:meglio|opportuno|consigliabile)|converrebbe)\\b\\s*", + "replacement": "usa ", + "context": "assistant", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "it_possibility", + "pattern": "\\b(?:è possibile che|potrebbe essere che|può darsi che)\\b\\s*", + "replacement": "forse ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "it_necessity", + "pattern": "\\b(?:è necessario che|occorre che|bisogna che|è indispensabile che)\\b\\s*", + "replacement": "serve che ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "it_conclusion", + "pattern": "\\b(?:in conclusione|per concludere|riassumendo|in sintesi|tirando le somme)\\b[,:]?\\s*", + "replacement": "", + "context": "assistant", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "it_double_negation", + "pattern": "\\bnon è (?:possibile|fattibile) (?:non |)\\b", + "replacement": "non si può ", + "context": "all", + "category": "structural", + "minIntensity": "full" + } + ] +} diff --git a/open-sse/services/compression/rules/it/ultra.json b/open-sse/services/compression/rules/it/ultra.json new file mode 100644 index 0000000000..6d8c486677 --- /dev/null +++ b/open-sse/services/compression/rules/it/ultra.json @@ -0,0 +1,106 @@ +{ + "language": "it", + "category": "ultra", + "rules": [ + { + "name": "it_articles", + "pattern": "\\b(?:[Ii]l|[Ll]o|[Ll]a|[Ii]|[Gg]li|[Ll]e|[Uu]n|[Uu]no|[Uu]na)\\s+(?=[a-zàèéìòù])", + "flags": "g", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_leader_phrases", + "pattern": "^(?:posso|possiamo|vado a|andiamo a|proviamo a|fammi|lasciami|si può)\\s+(?=[a-zàèéìòù])", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_ultra_database", + "pattern": "\\bbase(?:e|) dati\\b|\\bbase di dati\\b", + "replacement": "DB", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_ultra_config", + "pattern": "\\bconfigurazion(?:e|i)\\b", + "replacement": "config", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_ultra_function", + "pattern": "\\bfunzion(?:e|i)\\b", + "replacement": "fn", + "context": "all", + "category": "terse", + "minIntensity": "ultra" + }, + { + "name": "it_ultra_variable", + "pattern": "\\bvariabil(?:e|i)\\b", + "replacement": "var", + "context": "all", + "category": "terse", + "minIntensity": "ultra" + }, + { + "name": "it_ultra_application", + "pattern": "\\bapplicazion(?:e|i)\\b", + "replacement": "app", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_ultra_implementation", + "pattern": "\\bimplementazion(?:e|i)\\b", + "replacement": "impl", + "context": "all", + "category": "terse", + "minIntensity": "ultra" + }, + { + "name": "it_ultra_directory", + "pattern": "\\b(?:cartell(?:a|e)|director(?:y|ies))\\b", + "replacement": "dir", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_ultra_error", + "pattern": "\\bmessaggio di errore\\b", + "replacement": "errore", + "context": "all", + "category": "terse", + "minIntensity": "full" + }, + { + "name": "it_copula_drop", + "pattern": "\\b(?:che )?(?:è|sono) (?:un|una|il|la|lo|gli|le)\\s+(?=[a-zàèéìòù])", + "replacement": "", + "context": "all", + "category": "terse", + "minIntensity": "ultra" + }, + { + "name": "it_ultra_environment", + "pattern": "\\bambiente di (?:sviluppo|produzione)\\b", + "replacementMap": { + "ambiente di sviluppo": "dev", + "ambiente di produzione": "prod" + }, + "context": "all", + "category": "terse", + "minIntensity": "ultra" + } + ] +}