From 8cb51b79225c32232a136e8becf12016eb9dfc16 Mon Sep 17 00:00:00 2001 From: vinogradovnet Date: Sat, 8 Aug 2026 02:54:20 +0300 Subject: [PATCH] feat(compression): add Russian language pack (#9581) Merge-train validated (tip 6ce4effef8). Vitest failures confirmed as base-red (#9679). --- .../services/compression/languageDetector.ts | 1 + .../compression/rules/ru/context.json | 38 +++++++ .../services/compression/rules/ru/dedup.json | 30 ++++++ .../services/compression/rules/ru/filler.json | 86 +++++++++++++++ .../compression/rules/ru/structural.json | 101 ++++++++++++++++++ .../services/compression/rules/ru/ultra.json | 46 ++++++++ 6 files changed, 302 insertions(+) create mode 100644 open-sse/services/compression/rules/ru/context.json create mode 100644 open-sse/services/compression/rules/ru/dedup.json create mode 100644 open-sse/services/compression/rules/ru/filler.json create mode 100644 open-sse/services/compression/rules/ru/structural.json create mode 100644 open-sse/services/compression/rules/ru/ultra.json diff --git a/open-sse/services/compression/languageDetector.ts b/open-sse/services/compression/languageDetector.ts index d44295d225..9e1851443c 100644 --- a/open-sse/services/compression/languageDetector.ts +++ b/open-sse/services/compression/languageDetector.ts @@ -7,6 +7,7 @@ const LANGUAGE_HINTS: Record = { es: [/\b(?:necesito|archivo|codigo|código|fallo|gracias|puedes)\b/i], de: [/\b(?:ich|datei|fehler|bitte|kannst|konfiguration|danke)\b/i], fr: [/\b(?:fichier|erreur|merci|peux|besoin)\b/i], + ru: [/\b(?:\u044d\u0442\u043e|\u0447\u0442\u043e|\u043a\u0430\u043a|\u0435\u0441\u043b\u0438|\u0447\u0442\u043e\u0431\u044b|\u043a\u043e\u0442\u043e\u0440\u044b\u0439|\u043c\u043e\u0436\u0435\u0442|\u043d\u0443\u0436\u043d\u043e|\u0435\u0441\u0442\u044c|\u0431\u044b\u043b\u043e|\u0431\u0443\u0434\u0435\u0442|\u043c\u043e\u0436\u043d\u043e|\u0434\u043e\u043b\u0436\u0435\u043d|\u0444\u0430\u0439\u043b|\u043e\u0448\u0438\u0431\u043a\u0430|\u043f\u0440\u043e\u0431\u043b\u0435\u043c\u0430|\u0434\u0430\u043d\u043d\u044b\u0435)\b/i, /[\u0430-\u044f\u0451]/i], ja: [/[\u3040-\u30ff]/], id: [/\b(?:saya|kamu|anda|dengan|untuk|yang|tidak|bisa|terima\s+kasih|dari)\b/i], }; diff --git a/open-sse/services/compression/rules/ru/context.json b/open-sse/services/compression/rules/ru/context.json new file mode 100644 index 0000000000..26534688ff --- /dev/null +++ b/open-sse/services/compression/rules/ru/context.json @@ -0,0 +1,38 @@ +{ + "language": "ru", + "category": "context", + "rules": [ + { + "name": "subject_omission", + "pattern": "^(?:Я |Мы |Вы )(?:можем|должны|будем|хотим|нужно)\\b\\s*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "full" + }, + { + "name": "known_fact_hedging", + "pattern": "(?<=\\.)\\s*(?:Возможно|Наверное|Может быть),\\s+", + "replacement": "", + "context": "assistant", + "category": "context", + "minIntensity": "full" + }, + { + "name": "redundant_clarification", + "pattern": "\\b(?:как я уже говорил|как уже упоминалось|как было сказано)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "full" + }, + { + "name": "obvious_continuation", + "pattern": "\\b(?:далее|затем|после этого|в итоге)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "context", + "minIntensity": "ultra" + } + ] +} diff --git a/open-sse/services/compression/rules/ru/dedup.json b/open-sse/services/compression/rules/ru/dedup.json new file mode 100644 index 0000000000..6678979858 --- /dev/null +++ b/open-sse/services/compression/rules/ru/dedup.json @@ -0,0 +1,30 @@ +{ + "language": "ru", + "category": "dedup", + "rules": [ + { + "name": "thought_repetition", + "pattern": "([^.!?]+[.!?])\\s+\\1", + "replacement": "$1", + "context": "all", + "category": "dedup", + "minIntensity": "full" + }, + { + "name": "word_duplication", + "pattern": "\\b(\\w+)\\s+\\1\\b", + "replacement": "$1", + "context": "all", + "category": "dedup", + "minIntensity": "lite" + }, + { + "name": "synonymous_repetition", + "pattern": "\\b(проблема|ошибка)\\b[^.!?]*\\b(проблема|ошибка)\\b", + "replacement": "$1", + "context": "all", + "category": "dedup", + "minIntensity": "full" + } + ] +} diff --git a/open-sse/services/compression/rules/ru/filler.json b/open-sse/services/compression/rules/ru/filler.json new file mode 100644 index 0000000000..aa28ae6153 --- /dev/null +++ b/open-sse/services/compression/rules/ru/filler.json @@ -0,0 +1,86 @@ +{ + "language": "ru", + "category": "filler", + "rules": [ + { + "name": "pleasantries", + "pattern": "\\b(?:конечно|с радостью|рад помочь|могу помочь|обязательно|безусловно|разумеется)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "polite_framing", + "pattern": "\\b(?:пожалуйста|если хотите|если можно|будьте добры|будьте любезны|прошу вас)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "verbal_wrapping", + "pattern": "\\b(?:давайте разберём|давайте посмотрим|попробуем разобраться|постараюсь помочь)\\b[,.!?\\s]*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "hedging", + "pattern": "\\b(?:возможно|наверное|может быть|скорее всего|вероятно|видимо|похоже)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "filler_adverbs", + "pattern": "\\b(?:в целом|на самом деле|в принципе|как правило|по сути|фактически|буквально)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "empty_qualifiers", + "pattern": "\\b(?:удобный|хороший|эффективный|мощный|отличный|прекрасный)(?!\\s+(?:вариант|способ|решение|метод|инструмент))\\b", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "redundant_openers", + "pattern": "^(?:Привет|Здравствуйте|Добрый день|Доброе утро|Добрый вечер)\\s*[,.!?\\s]?\\s*", + "replacement": "", + "context": "user", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "excessive_gratitude", + "pattern": "\\b(?:Большое спасибо|Огромное спасибо|Спасибо заранее|Заранее благодарю|Очень признателен)\\b[,.!?\\s]*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "softeners", + "pattern": "\\b(?:немного|немножко|чуть-чуть|слегка|несколько|как-то)\\b\\s*", + "replacement": "", + "context": "all", + "category": "filler", + "minIntensity": "lite" + }, + { + "name": "assistant_fillers", + "pattern": "^(?:Вот|Ниже|Это|Здесь)\\s+(?:есть|находится)?\\s*", + "replacement": "", + "context": "assistant", + "category": "filler", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/ru/structural.json b/open-sse/services/compression/rules/ru/structural.json new file mode 100644 index 0000000000..78bf745f29 --- /dev/null +++ b/open-sse/services/compression/rules/ru/structural.json @@ -0,0 +1,101 @@ +{ + "language": "ru", + "category": "structural", + "rules": [ + { + "name": "problem_phrasing", + "pattern": "\\b(?:проблема заключается в том, что|дело в том, что|суть в том, что)\\b\\s*", + "replacement": "проблема: ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "causality_verbose", + "pattern": "\\b(?:это приводит к тому, что|это означает, что|из этого следует, что)\\b\\s*", + "replacement": "→ ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "purpose_phrases", + "pattern": "\\b(?:для того чтобы|с целью того чтобы)\\b\\s*", + "replacement": "чтобы ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "causality_phrases", + "pattern": "\\b(?:в связи с тем, что|по причине того, что|ввиду того, что)\\b\\s*", + "replacement": "из-за ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "concession_phrases", + "pattern": "\\b(?:несмотря на то, что|хотя и)\\b\\s*", + "replacement": "хотя ", + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "note_phrases", + "pattern": "\\b(?:стоит отметить, что|следует иметь в виду, что|важно понимать, что|необходимо учитывать, что)\\b\\s*", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "redundant_directive", + "pattern": "\\b(?:важно помнить|не забывайте|помните о том)\\b\\s*", + "replacement": "", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "approximation", + "pattern": "\\b(?:примерно|приблизительно)\\b\\s*", + "replacement": "≈ ", + "context": "all", + "category": "structural", + "minIntensity": "full" + }, + { + "name": "forbidden_abbreviations_dots", + "pattern": "\\b(?:т\\.к\\.|т\\.е\\.|и т\\.д\\.|и т\\.п\\.|см\\.|напр\\.|и др\\.|в т\\.ч\\.)\\b", + "replacement": "", + "replacementMap": { + "т.к.": "так как", + "т.е.": "то есть", + "и т.д.": "", + "и т.п.": "", + "см.": "см", + "напр.": "например", + "и др.": "", + "в т.ч.": "" + }, + "context": "all", + "category": "structural", + "minIntensity": "lite" + }, + { + "name": "forbidden_abbreviations_dash", + "pattern": "\\b(?:кол-во|к-рый|св-во)\\b", + "replacement": "", + "replacementMap": { + "кол-во": "количество", + "к-рый": "который", + "св-во": "свойство" + }, + "context": "all", + "category": "structural", + "minIntensity": "lite" + } + ] +} diff --git a/open-sse/services/compression/rules/ru/ultra.json b/open-sse/services/compression/rules/ru/ultra.json new file mode 100644 index 0000000000..ca2609aac7 --- /dev/null +++ b/open-sse/services/compression/rules/ru/ultra.json @@ -0,0 +1,46 @@ +{ + "language": "ru", + "category": "ultra", + "rules": [ + { + "name": "ultra_compression_conjunctions", + "pattern": "\\b(?:однако|тем не менее|в то время как)\\b\\s*", + "replacement": "—", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ultra_compression_articles", + "pattern": "\\b(?:является|представляет собой)\\b\\s*", + "replacement": "—", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ultra_compression_verbs", + "pattern": "\\b(?:необходимо|требуется|нужно)\\b\\s*", + "replacement": "", + "context": "all", + "category": "ultra", + "minIntensity": "ultra" + }, + { + "name": "ultra_punctuation", + "pattern": "[,:;]\\s+", + "replacement": " ", + "context": "all", + "category": "ultra", + "minIntensity": "notes" + }, + { + "name": "ultra_lowercase", + "pattern": "(?<=\\.)\\s+([А-ЯЁ])", + "replacement": " $1", + "context": "all", + "category": "ultra", + "minIntensity": "notes" + } + ] +}