feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.
This commit is contained in:
Grigoriy
2026-10-24 05:26:13 +03:00
committed by GitHub
parent c3037fc716
commit 2f62dadd25
34 changed files with 379 additions and 35 deletions
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "العقد",
"eventOutboundDown": "غير متصل",
"eventOutboundUp": "متصل",
"eventDownThresholdHint": "فحوصات فاشلة متتالية",
"eventXrayCrash": "تعطّل",
"eventNodeDown": "غير متصلة",
"eventNodeUp": "متصلة",
+1
View File
@@ -1610,6 +1610,7 @@
"eventGroupNode": "Nodes",
"eventOutboundDown": "Down",
"eventOutboundUp": "Up",
"eventDownThresholdHint": "failed checks in a row",
"eventXrayCrash": "Crash",
"eventNodeDown": "Down",
"eventNodeUp": "Up",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Nodos",
"eventOutboundDown": "Caído",
"eventOutboundUp": "Activo",
"eventDownThresholdHint": "comprobaciones fallidas seguidas",
"eventXrayCrash": "Caída",
"eventNodeDown": "Caído",
"eventNodeUp": "Activo",
+1
View File
@@ -1492,6 +1492,7 @@
"eventGroupNode": "نودها",
"eventOutboundDown": "قطع",
"eventOutboundUp": "وصل",
"eventDownThresholdHint": "بررسی ناموفق پیاپی",
"eventXrayCrash": "کرش",
"eventNodeDown": "قطع",
"eventNodeUp": "وصل",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Node",
"eventOutboundDown": "Mati",
"eventOutboundUp": "Aktif",
"eventDownThresholdHint": "pemeriksaan gagal berturut-turut",
"eventXrayCrash": "Crash",
"eventNodeDown": "Mati",
"eventNodeUp": "Aktif",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "ノード",
"eventOutboundDown": "ダウン",
"eventOutboundUp": "アップ",
"eventDownThresholdHint": "回連続でチェック失敗",
"eventXrayCrash": "クラッシュ",
"eventNodeDown": "ダウン",
"eventNodeUp": "アップ",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Nós",
"eventOutboundDown": "Inativo",
"eventOutboundUp": "Ativo",
"eventDownThresholdHint": "verificações com falha seguidas",
"eventXrayCrash": "Falha",
"eventNodeDown": "Inativo",
"eventNodeUp": "Ativo",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Узлы",
"eventOutboundDown": "Недоступен",
"eventOutboundUp": "Работает",
"eventDownThresholdHint": "неудачных проверок подряд",
"eventXrayCrash": "Сбой",
"eventNodeDown": "Недоступен",
"eventNodeUp": "В сети",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Düğümler",
"eventOutboundDown": "Çevrimdışı",
"eventOutboundUp": "Çevrimiçi",
"eventDownThresholdHint": "art arda başarısız kontrol",
"eventXrayCrash": "Çökme",
"eventNodeDown": "Çevrimdışı",
"eventNodeUp": "Çevrimiçi",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Вузли",
"eventOutboundDown": "Недоступне",
"eventOutboundUp": "Доступне",
"eventDownThresholdHint": "невдалих перевірок поспіль",
"eventXrayCrash": "Збій",
"eventNodeDown": "Недоступний",
"eventNodeUp": "Доступний",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "Node",
"eventOutboundDown": "Ngừng hoạt động",
"eventOutboundUp": "Hoạt động",
"eventDownThresholdHint": "lần kiểm tra thất bại liên tiếp",
"eventXrayCrash": "Sự cố",
"eventNodeDown": "Ngừng hoạt động",
"eventNodeUp": "Hoạt động",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "节点",
"eventOutboundDown": "断开",
"eventOutboundUp": "恢复",
"eventDownThresholdHint": "次连续检查失败",
"eventXrayCrash": "崩溃",
"eventNodeDown": "离线",
"eventNodeUp": "上线",
+1
View File
@@ -1488,6 +1488,7 @@
"eventGroupNode": "節點",
"eventOutboundDown": "中斷",
"eventOutboundUp": "恢復",
"eventDownThresholdHint": "次連續檢查失敗",
"eventXrayCrash": "當機",
"eventNodeDown": "離線",
"eventNodeUp": "上線",