feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.
This commit is contained in:
Grigoriy
2026-10-24 05:26:13 +03:00
committed by GitHub
parent c3037fc716
commit 2f62dadd25
34 changed files with 379 additions and 35 deletions
@@ -18,6 +18,7 @@ import (
"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
)
// goingDownNodes seeds n online nodes whose address refuses connections, so the
@@ -28,6 +29,9 @@ func goingDownNodes(t *testing.T, n int) {
dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db"))
runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}}))
t.Cleanup(func() { runtime.SetManager(nil) })
if err := (&service.SettingService{}).SetNodeDownThreshold(1); err != nil {
t.Fatalf("SetNodeDownThreshold: %v", err)
}
srv := httptest.NewServer(nil)
host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
portNum, _ := strconv.Atoi(port)