feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.

internal/database/db.go

@@ -1475,7 +1475,7 @@ func runSeeders(isUsersEmpty bool) error {
 	}
 
 	if empty && isUsersEmpty {
-		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "ResetIpLimitNoFail2ban"}
+		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "NodeDownNotified", "ResetIpLimitNoFail2ban"}
 		for _, name := range seeders {
 			if err := db.Create(&model.HistoryOfSeeders{SeederName: name}).Error; err != nil {
 				return err
@@ -1640,6 +1640,12 @@ func runSeeders(isUsersEmpty bool) error {
 		}
 	}
 
+	if !slices.Contains(seedersHistory, "NodeDownNotified") {
+		if err := seedNodeDownNotified(); err != nil {
+			return err
+		}
+	}
+
 	if err := seedHostsFromExternalProxy(); err != nil {
 		return err
 	}
@@ -1689,6 +1695,17 @@ func seedNodeInboundsAdopted() error {
 	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeInboundsAdopted"}).Error
 }
 
+// seedNodeDownNotified treats nodes already offline at upgrade as announced,
+// so the first heartbeat after it does not re-send their node.down.
+func seedNodeDownNotified() error {
+	if err := db.Model(&model.Node{}).
+		Where("status = ?", "offline").
+		Update("down_notified", true).Error; err != nil {
+		return err
+	}
+	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeDownNotified"}).Error
+}
+
 // backfillEmptyHostGroupIds is idempotent and not seeder-gated: builds that
 // predate group ids on the inbound-import path (and restored backups) can
 // re-introduce hosts rows with an empty group_id, and such rows render as a

internal/database/model/model.go

@@ -878,6 +878,8 @@ type Node struct {
 	// save that grows the selection zeroes it so reconcile waits before sweeping.
 	InboundsAdoptedAt int64 `json:"-" gorm:"column:inbounds_adopted_at;default:0"`
 
+	DownNotified bool `json:"-" gorm:"column:down_notified;default:false"`
+
 	InboundCount  int `json:"inboundCount" gorm:"-" example:"5"`
 	ClientCount   int `json:"clientCount" gorm:"-" example:"27"`
 	OnlineCount   int `json:"onlineCount" gorm:"-" example:"3"`

internal/database/node_down_notified_migration_test.go

@@ -0,0 +1,43 @@
+package database
+
+import (
+	"testing"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/config"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+func TestNodeDownNotifiedSeederMarksOfflineNodesAnnounced(t *testing.T) {
+	t.Setenv("XUI_DB_FOLDER", t.TempDir())
+	if err := InitDB(config.GetDBPath()); err != nil {
+		t.Fatalf("init db: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+	if err := db.Where("seeder_name = ?", "NodeDownNotified").Delete(&model.HistoryOfSeeders{}).Error; err != nil {
+		t.Fatalf("clear seeder history: %v", err)
+	}
+	offline := model.Node{Name: "offline", Address: "10.0.0.1", Port: 1, Status: "offline"}
+	online := model.Node{Name: "online", Address: "10.0.0.2", Port: 1, Status: "online"}
+	for _, n := range []*model.Node{&offline, &online} {
+		if err := db.Create(n).Error; err != nil {
+			t.Fatalf("create node %s: %v", n.Name, err)
+		}
+	}
+
+	if err := runSeeders(false); err != nil {
+		t.Fatalf("runSeeders: %v", err)
+	}
+
+	for _, tc := range []struct {
+		node *model.Node
+		want bool
+	}{{&offline, true}, {&online, false}} {
+		var got model.Node
+		if err := db.First(&got, tc.node.Id).Error; err != nil {
+			t.Fatalf("reload node %s: %v", tc.node.Name, err)
+		}
+		if got.DownNotified != tc.want {
+			t.Errorf("node %s DownNotified = %v, want %v", tc.node.Name, got.DownNotified, tc.want)
+		}
+	}
+}

internal/web/entity/entity.go

@@ -80,6 +80,7 @@ type AllSetting struct {
 	DiscordEnabledEvents string `json:"discordEnabledEvents" form:"discordEnabledEvents"`
 
 	OutboundDownThreshold int `json:"outboundDownThreshold" form:"outboundDownThreshold" validate:"gte=1,lte=100"`
+	NodeDownThreshold     int `json:"nodeDownThreshold" form:"nodeDownThreshold" validate:"gte=1,lte=100"`
 
 	TimeLocation    string `json:"timeLocation" form:"timeLocation"`
 	TwoFactorEnable bool   `json:"twoFactorEnable" form:"twoFactorEnable"`

internal/web/job/node_down_threshold_test.go

@@ -0,0 +1,118 @@
+package job
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/op/go-logging"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
+)
+
+func switchableNode(t *testing.T) *atomic.Bool {
+	t.Helper()
+	xuilogger.InitLogger(logging.ERROR)
+	dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db"))
+	runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}}))
+	t.Cleanup(func() { runtime.SetManager(nil) })
+
+	var up atomic.Bool
+	up.Store(true)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !up.Load() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "server/status") {
+			_, _ = w.Write([]byte(`{"success":true,"obj":{"panelGuid":"flaky-guid","xray":{"state":"running"}}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"success":true,"obj":[]}`))
+	}))
+	t.Cleanup(srv.Close)
+	host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
+	portNum, _ := strconv.Atoi(port)
+	node := &model.Node{
+		Name: "flaky", Scheme: "http", Address: host, Port: portNum, BasePath: "/", ApiToken: "tok",
+		Enable: true, Status: "online", AllowPrivateAddress: true, TlsVerifyMode: "verify",
+	}
+	if err := database.GetDB().Create(node).Error; err != nil {
+		t.Fatalf("create node: %v", err)
+	}
+	return &up
+}
+
+func TestHeartbeatHoldsNodeDownUntilThreshold(t *testing.T) {
+	cases := []struct {
+		name          string
+		probes        []bool
+		restartBefore int
+		want          []string
+	}{
+		{
+			name:   "flapping below threshold",
+			probes: []bool{false, true, false, false, true, false, true},
+		},
+		{
+			name:   "outage reaching threshold",
+			probes: []bool{false, false, false, false, true},
+			want:   []string{"probe 3: node.down", "probe 5: node.up"},
+		},
+		{
+			name:          "restart mid-streak still announces the outage",
+			probes:        []bool{false, false, false, false, true},
+			restartBefore: 2,
+			want:          []string{"probe 4: node.down", "probe 5: node.up"},
+		},
+		{
+			name:          "restart after announced outage does not repeat it",
+			probes:        []bool{false, false, false, false, false, false, true},
+			restartBefore: 5,
+			want:          []string{"probe 3: node.down", "probe 7: node.up"},
+		},
+		{
+			name:          "restart after recovery announces the next outage",
+			probes:        []bool{false, false, false, true, false, false, false, true},
+			restartBefore: 5,
+			want:          []string{"probe 3: node.down", "probe 4: node.up", "probe 7: node.down", "probe 8: node.up"},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			up := switchableNode(t)
+			if err := (&service.SettingService{}).SetNodeDownThreshold(3); err != nil {
+				t.Fatalf("SetNodeDownThreshold: %v", err)
+			}
+			events := collectNodeEvents(t)
+			job := NewNodeHeartbeatJob()
+
+			var got []string
+			for i, probeOK := range tc.probes {
+				if i+1 == tc.restartBefore {
+					job = NewNodeHeartbeatJob()
+				}
+				up.Store(probeOK)
+				job.Run()
+				for _, e := range events()[len(got):] {
+					got = append(got, fmt.Sprintf("probe %d: %s", i+1, e.Type))
+				}
+			}
+
+			if strings.Join(got, ", ") != strings.Join(tc.want, ", ") {
+				t.Fatalf("notifications = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

internal/web/job/node_heartbeat_job.go

@@ -27,8 +27,20 @@ const (
 )
 
 type NodeHeartbeatJob struct {
-	nodeService service.NodeService
-	running     sync.Mutex
+	nodeService    service.NodeService
+	settingService service.SettingService
+	running        sync.Mutex
+	health         map[int]nodeHealth
+}
+
+type nodeHealth struct {
+	failStreak int
+	notified   bool
+}
+
+type nodeProbe struct {
+	node  *model.Node
+	patch service.HeartbeatPatch
 }
 
 func NewNodeHeartbeatJob() *NodeHeartbeatJob {
@@ -53,8 +65,8 @@ func (j *NodeHeartbeatJob) Run() {
 
 	sem := make(chan struct{}, nodeHeartbeatConcurrency)
 	var wg sync.WaitGroup
-	var transitionsMu sync.Mutex
-	var transitions []eventbus.Event
+	var probesMu sync.Mutex
+	var probes []nodeProbe
 	for _, n := range nodes {
 		if !n.Enable {
 			continue
@@ -65,15 +77,14 @@ func (j *NodeHeartbeatJob) Run() {
 		common.GoRecover("node-heartbeat:"+n.Name, func() {
 			defer wg.Done()
 			defer func() { <-sem }()
-			if event := j.probeOne(n); event != nil {
-				transitionsMu.Lock()
-				transitions = append(transitions, *event)
-				transitionsMu.Unlock()
-			}
+			patch := j.probeOne(n)
+			probesMu.Lock()
+			probes = append(probes, nodeProbe{node: n, patch: patch})
+			probesMu.Unlock()
 		})
 	}
 	wg.Wait()
-	publishNodeTransitions(transitions)
+	publishNodeTransitions(j.healthTransitions(probes))
 
 	if !websocket.HasClients() {
 		return
@@ -86,10 +97,9 @@ func (j *NodeHeartbeatJob) Run() {
 	websocket.BroadcastNodes(updated)
 }
 
-func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event {
+func (j *NodeHeartbeatJob) probeOne(n *model.Node) service.HeartbeatPatch {
 	ctx, cancel := context.WithTimeout(context.Background(), nodeHeartbeatRequestTimeout)
 	defer cancel()
-	prevStatus := n.Status
 	patch, err := j.nodeService.Probe(ctx, n)
 	if err != nil {
 		patch.Status = "offline"
@@ -109,26 +119,53 @@ func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event {
 	} else {
 		j.nodeService.ClearDescendants(n.Id)
 	}
-	return nodeTransitionEvent(n, prevStatus, patch)
+	return patch
 }
 
-// nodeTransitionEvent is node.down / node.up on a genuine state change only; an unknown
-// previous status (fresh start) counts as not-online, so it never yields node.down.
-func nodeTransitionEvent(n *model.Node, prevStatus string, patch service.HeartbeatPatch) *eventbus.Event {
-	var eventType eventbus.EventType
-	switch {
-	case prevStatus == "online" && patch.Status == "offline":
-		eventType = eventbus.EventNodeDown
-	case prevStatus != "online" && patch.Status == "online":
-		eventType = eventbus.EventNodeUp
-	default:
-		return nil
+func (j *NodeHeartbeatJob) healthTransitions(probes []nodeProbe) []eventbus.Event {
+	threshold := 1
+	if v, err := j.settingService.GetNodeDownThreshold(); err == nil && v > 0 {
+		threshold = v
 	}
+	prev := j.health
+	j.health = make(map[int]nodeHealth, len(probes))
+	var events []eventbus.Event
+	for _, p := range probes {
+		h, seen := prev[p.node.Id]
+		if !seen {
+			h.notified = p.node.DownNotified
+		}
+		if p.patch.Status == "online" {
+			if h.notified {
+				events = append(events, nodeHealthEvent(eventbus.EventNodeUp, p.node, p.patch))
+				j.saveDownNotified(p.node.Id, false)
+			}
+			h = nodeHealth{}
+		} else {
+			h.failStreak++
+			if h.failStreak >= threshold && !h.notified {
+				events = append(events, nodeHealthEvent(eventbus.EventNodeDown, p.node, p.patch))
+				h.notified = true
+				j.saveDownNotified(p.node.Id, true)
+			}
+		}
+		j.health[p.node.Id] = h
+	}
+	return events
+}
+
+func (j *NodeHeartbeatJob) saveDownNotified(id int, notified bool) {
+	if err := j.nodeService.SetDownNotified(id, notified); err != nil {
+		logger.Warning("node heartbeat: save down-notified flag for node", id, "failed:", err)
+	}
+}
+
+func nodeHealthEvent(eventType eventbus.EventType, n *model.Node, patch service.HeartbeatPatch) eventbus.Event {
 	source := n.Name
 	if source == "" {
 		source = "node-" + strconv.Itoa(n.Id)
 	}
-	return &eventbus.Event{
+	return eventbus.Event{
 		Type:   eventType,
 		Source: source,
 		Data: &eventbus.NodeHealthData{

internal/web/job/node_transition_burst_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
 	xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
 )
 
 // goingDownNodes seeds n online nodes whose address refuses connections, so the
@@ -28,6 +29,9 @@ func goingDownNodes(t *testing.T, n int) {
 	dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db"))
 	runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}}))
 	t.Cleanup(func() { runtime.SetManager(nil) })
+	if err := (&service.SettingService{}).SetNodeDownThreshold(1); err != nil {
+		t.Fatalf("SetNodeDownThreshold: %v", err)
+	}
 	srv := httptest.NewServer(nil)
 	host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
 	portNum, _ := strconv.Atoi(port)

internal/web/service/node.go

@@ -1064,6 +1064,12 @@ func (s *NodeService) ClearNodeDirty(id int, dirtyAt int64) error {
 		Update("config_dirty", false).Error
 }
 
+func (s *NodeService) SetDownNotified(id int, notified bool) error {
+	return database.GetDB().Model(model.Node{}).
+		Where("id = ?", id).
+		Update("down_notified", notified).Error
+}
+
 func (s *NodeService) MarkNodeInboundsAdopted(id int) error {
 	if id <= 0 {
 		return nil

internal/web/service/setting.go

@@ -238,6 +238,8 @@ var defaultValueMap = map[string]string{
 	// Consecutive failed observatory probes before an outbound.down event fires
 	"outboundDownThreshold": "3",
 
+	"nodeDownThreshold": "1",
+
 	// Email (SMTP) notifications
 	"smtpEnable":         "false",
 	"smtpHost":           "",
@@ -1641,6 +1643,14 @@ func (s *SettingService) SetOutboundDownThreshold(value int) error {
 	return s.setInt("outboundDownThreshold", value)
 }
 
+func (s *SettingService) GetNodeDownThreshold() (int, error) {
+	return s.getInt("nodeDownThreshold")
+}
+
+func (s *SettingService) SetNodeDownThreshold(value int) error {
+	return s.setInt("nodeDownThreshold", value)
+}
+
 // SecretClears marks redacted secrets the user explicitly emptied. Without a
 // flag, a blank submitted secret means "unchanged" (the field is always served
 // blank to the browser) and the stored value is preserved.

internal/web/translation/ar-EG.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "العقد",
       "eventOutboundDown": "غير متصل",
       "eventOutboundUp": "متصل",
+      "eventDownThresholdHint": "فحوصات فاشلة متتالية",
       "eventXrayCrash": "تعطّل",
       "eventNodeDown": "غير متصلة",
       "eventNodeUp": "متصلة",

internal/web/translation/en-US.json

@@ -1610,6 +1610,7 @@
       "eventGroupNode": "Nodes",
       "eventOutboundDown": "Down",
       "eventOutboundUp": "Up",
+      "eventDownThresholdHint": "failed checks in a row",
       "eventXrayCrash": "Crash",
       "eventNodeDown": "Down",
       "eventNodeUp": "Up",

internal/web/translation/es-ES.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Nodos",
       "eventOutboundDown": "Caído",
       "eventOutboundUp": "Activo",
+      "eventDownThresholdHint": "comprobaciones fallidas seguidas",
       "eventXrayCrash": "Caída",
       "eventNodeDown": "Caído",
       "eventNodeUp": "Activo",

internal/web/translation/fa-IR.json

@@ -1492,6 +1492,7 @@
       "eventGroupNode": "نودها",
       "eventOutboundDown": "قطع",
       "eventOutboundUp": "وصل",
+      "eventDownThresholdHint": "بررسی ناموفق پیاپی",
       "eventXrayCrash": "کرش",
       "eventNodeDown": "قطع",
       "eventNodeUp": "وصل",

internal/web/translation/id-ID.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Node",
       "eventOutboundDown": "Mati",
       "eventOutboundUp": "Aktif",
+      "eventDownThresholdHint": "pemeriksaan gagal berturut-turut",
       "eventXrayCrash": "Crash",
       "eventNodeDown": "Mati",
       "eventNodeUp": "Aktif",

internal/web/translation/ja-JP.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "ノード",
       "eventOutboundDown": "ダウン",
       "eventOutboundUp": "アップ",
+      "eventDownThresholdHint": "回連続でチェック失敗",
       "eventXrayCrash": "クラッシュ",
       "eventNodeDown": "ダウン",
       "eventNodeUp": "アップ",

internal/web/translation/pt-BR.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Nós",
       "eventOutboundDown": "Inativo",
       "eventOutboundUp": "Ativo",
+      "eventDownThresholdHint": "verificações com falha seguidas",
       "eventXrayCrash": "Falha",
       "eventNodeDown": "Inativo",
       "eventNodeUp": "Ativo",

internal/web/translation/ru-RU.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Узлы",
       "eventOutboundDown": "Недоступен",
       "eventOutboundUp": "Работает",
+      "eventDownThresholdHint": "неудачных проверок подряд",
       "eventXrayCrash": "Сбой",
       "eventNodeDown": "Недоступен",
       "eventNodeUp": "В сети",

internal/web/translation/tr-TR.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Düğümler",
       "eventOutboundDown": "Çevrimdışı",
       "eventOutboundUp": "Çevrimiçi",
+      "eventDownThresholdHint": "art arda başarısız kontrol",
       "eventXrayCrash": "Çökme",
       "eventNodeDown": "Çevrimdışı",
       "eventNodeUp": "Çevrimiçi",

internal/web/translation/uk-UA.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Вузли",
       "eventOutboundDown": "Недоступне",
       "eventOutboundUp": "Доступне",
+      "eventDownThresholdHint": "невдалих перевірок поспіль",
       "eventXrayCrash": "Збій",
       "eventNodeDown": "Недоступний",
       "eventNodeUp": "Доступний",

internal/web/translation/vi-VN.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Node",
       "eventOutboundDown": "Ngừng hoạt động",
       "eventOutboundUp": "Hoạt động",
+      "eventDownThresholdHint": "lần kiểm tra thất bại liên tiếp",
       "eventXrayCrash": "Sự cố",
       "eventNodeDown": "Ngừng hoạt động",
       "eventNodeUp": "Hoạt động",

internal/web/translation/zh-CN.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "节点",
       "eventOutboundDown": "断开",
       "eventOutboundUp": "恢复",
+      "eventDownThresholdHint": "次连续检查失败",
       "eventXrayCrash": "崩溃",
       "eventNodeDown": "离线",
       "eventNodeUp": "上线",

internal/web/translation/zh-TW.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "節點",
       "eventOutboundDown": "中斷",
       "eventOutboundUp": "恢復",
+      "eventDownThresholdHint": "次連續檢查失敗",
       "eventXrayCrash": "當機",
       "eventNodeDown": "離線",
       "eventNodeUp": "上線",