feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.

internal/database/db.go

@@ -1475,7 +1475,7 @@ func runSeeders(isUsersEmpty bool) error {
 	}
 
 	if empty && isUsersEmpty {
-		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "ResetIpLimitNoFail2ban"}
+		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "NodeDownNotified", "ResetIpLimitNoFail2ban"}
 		for _, name := range seeders {
 			if err := db.Create(&model.HistoryOfSeeders{SeederName: name}).Error; err != nil {
 				return err
@@ -1640,6 +1640,12 @@ func runSeeders(isUsersEmpty bool) error {
 		}
 	}
 
+	if !slices.Contains(seedersHistory, "NodeDownNotified") {
+		if err := seedNodeDownNotified(); err != nil {
+			return err
+		}
+	}
+
 	if err := seedHostsFromExternalProxy(); err != nil {
 		return err
 	}
@@ -1689,6 +1695,17 @@ func seedNodeInboundsAdopted() error {
 	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeInboundsAdopted"}).Error
 }
 
+// seedNodeDownNotified treats nodes already offline at upgrade as announced,
+// so the first heartbeat after it does not re-send their node.down.
+func seedNodeDownNotified() error {
+	if err := db.Model(&model.Node{}).
+		Where("status = ?", "offline").
+		Update("down_notified", true).Error; err != nil {
+		return err
+	}
+	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeDownNotified"}).Error
+}
+
 // backfillEmptyHostGroupIds is idempotent and not seeder-gated: builds that
 // predate group ids on the inbound-import path (and restored backups) can
 // re-introduce hosts rows with an empty group_id, and such rows render as a

internal/database/model/model.go

@@ -878,6 +878,8 @@ type Node struct {
 	// save that grows the selection zeroes it so reconcile waits before sweeping.
 	InboundsAdoptedAt int64 `json:"-" gorm:"column:inbounds_adopted_at;default:0"`
 
+	DownNotified bool `json:"-" gorm:"column:down_notified;default:false"`
+
 	InboundCount  int `json:"inboundCount" gorm:"-" example:"5"`
 	ClientCount   int `json:"clientCount" gorm:"-" example:"27"`
 	OnlineCount   int `json:"onlineCount" gorm:"-" example:"3"`

internal/database/node_down_notified_migration_test.go

@@ -0,0 +1,43 @@
+package database
+
+import (
+	"testing"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/config"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+func TestNodeDownNotifiedSeederMarksOfflineNodesAnnounced(t *testing.T) {
+	t.Setenv("XUI_DB_FOLDER", t.TempDir())
+	if err := InitDB(config.GetDBPath()); err != nil {
+		t.Fatalf("init db: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+	if err := db.Where("seeder_name = ?", "NodeDownNotified").Delete(&model.HistoryOfSeeders{}).Error; err != nil {
+		t.Fatalf("clear seeder history: %v", err)
+	}
+	offline := model.Node{Name: "offline", Address: "10.0.0.1", Port: 1, Status: "offline"}
+	online := model.Node{Name: "online", Address: "10.0.0.2", Port: 1, Status: "online"}
+	for _, n := range []*model.Node{&offline, &online} {
+		if err := db.Create(n).Error; err != nil {
+			t.Fatalf("create node %s: %v", n.Name, err)
+		}
+	}
+
+	if err := runSeeders(false); err != nil {
+		t.Fatalf("runSeeders: %v", err)
+	}
+
+	for _, tc := range []struct {
+		node *model.Node
+		want bool
+	}{{&offline, true}, {&online, false}} {
+		var got model.Node
+		if err := db.First(&got, tc.node.Id).Error; err != nil {
+			t.Fatalf("reload node %s: %v", tc.node.Name, err)
+		}
+		if got.DownNotified != tc.want {
+			t.Errorf("node %s DownNotified = %v, want %v", tc.node.Name, got.DownNotified, tc.want)
+		}
+	}
+}