feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.

docs/public/openapi.json

@@ -152,6 +152,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "nodeDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "outboundDownThreshold": {
             "maximum": 100,
             "minimum": 1,
@@ -639,6 +644,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "nodeDownThreshold",
           "outboundDownThreshold",
           "pageSize",
           "panelOutbound",
@@ -932,6 +938,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "nodeDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "outboundDownThreshold": {
             "maximum": 100,
             "minimum": 1,
@@ -1427,6 +1438,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "nodeDownThreshold",
           "outboundDownThreshold",
           "pageSize",
           "panelOutbound",

frontend/public/openapi.json

@@ -152,6 +152,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "nodeDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "outboundDownThreshold": {
             "maximum": 100,
             "minimum": 1,
@@ -639,6 +644,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "nodeDownThreshold",
           "outboundDownThreshold",
           "pageSize",
           "panelOutbound",
@@ -932,6 +938,11 @@
           "ldapVlessField": {
             "type": "string"
           },
+          "nodeDownThreshold": {
+            "maximum": 100,
+            "minimum": 1,
+            "type": "integer"
+          },
           "outboundDownThreshold": {
             "maximum": 100,
             "minimum": 1,
@@ -1427,6 +1438,7 @@
           "ldapUserAttr",
           "ldapUserFilter",
           "ldapVlessField",
+          "nodeDownThreshold",
           "outboundDownThreshold",
           "pageSize",
           "panelOutbound",

frontend/src/components/ui/notifications/EmailNotifications.tsx

@@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [
         key: 'outbound.down',
         label: 'eventOutboundDown',
         settingKey: 'outboundDownThreshold',
+        hint: 'eventDownThresholdHint',
         extra: ({ value, onChange, ariaLabel }) => (
           <InputNumber
             size="small"
@@ -44,7 +45,23 @@ const GROUPS: NotificationGroupConfig[] = [
     icon: <DesktopOutlined aria-hidden />,
     title: 'eventGroupNode',
     events: [
-      { key: 'node.down', label: 'eventNodeDown', settingKey: '' },
+      {
+        key: 'node.down',
+        label: 'eventNodeDown',
+        settingKey: 'nodeDownThreshold',
+        hint: 'eventDownThresholdHint',
+        extra: ({ value, onChange, ariaLabel }) => (
+          <InputNumber
+            size="small"
+            min={1}
+            max={100}
+            value={value}
+            onChange={onChange}
+            aria-label={ariaLabel}
+            style={{ width: 80 }}
+          />
+        ),
+      },
       { key: 'node.up', label: 'eventNodeUp', settingKey: '' },
     ],
   },

frontend/src/components/ui/notifications/NotificationGroup.tsx

@@ -1,4 +1,4 @@
-import { Space } from 'antd';
+import { Space, Typography } from 'antd';
 import { useTranslation } from 'react-i18next';
 import type { AllSetting } from '@/models/setting';
 import type { NotificationGroupConfig } from './types';
@@ -55,12 +55,22 @@ export function NotificationGroup({
             checked={selected.includes(event.key)}
             onToggle={() => onToggle(event.key)}
           >
-            {event.extra?.({
+            {event.extra && (
-              value:
+              <Space size={8}>
-                Number((allSetting as unknown as Record<string, unknown>)[event.settingKey]) || 0,
+                {event.extra({
-              onChange: (v) => updateSetting({ [event.settingKey]: v }),
+                  value:
-              ariaLabel: t(`pages.settings.${event.label}`),
+                    Number((allSetting as unknown as Record<string, unknown>)[event.settingKey]) ||
-            })}
+                    0,
+                  onChange: (v) => updateSetting({ [event.settingKey]: v }),
+                  ariaLabel: t(`pages.settings.${event.label}`),
+                })}
+                {event.hint && (
+                  <Typography.Text type="secondary">
+                    {t(`pages.settings.${event.hint}`)}
+                  </Typography.Text>
+                )}
+              </Space>
+            )}
           </NotificationEvent>
         ))}
       </Space>

frontend/src/components/ui/notifications/TelegramNotifications.tsx

@@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [
         key: 'outbound.down',
         label: 'eventOutboundDown',
         settingKey: 'outboundDownThreshold',
+        hint: 'eventDownThresholdHint',
         extra: ({ value, onChange, ariaLabel }) => (
           <InputNumber
             size="small"
@@ -44,7 +45,23 @@ const GROUPS: NotificationGroupConfig[] = [
     icon: <DesktopOutlined aria-hidden />,
     title: 'eventGroupNode',
     events: [
-      { key: 'node.down', label: 'eventNodeDown', settingKey: '' },
+      {
+        key: 'node.down',
+        label: 'eventNodeDown',
+        settingKey: 'nodeDownThreshold',
+        hint: 'eventDownThresholdHint',
+        extra: ({ value, onChange, ariaLabel }) => (
+          <InputNumber
+            size="small"
+            min={1}
+            max={100}
+            value={value}
+            onChange={onChange}
+            aria-label={ariaLabel}
+            style={{ width: 80 }}
+          />
+        ),
+      },
       { key: 'node.up', label: 'eventNodeUp', settingKey: '' },
     ],
   },

frontend/src/components/ui/notifications/types.ts

@@ -4,6 +4,7 @@ export interface NotificationEventConfig {
   key: string;
   label: string;
   settingKey: string;
+  hint?: string;
   extra?: (props: {
     value: number;
     onChange: (v: number | null) => void;

frontend/src/generated/examples.ts

@@ -39,6 +39,7 @@ export const EXAMPLES: Record<string, unknown> = {
     "ldapUserAttr": "",
     "ldapUserFilter": "",
     "ldapVlessField": "",
+    "nodeDownThreshold": 1,
     "outboundDownThreshold": 1,
     "pageSize": 0,
     "panelOutbound": "",
@@ -227,6 +228,7 @@ export const EXAMPLES: Record<string, unknown> = {
     "ldapUserAttr": "",
     "ldapUserFilter": "",
     "ldapVlessField": "",
+    "nodeDownThreshold": 1,
     "outboundDownThreshold": 1,
     "pageSize": 0,
     "panelOutbound": "",

frontend/src/generated/schemas.ts

@@ -126,6 +126,11 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapVlessField": {
         "type": "string"
       },
+      "nodeDownThreshold": {
+        "maximum": 100,
+        "minimum": 1,
+        "type": "integer"
+      },
       "outboundDownThreshold": {
         "maximum": 100,
         "minimum": 1,
@@ -613,6 +618,7 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapUserAttr",
       "ldapUserFilter",
       "ldapVlessField",
+      "nodeDownThreshold",
       "outboundDownThreshold",
       "pageSize",
       "panelOutbound",
@@ -906,6 +912,11 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapVlessField": {
         "type": "string"
       },
+      "nodeDownThreshold": {
+        "maximum": 100,
+        "minimum": 1,
+        "type": "integer"
+      },
       "outboundDownThreshold": {
         "maximum": 100,
         "minimum": 1,
@@ -1401,6 +1412,7 @@ export const SCHEMAS: Record<string, unknown> = {
       "ldapUserAttr",
       "ldapUserFilter",
       "ldapVlessField",
+      "nodeDownThreshold",
       "outboundDownThreshold",
       "pageSize",
       "panelOutbound",

frontend/src/generated/types.ts

@@ -47,6 +47,7 @@ export interface AllSetting {
   ldapUserAttr: string;
   ldapUserFilter: string;
   ldapVlessField: string;
+  nodeDownThreshold: number;
   outboundDownThreshold: number;
   pageSize: number;
   panelOutbound: string;
@@ -236,6 +237,7 @@ export interface AllSettingView {
   ldapUserAttr: string;
   ldapUserFilter: string;
   ldapVlessField: string;
+  nodeDownThreshold: number;
   outboundDownThreshold: number;
   pageSize: number;
   panelOutbound: string;

frontend/src/generated/zod.ts

@@ -63,6 +63,7 @@ export const AllSettingSchema = z.object({
   ldapUserAttr: z.string(),
   ldapUserFilter: z.string(),
   ldapVlessField: z.string(),
+  nodeDownThreshold: z.number().int().min(1).max(100),
   outboundDownThreshold: z.number().int().min(1).max(100),
   pageSize: z.number().int().min(0).max(1000),
   panelOutbound: z.string(),
@@ -253,6 +254,7 @@ export const AllSettingViewSchema = z.object({
   ldapUserAttr: z.string(),
   ldapUserFilter: z.string(),
   ldapVlessField: z.string(),
+  nodeDownThreshold: z.number().int().min(1).max(100),
   outboundDownThreshold: z.number().int().min(1).max(100),
   pageSize: z.number().int().min(0).max(1000),
   panelOutbound: z.string(),

frontend/src/models/setting.ts

@@ -174,6 +174,7 @@ export class AllSetting {
   smtpCpu = 80;
   smtpMemory = 80;
   outboundDownThreshold = 3;
+  nodeDownThreshold = 1;
   hasTgBotToken = false;
   hasTwoFactorToken = false;
   hasLdapPassword = false;
@@ -216,6 +217,10 @@ export class AllSetting {
     this.outboundDownThreshold = Number.isFinite(threshold)
       ? Math.min(100, Math.max(1, threshold))
       : 3;
+    const nodeThreshold = Math.round(Number(this.nodeDownThreshold));
+    this.nodeDownThreshold = Number.isFinite(nodeThreshold)
+      ? Math.min(100, Math.max(1, nodeThreshold))
+      : 1;
   }
 
   equals(other: AllSetting): boolean {

frontend/src/schemas/setting.ts

@@ -39,6 +39,7 @@ export const AllSettingSchema = z
     tgBotBackup: z.boolean().optional(),
     tgCpu: z.number().int().min(0).max(100).optional(),
     outboundDownThreshold: z.number().int().min(1).max(100).optional(),
+    nodeDownThreshold: z.number().int().min(1).max(100).optional(),
     tgLang: z.string().optional(),
     twoFactorEnable: z.boolean().optional(),
     twoFactorToken: z.string().optional(),

internal/database/db.go

@@ -1475,7 +1475,7 @@ func runSeeders(isUsersEmpty bool) error {
 	}
 
 	if empty && isUsersEmpty {
-		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "ResetIpLimitNoFail2ban"}
+		seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "NodeDownNotified", "ResetIpLimitNoFail2ban"}
 		for _, name := range seeders {
 			if err := db.Create(&model.HistoryOfSeeders{SeederName: name}).Error; err != nil {
 				return err
@@ -1640,6 +1640,12 @@ func runSeeders(isUsersEmpty bool) error {
 		}
 	}
 
+	if !slices.Contains(seedersHistory, "NodeDownNotified") {
+		if err := seedNodeDownNotified(); err != nil {
+			return err
+		}
+	}
+
 	if err := seedHostsFromExternalProxy(); err != nil {
 		return err
 	}
@@ -1689,6 +1695,17 @@ func seedNodeInboundsAdopted() error {
 	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeInboundsAdopted"}).Error
 }
 
+// seedNodeDownNotified treats nodes already offline at upgrade as announced,
+// so the first heartbeat after it does not re-send their node.down.
+func seedNodeDownNotified() error {
+	if err := db.Model(&model.Node{}).
+		Where("status = ?", "offline").
+		Update("down_notified", true).Error; err != nil {
+		return err
+	}
+	return db.Create(&model.HistoryOfSeeders{SeederName: "NodeDownNotified"}).Error
+}
+
 // backfillEmptyHostGroupIds is idempotent and not seeder-gated: builds that
 // predate group ids on the inbound-import path (and restored backups) can
 // re-introduce hosts rows with an empty group_id, and such rows render as a

internal/database/model/model.go

@@ -878,6 +878,8 @@ type Node struct {
 	// save that grows the selection zeroes it so reconcile waits before sweeping.
 	InboundsAdoptedAt int64 `json:"-" gorm:"column:inbounds_adopted_at;default:0"`
 
+	DownNotified bool `json:"-" gorm:"column:down_notified;default:false"`
+
 	InboundCount  int `json:"inboundCount" gorm:"-" example:"5"`
 	ClientCount   int `json:"clientCount" gorm:"-" example:"27"`
 	OnlineCount   int `json:"onlineCount" gorm:"-" example:"3"`

internal/database/node_down_notified_migration_test.go

@@ -0,0 +1,43 @@
+package database
+
+import (
+	"testing"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/config"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+)
+
+func TestNodeDownNotifiedSeederMarksOfflineNodesAnnounced(t *testing.T) {
+	t.Setenv("XUI_DB_FOLDER", t.TempDir())
+	if err := InitDB(config.GetDBPath()); err != nil {
+		t.Fatalf("init db: %v", err)
+	}
+	t.Cleanup(func() { _ = CloseDB() })
+	if err := db.Where("seeder_name = ?", "NodeDownNotified").Delete(&model.HistoryOfSeeders{}).Error; err != nil {
+		t.Fatalf("clear seeder history: %v", err)
+	}
+	offline := model.Node{Name: "offline", Address: "10.0.0.1", Port: 1, Status: "offline"}
+	online := model.Node{Name: "online", Address: "10.0.0.2", Port: 1, Status: "online"}
+	for _, n := range []*model.Node{&offline, &online} {
+		if err := db.Create(n).Error; err != nil {
+			t.Fatalf("create node %s: %v", n.Name, err)
+		}
+	}
+
+	if err := runSeeders(false); err != nil {
+		t.Fatalf("runSeeders: %v", err)
+	}
+
+	for _, tc := range []struct {
+		node *model.Node
+		want bool
+	}{{&offline, true}, {&online, false}} {
+		var got model.Node
+		if err := db.First(&got, tc.node.Id).Error; err != nil {
+			t.Fatalf("reload node %s: %v", tc.node.Name, err)
+		}
+		if got.DownNotified != tc.want {
+			t.Errorf("node %s DownNotified = %v, want %v", tc.node.Name, got.DownNotified, tc.want)
+		}
+	}
+}

internal/web/entity/entity.go

@@ -80,6 +80,7 @@ type AllSetting struct {
 	DiscordEnabledEvents string `json:"discordEnabledEvents" form:"discordEnabledEvents"`
 
 	OutboundDownThreshold int `json:"outboundDownThreshold" form:"outboundDownThreshold" validate:"gte=1,lte=100"`
+	NodeDownThreshold     int `json:"nodeDownThreshold" form:"nodeDownThreshold" validate:"gte=1,lte=100"`
 
 	TimeLocation    string `json:"timeLocation" form:"timeLocation"`
 	TwoFactorEnable bool   `json:"twoFactorEnable" form:"twoFactorEnable"`

internal/web/job/node_down_threshold_test.go

@@ -0,0 +1,118 @@
+package job
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/op/go-logging"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/database"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
+	"github.com/mhsanaei/3x-ui/v3/internal/database/model"
+	xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
+)
+
+func switchableNode(t *testing.T) *atomic.Bool {
+	t.Helper()
+	xuilogger.InitLogger(logging.ERROR)
+	dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db"))
+	runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}}))
+	t.Cleanup(func() { runtime.SetManager(nil) })
+
+	var up atomic.Bool
+	up.Store(true)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !up.Load() {
+			w.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "server/status") {
+			_, _ = w.Write([]byte(`{"success":true,"obj":{"panelGuid":"flaky-guid","xray":{"state":"running"}}}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{"success":true,"obj":[]}`))
+	}))
+	t.Cleanup(srv.Close)
+	host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
+	portNum, _ := strconv.Atoi(port)
+	node := &model.Node{
+		Name: "flaky", Scheme: "http", Address: host, Port: portNum, BasePath: "/", ApiToken: "tok",
+		Enable: true, Status: "online", AllowPrivateAddress: true, TlsVerifyMode: "verify",
+	}
+	if err := database.GetDB().Create(node).Error; err != nil {
+		t.Fatalf("create node: %v", err)
+	}
+	return &up
+}
+
+func TestHeartbeatHoldsNodeDownUntilThreshold(t *testing.T) {
+	cases := []struct {
+		name          string
+		probes        []bool
+		restartBefore int
+		want          []string
+	}{
+		{
+			name:   "flapping below threshold",
+			probes: []bool{false, true, false, false, true, false, true},
+		},
+		{
+			name:   "outage reaching threshold",
+			probes: []bool{false, false, false, false, true},
+			want:   []string{"probe 3: node.down", "probe 5: node.up"},
+		},
+		{
+			name:          "restart mid-streak still announces the outage",
+			probes:        []bool{false, false, false, false, true},
+			restartBefore: 2,
+			want:          []string{"probe 4: node.down", "probe 5: node.up"},
+		},
+		{
+			name:          "restart after announced outage does not repeat it",
+			probes:        []bool{false, false, false, false, false, false, true},
+			restartBefore: 5,
+			want:          []string{"probe 3: node.down", "probe 7: node.up"},
+		},
+		{
+			name:          "restart after recovery announces the next outage",
+			probes:        []bool{false, false, false, true, false, false, false, true},
+			restartBefore: 5,
+			want:          []string{"probe 3: node.down", "probe 4: node.up", "probe 7: node.down", "probe 8: node.up"},
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			up := switchableNode(t)
+			if err := (&service.SettingService{}).SetNodeDownThreshold(3); err != nil {
+				t.Fatalf("SetNodeDownThreshold: %v", err)
+			}
+			events := collectNodeEvents(t)
+			job := NewNodeHeartbeatJob()
+
+			var got []string
+			for i, probeOK := range tc.probes {
+				if i+1 == tc.restartBefore {
+					job = NewNodeHeartbeatJob()
+				}
+				up.Store(probeOK)
+				job.Run()
+				for _, e := range events()[len(got):] {
+					got = append(got, fmt.Sprintf("probe %d: %s", i+1, e.Type))
+				}
+			}
+
+			if strings.Join(got, ", ") != strings.Join(tc.want, ", ") {
+				t.Fatalf("notifications = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

internal/web/job/node_heartbeat_job.go

@@ -27,8 +27,20 @@ const (
 )
 
 type NodeHeartbeatJob struct {
-	nodeService service.NodeService
+	nodeService    service.NodeService
-	running     sync.Mutex
+	settingService service.SettingService
+	running        sync.Mutex
+	health         map[int]nodeHealth
+}
+
+type nodeHealth struct {
+	failStreak int
+	notified   bool
+}
+
+type nodeProbe struct {
+	node  *model.Node
+	patch service.HeartbeatPatch
 }
 
 func NewNodeHeartbeatJob() *NodeHeartbeatJob {
@@ -53,8 +65,8 @@ func (j *NodeHeartbeatJob) Run() {
 
 	sem := make(chan struct{}, nodeHeartbeatConcurrency)
 	var wg sync.WaitGroup
-	var transitionsMu sync.Mutex
+	var probesMu sync.Mutex
-	var transitions []eventbus.Event
+	var probes []nodeProbe
 	for _, n := range nodes {
 		if !n.Enable {
 			continue
@@ -65,15 +77,14 @@ func (j *NodeHeartbeatJob) Run() {
 		common.GoRecover("node-heartbeat:"+n.Name, func() {
 			defer wg.Done()
 			defer func() { <-sem }()
-			if event := j.probeOne(n); event != nil {
+			patch := j.probeOne(n)
-				transitionsMu.Lock()
+			probesMu.Lock()
-				transitions = append(transitions, *event)
+			probes = append(probes, nodeProbe{node: n, patch: patch})
-				transitionsMu.Unlock()
+			probesMu.Unlock()
-			}
 		})
 	}
 	wg.Wait()
-	publishNodeTransitions(transitions)
+	publishNodeTransitions(j.healthTransitions(probes))
 
 	if !websocket.HasClients() {
 		return
@@ -86,10 +97,9 @@ func (j *NodeHeartbeatJob) Run() {
 	websocket.BroadcastNodes(updated)
 }
 
-func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event {
+func (j *NodeHeartbeatJob) probeOne(n *model.Node) service.HeartbeatPatch {
 	ctx, cancel := context.WithTimeout(context.Background(), nodeHeartbeatRequestTimeout)
 	defer cancel()
-	prevStatus := n.Status
 	patch, err := j.nodeService.Probe(ctx, n)
 	if err != nil {
 		patch.Status = "offline"
@@ -109,26 +119,53 @@ func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event {
 	} else {
 		j.nodeService.ClearDescendants(n.Id)
 	}
-	return nodeTransitionEvent(n, prevStatus, patch)
+	return patch
 }
 
-// nodeTransitionEvent is node.down / node.up on a genuine state change only; an unknown
+func (j *NodeHeartbeatJob) healthTransitions(probes []nodeProbe) []eventbus.Event {
-// previous status (fresh start) counts as not-online, so it never yields node.down.
+	threshold := 1
-func nodeTransitionEvent(n *model.Node, prevStatus string, patch service.HeartbeatPatch) *eventbus.Event {
+	if v, err := j.settingService.GetNodeDownThreshold(); err == nil && v > 0 {
-	var eventType eventbus.EventType
+		threshold = v
-	switch {
-	case prevStatus == "online" && patch.Status == "offline":
-		eventType = eventbus.EventNodeDown
-	case prevStatus != "online" && patch.Status == "online":
-		eventType = eventbus.EventNodeUp
-	default:
-		return nil
 	}
+	prev := j.health
+	j.health = make(map[int]nodeHealth, len(probes))
+	var events []eventbus.Event
+	for _, p := range probes {
+		h, seen := prev[p.node.Id]
+		if !seen {
+			h.notified = p.node.DownNotified
+		}
+		if p.patch.Status == "online" {
+			if h.notified {
+				events = append(events, nodeHealthEvent(eventbus.EventNodeUp, p.node, p.patch))
+				j.saveDownNotified(p.node.Id, false)
+			}
+			h = nodeHealth{}
+		} else {
+			h.failStreak++
+			if h.failStreak >= threshold && !h.notified {
+				events = append(events, nodeHealthEvent(eventbus.EventNodeDown, p.node, p.patch))
+				h.notified = true
+				j.saveDownNotified(p.node.Id, true)
+			}
+		}
+		j.health[p.node.Id] = h
+	}
+	return events
+}
+
+func (j *NodeHeartbeatJob) saveDownNotified(id int, notified bool) {
+	if err := j.nodeService.SetDownNotified(id, notified); err != nil {
+		logger.Warning("node heartbeat: save down-notified flag for node", id, "failed:", err)
+	}
+}
+
+func nodeHealthEvent(eventType eventbus.EventType, n *model.Node, patch service.HeartbeatPatch) eventbus.Event {
 	source := n.Name
 	if source == "" {
 		source = "node-" + strconv.Itoa(n.Id)
 	}
-	return &eventbus.Event{
+	return eventbus.Event{
 		Type:   eventType,
 		Source: source,
 		Data: &eventbus.NodeHealthData{

internal/web/job/node_transition_burst_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
 	xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger"
 	"github.com/mhsanaei/3x-ui/v3/internal/web/runtime"
+	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
 )
 
 // goingDownNodes seeds n online nodes whose address refuses connections, so the
@@ -28,6 +29,9 @@ func goingDownNodes(t *testing.T, n int) {
 	dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db"))
 	runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}}))
 	t.Cleanup(func() { runtime.SetManager(nil) })
+	if err := (&service.SettingService{}).SetNodeDownThreshold(1); err != nil {
+		t.Fatalf("SetNodeDownThreshold: %v", err)
+	}
 	srv := httptest.NewServer(nil)
 	host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
 	portNum, _ := strconv.Atoi(port)

internal/web/service/node.go

@@ -1064,6 +1064,12 @@ func (s *NodeService) ClearNodeDirty(id int, dirtyAt int64) error {
 		Update("config_dirty", false).Error
 }
 
+func (s *NodeService) SetDownNotified(id int, notified bool) error {
+	return database.GetDB().Model(model.Node{}).
+		Where("id = ?", id).
+		Update("down_notified", notified).Error
+}
+
 func (s *NodeService) MarkNodeInboundsAdopted(id int) error {
 	if id <= 0 {
 		return nil

internal/web/service/setting.go

@@ -238,6 +238,8 @@ var defaultValueMap = map[string]string{
 	// Consecutive failed observatory probes before an outbound.down event fires
 	"outboundDownThreshold": "3",
 
+	"nodeDownThreshold": "1",
+
 	// Email (SMTP) notifications
 	"smtpEnable":         "false",
 	"smtpHost":           "",
@@ -1641,6 +1643,14 @@ func (s *SettingService) SetOutboundDownThreshold(value int) error {
 	return s.setInt("outboundDownThreshold", value)
 }
 
+func (s *SettingService) GetNodeDownThreshold() (int, error) {
+	return s.getInt("nodeDownThreshold")
+}
+
+func (s *SettingService) SetNodeDownThreshold(value int) error {
+	return s.setInt("nodeDownThreshold", value)
+}
+
 // SecretClears marks redacted secrets the user explicitly emptied. Without a
 // flag, a blank submitted secret means "unchanged" (the field is always served
 // blank to the browser) and the stored value is preserved.

internal/web/translation/ar-EG.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "العقد",
       "eventOutboundDown": "غير متصل",
       "eventOutboundUp": "متصل",
+      "eventDownThresholdHint": "فحوصات فاشلة متتالية",
       "eventXrayCrash": "تعطّل",
       "eventNodeDown": "غير متصلة",
       "eventNodeUp": "متصلة",

internal/web/translation/en-US.json

@@ -1610,6 +1610,7 @@
       "eventGroupNode": "Nodes",
       "eventOutboundDown": "Down",
       "eventOutboundUp": "Up",
+      "eventDownThresholdHint": "failed checks in a row",
       "eventXrayCrash": "Crash",
       "eventNodeDown": "Down",
       "eventNodeUp": "Up",

internal/web/translation/es-ES.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Nodos",
       "eventOutboundDown": "Caído",
       "eventOutboundUp": "Activo",
+      "eventDownThresholdHint": "comprobaciones fallidas seguidas",
       "eventXrayCrash": "Caída",
       "eventNodeDown": "Caído",
       "eventNodeUp": "Activo",

internal/web/translation/fa-IR.json

@@ -1492,6 +1492,7 @@
       "eventGroupNode": "نودها",
       "eventOutboundDown": "قطع",
       "eventOutboundUp": "وصل",
+      "eventDownThresholdHint": "بررسی ناموفق پیاپی",
       "eventXrayCrash": "کرش",
       "eventNodeDown": "قطع",
       "eventNodeUp": "وصل",

internal/web/translation/id-ID.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Node",
       "eventOutboundDown": "Mati",
       "eventOutboundUp": "Aktif",
+      "eventDownThresholdHint": "pemeriksaan gagal berturut-turut",
       "eventXrayCrash": "Crash",
       "eventNodeDown": "Mati",
       "eventNodeUp": "Aktif",

internal/web/translation/ja-JP.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "ノード",
       "eventOutboundDown": "ダウン",
       "eventOutboundUp": "アップ",
+      "eventDownThresholdHint": "回連続でチェック失敗",
       "eventXrayCrash": "クラッシュ",
       "eventNodeDown": "ダウン",
       "eventNodeUp": "アップ",

internal/web/translation/pt-BR.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Nós",
       "eventOutboundDown": "Inativo",
       "eventOutboundUp": "Ativo",
+      "eventDownThresholdHint": "verificações com falha seguidas",
       "eventXrayCrash": "Falha",
       "eventNodeDown": "Inativo",
       "eventNodeUp": "Ativo",

internal/web/translation/ru-RU.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Узлы",
       "eventOutboundDown": "Недоступен",
       "eventOutboundUp": "Работает",
+      "eventDownThresholdHint": "неудачных проверок подряд",
       "eventXrayCrash": "Сбой",
       "eventNodeDown": "Недоступен",
       "eventNodeUp": "В сети",

internal/web/translation/tr-TR.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Düğümler",
       "eventOutboundDown": "Çevrimdışı",
       "eventOutboundUp": "Çevrimiçi",
+      "eventDownThresholdHint": "art arda başarısız kontrol",
       "eventXrayCrash": "Çökme",
       "eventNodeDown": "Çevrimdışı",
       "eventNodeUp": "Çevrimiçi",

internal/web/translation/uk-UA.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Вузли",
       "eventOutboundDown": "Недоступне",
       "eventOutboundUp": "Доступне",
+      "eventDownThresholdHint": "невдалих перевірок поспіль",
       "eventXrayCrash": "Збій",
       "eventNodeDown": "Недоступний",
       "eventNodeUp": "Доступний",

internal/web/translation/vi-VN.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "Node",
       "eventOutboundDown": "Ngừng hoạt động",
       "eventOutboundUp": "Hoạt động",
+      "eventDownThresholdHint": "lần kiểm tra thất bại liên tiếp",
       "eventXrayCrash": "Sự cố",
       "eventNodeDown": "Ngừng hoạt động",
       "eventNodeUp": "Hoạt động",

internal/web/translation/zh-CN.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "节点",
       "eventOutboundDown": "断开",
       "eventOutboundUp": "恢复",
+      "eventDownThresholdHint": "次连续检查失败",
       "eventXrayCrash": "崩溃",
       "eventNodeDown": "离线",
       "eventNodeUp": "上线",

internal/web/translation/zh-TW.json

@@ -1488,6 +1488,7 @@
       "eventGroupNode": "節點",
       "eventOutboundDown": "中斷",
       "eventOutboundUp": "恢復",
+      "eventDownThresholdHint": "次連續檢查失敗",
       "eventXrayCrash": "當機",
       "eventNodeDown": "離線",
       "eventNodeUp": "上線",