From 2f62dadd25f6b2a20020885e95a5b1d3f1aa5754 Mon Sep 17 00:00:00 2001 From: Grigoriy Date: Sat, 24 Oct 2026 05:26:13 +0300 Subject: [PATCH] feat(notifications): add a failed-check threshold for node.down alerts (#6768) One lost heartbeat on a lossy link flipped a node offline and sent node.down, and the next probe sent node.up, so a flaky node flooded Telegram, email and Discord with alert pairs. A new nodeDownThreshold setting (default 1, today's behaviour) works like outboundDownThreshold: node.down fires once the failed-check streak reaches it, once per outage, and node.up only after a node.down was sent. The announcement is persisted in nodes.down_notified, so a restart inside the threshold window neither drops the outage's node.down nor re-announces nodes already reported offline; nodes offline at upgrade are seeded as announced. --- docs/public/openapi.json | 12 ++ frontend/public/openapi.json | 12 ++ .../ui/notifications/EmailNotifications.tsx | 19 ++- .../ui/notifications/NotificationGroup.tsx | 24 ++-- .../notifications/TelegramNotifications.tsx | 19 ++- .../src/components/ui/notifications/types.ts | 1 + frontend/src/generated/examples.ts | 2 + frontend/src/generated/schemas.ts | 12 ++ frontend/src/generated/types.ts | 2 + frontend/src/generated/zod.ts | 2 + frontend/src/models/setting.ts | 5 + frontend/src/schemas/setting.ts | 1 + internal/database/db.go | 19 ++- internal/database/model/model.go | 2 + .../node_down_notified_migration_test.go | 43 +++++++ internal/web/entity/entity.go | 1 + internal/web/job/node_down_threshold_test.go | 118 ++++++++++++++++++ internal/web/job/node_heartbeat_job.go | 87 +++++++++---- .../web/job/node_transition_burst_test.go | 4 + internal/web/service/node.go | 6 + internal/web/service/setting.go | 10 ++ internal/web/translation/ar-EG.json | 1 + internal/web/translation/en-US.json | 1 + internal/web/translation/es-ES.json | 1 + internal/web/translation/fa-IR.json | 1 + internal/web/translation/id-ID.json | 1 + internal/web/translation/ja-JP.json | 1 + internal/web/translation/pt-BR.json | 1 + internal/web/translation/ru-RU.json | 1 + internal/web/translation/tr-TR.json | 1 + internal/web/translation/uk-UA.json | 1 + internal/web/translation/vi-VN.json | 1 + internal/web/translation/zh-CN.json | 1 + internal/web/translation/zh-TW.json | 1 + 34 files changed, 379 insertions(+), 35 deletions(-) create mode 100644 internal/database/node_down_notified_migration_test.go create mode 100644 internal/web/job/node_down_threshold_test.go diff --git a/docs/public/openapi.json b/docs/public/openapi.json index c151d54bf..668cd73dd 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -152,6 +152,11 @@ "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -639,6 +644,7 @@ "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", @@ -932,6 +938,11 @@ "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -1427,6 +1438,7 @@ "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", diff --git a/frontend/public/openapi.json b/frontend/public/openapi.json index c151d54bf..668cd73dd 100644 --- a/frontend/public/openapi.json +++ b/frontend/public/openapi.json @@ -152,6 +152,11 @@ "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -639,6 +644,7 @@ "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", @@ -932,6 +938,11 @@ "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -1427,6 +1438,7 @@ "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", diff --git a/frontend/src/components/ui/notifications/EmailNotifications.tsx b/frontend/src/components/ui/notifications/EmailNotifications.tsx index abcdd1e46..8ee1fba06 100644 --- a/frontend/src/components/ui/notifications/EmailNotifications.tsx +++ b/frontend/src/components/ui/notifications/EmailNotifications.tsx @@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [ key: 'outbound.down', label: 'eventOutboundDown', settingKey: 'outboundDownThreshold', + hint: 'eventDownThresholdHint', extra: ({ value, onChange, ariaLabel }) => ( , title: 'eventGroupNode', events: [ - { key: 'node.down', label: 'eventNodeDown', settingKey: '' }, + { + key: 'node.down', + label: 'eventNodeDown', + settingKey: 'nodeDownThreshold', + hint: 'eventDownThresholdHint', + extra: ({ value, onChange, ariaLabel }) => ( + + ), + }, { key: 'node.up', label: 'eventNodeUp', settingKey: '' }, ], }, diff --git a/frontend/src/components/ui/notifications/NotificationGroup.tsx b/frontend/src/components/ui/notifications/NotificationGroup.tsx index f43b34a8e..c263f6416 100644 --- a/frontend/src/components/ui/notifications/NotificationGroup.tsx +++ b/frontend/src/components/ui/notifications/NotificationGroup.tsx @@ -1,4 +1,4 @@ -import { Space } from 'antd'; +import { Space, Typography } from 'antd'; import { useTranslation } from 'react-i18next'; import type { AllSetting } from '@/models/setting'; import type { NotificationGroupConfig } from './types'; @@ -55,12 +55,22 @@ export function NotificationGroup({ checked={selected.includes(event.key)} onToggle={() => onToggle(event.key)} > - {event.extra?.({ - value: - Number((allSetting as unknown as Record)[event.settingKey]) || 0, - onChange: (v) => updateSetting({ [event.settingKey]: v }), - ariaLabel: t(`pages.settings.${event.label}`), - })} + {event.extra && ( + + {event.extra({ + value: + Number((allSetting as unknown as Record)[event.settingKey]) || + 0, + onChange: (v) => updateSetting({ [event.settingKey]: v }), + ariaLabel: t(`pages.settings.${event.label}`), + })} + {event.hint && ( + + {t(`pages.settings.${event.hint}`)} + + )} + + )} ))} diff --git a/frontend/src/components/ui/notifications/TelegramNotifications.tsx b/frontend/src/components/ui/notifications/TelegramNotifications.tsx index 826ee08ed..9c630382e 100644 --- a/frontend/src/components/ui/notifications/TelegramNotifications.tsx +++ b/frontend/src/components/ui/notifications/TelegramNotifications.tsx @@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [ key: 'outbound.down', label: 'eventOutboundDown', settingKey: 'outboundDownThreshold', + hint: 'eventDownThresholdHint', extra: ({ value, onChange, ariaLabel }) => ( , title: 'eventGroupNode', events: [ - { key: 'node.down', label: 'eventNodeDown', settingKey: '' }, + { + key: 'node.down', + label: 'eventNodeDown', + settingKey: 'nodeDownThreshold', + hint: 'eventDownThresholdHint', + extra: ({ value, onChange, ariaLabel }) => ( + + ), + }, { key: 'node.up', label: 'eventNodeUp', settingKey: '' }, ], }, diff --git a/frontend/src/components/ui/notifications/types.ts b/frontend/src/components/ui/notifications/types.ts index fb742d2b4..dc2a2bd9f 100644 --- a/frontend/src/components/ui/notifications/types.ts +++ b/frontend/src/components/ui/notifications/types.ts @@ -4,6 +4,7 @@ export interface NotificationEventConfig { key: string; label: string; settingKey: string; + hint?: string; extra?: (props: { value: number; onChange: (v: number | null) => void; diff --git a/frontend/src/generated/examples.ts b/frontend/src/generated/examples.ts index 116b6a1c8..9b9bf9603 100644 --- a/frontend/src/generated/examples.ts +++ b/frontend/src/generated/examples.ts @@ -39,6 +39,7 @@ export const EXAMPLES: Record = { "ldapUserAttr": "", "ldapUserFilter": "", "ldapVlessField": "", + "nodeDownThreshold": 1, "outboundDownThreshold": 1, "pageSize": 0, "panelOutbound": "", @@ -227,6 +228,7 @@ export const EXAMPLES: Record = { "ldapUserAttr": "", "ldapUserFilter": "", "ldapVlessField": "", + "nodeDownThreshold": 1, "outboundDownThreshold": 1, "pageSize": 0, "panelOutbound": "", diff --git a/frontend/src/generated/schemas.ts b/frontend/src/generated/schemas.ts index 421089bce..0e39ecdd6 100644 --- a/frontend/src/generated/schemas.ts +++ b/frontend/src/generated/schemas.ts @@ -126,6 +126,11 @@ export const SCHEMAS: Record = { "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -613,6 +618,7 @@ export const SCHEMAS: Record = { "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", @@ -906,6 +912,11 @@ export const SCHEMAS: Record = { "ldapVlessField": { "type": "string" }, + "nodeDownThreshold": { + "maximum": 100, + "minimum": 1, + "type": "integer" + }, "outboundDownThreshold": { "maximum": 100, "minimum": 1, @@ -1401,6 +1412,7 @@ export const SCHEMAS: Record = { "ldapUserAttr", "ldapUserFilter", "ldapVlessField", + "nodeDownThreshold", "outboundDownThreshold", "pageSize", "panelOutbound", diff --git a/frontend/src/generated/types.ts b/frontend/src/generated/types.ts index 1e1249692..34945dfd2 100644 --- a/frontend/src/generated/types.ts +++ b/frontend/src/generated/types.ts @@ -47,6 +47,7 @@ export interface AllSetting { ldapUserAttr: string; ldapUserFilter: string; ldapVlessField: string; + nodeDownThreshold: number; outboundDownThreshold: number; pageSize: number; panelOutbound: string; @@ -236,6 +237,7 @@ export interface AllSettingView { ldapUserAttr: string; ldapUserFilter: string; ldapVlessField: string; + nodeDownThreshold: number; outboundDownThreshold: number; pageSize: number; panelOutbound: string; diff --git a/frontend/src/generated/zod.ts b/frontend/src/generated/zod.ts index 1578fbe97..8633c2846 100644 --- a/frontend/src/generated/zod.ts +++ b/frontend/src/generated/zod.ts @@ -63,6 +63,7 @@ export const AllSettingSchema = z.object({ ldapUserAttr: z.string(), ldapUserFilter: z.string(), ldapVlessField: z.string(), + nodeDownThreshold: z.number().int().min(1).max(100), outboundDownThreshold: z.number().int().min(1).max(100), pageSize: z.number().int().min(0).max(1000), panelOutbound: z.string(), @@ -253,6 +254,7 @@ export const AllSettingViewSchema = z.object({ ldapUserAttr: z.string(), ldapUserFilter: z.string(), ldapVlessField: z.string(), + nodeDownThreshold: z.number().int().min(1).max(100), outboundDownThreshold: z.number().int().min(1).max(100), pageSize: z.number().int().min(0).max(1000), panelOutbound: z.string(), diff --git a/frontend/src/models/setting.ts b/frontend/src/models/setting.ts index df4fc54c7..f69f8a624 100644 --- a/frontend/src/models/setting.ts +++ b/frontend/src/models/setting.ts @@ -174,6 +174,7 @@ export class AllSetting { smtpCpu = 80; smtpMemory = 80; outboundDownThreshold = 3; + nodeDownThreshold = 1; hasTgBotToken = false; hasTwoFactorToken = false; hasLdapPassword = false; @@ -216,6 +217,10 @@ export class AllSetting { this.outboundDownThreshold = Number.isFinite(threshold) ? Math.min(100, Math.max(1, threshold)) : 3; + const nodeThreshold = Math.round(Number(this.nodeDownThreshold)); + this.nodeDownThreshold = Number.isFinite(nodeThreshold) + ? Math.min(100, Math.max(1, nodeThreshold)) + : 1; } equals(other: AllSetting): boolean { diff --git a/frontend/src/schemas/setting.ts b/frontend/src/schemas/setting.ts index fb21f2337..41075d984 100644 --- a/frontend/src/schemas/setting.ts +++ b/frontend/src/schemas/setting.ts @@ -39,6 +39,7 @@ export const AllSettingSchema = z tgBotBackup: z.boolean().optional(), tgCpu: z.number().int().min(0).max(100).optional(), outboundDownThreshold: z.number().int().min(1).max(100).optional(), + nodeDownThreshold: z.number().int().min(1).max(100).optional(), tgLang: z.string().optional(), twoFactorEnable: z.boolean().optional(), twoFactorToken: z.string().optional(), diff --git a/internal/database/db.go b/internal/database/db.go index 00ae09b51..cf319df60 100644 --- a/internal/database/db.go +++ b/internal/database/db.go @@ -1475,7 +1475,7 @@ func runSeeders(isUsersEmpty bool) error { } if empty && isUsersEmpty { - seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "ResetIpLimitNoFail2ban"} + seeders := []string{"UserPasswordHash", "ClientsTable", "InboundClientsArrayFix", "InboundClientTgIdFix2", "InboundClientSubIdFix", "FreedomFinalRulesReverseFix", "FreedomFinalRulesPrivateEgressBlock", "UppercaseFreedomFinalRulesFix", "InboundRealityFinalmaskTcpStrip", "ApiTokensHash", "LegacyProxySettingsCleanup", "OutboundRemovedKeysFix", "FreedomDomainStrategyFix", "DNSOutboundLegacyKeysFix", "DNSOutboundQTypeZeroFix", "WireguardDomainStrategyFix", "XdnsFinalmaskNamesAddrsFix", "UDPHopClientMaskFix", "HostVerifyPeerCertByNameBoolFix", "WireguardPeersToClients", "MtprotoSecretsToClients", "NodeInboundsAdopted", "NodeDownNotified", "ResetIpLimitNoFail2ban"} for _, name := range seeders { if err := db.Create(&model.HistoryOfSeeders{SeederName: name}).Error; err != nil { return err @@ -1640,6 +1640,12 @@ func runSeeders(isUsersEmpty bool) error { } } + if !slices.Contains(seedersHistory, "NodeDownNotified") { + if err := seedNodeDownNotified(); err != nil { + return err + } + } + if err := seedHostsFromExternalProxy(); err != nil { return err } @@ -1689,6 +1695,17 @@ func seedNodeInboundsAdopted() error { return db.Create(&model.HistoryOfSeeders{SeederName: "NodeInboundsAdopted"}).Error } +// seedNodeDownNotified treats nodes already offline at upgrade as announced, +// so the first heartbeat after it does not re-send their node.down. +func seedNodeDownNotified() error { + if err := db.Model(&model.Node{}). + Where("status = ?", "offline"). + Update("down_notified", true).Error; err != nil { + return err + } + return db.Create(&model.HistoryOfSeeders{SeederName: "NodeDownNotified"}).Error +} + // backfillEmptyHostGroupIds is idempotent and not seeder-gated: builds that // predate group ids on the inbound-import path (and restored backups) can // re-introduce hosts rows with an empty group_id, and such rows render as a diff --git a/internal/database/model/model.go b/internal/database/model/model.go index e2b5f2a43..f5dcd781d 100644 --- a/internal/database/model/model.go +++ b/internal/database/model/model.go @@ -878,6 +878,8 @@ type Node struct { // save that grows the selection zeroes it so reconcile waits before sweeping. InboundsAdoptedAt int64 `json:"-" gorm:"column:inbounds_adopted_at;default:0"` + DownNotified bool `json:"-" gorm:"column:down_notified;default:false"` + InboundCount int `json:"inboundCount" gorm:"-" example:"5"` ClientCount int `json:"clientCount" gorm:"-" example:"27"` OnlineCount int `json:"onlineCount" gorm:"-" example:"3"` diff --git a/internal/database/node_down_notified_migration_test.go b/internal/database/node_down_notified_migration_test.go new file mode 100644 index 000000000..08f302165 --- /dev/null +++ b/internal/database/node_down_notified_migration_test.go @@ -0,0 +1,43 @@ +package database + +import ( + "testing" + + "github.com/mhsanaei/3x-ui/v3/internal/config" + "github.com/mhsanaei/3x-ui/v3/internal/database/model" +) + +func TestNodeDownNotifiedSeederMarksOfflineNodesAnnounced(t *testing.T) { + t.Setenv("XUI_DB_FOLDER", t.TempDir()) + if err := InitDB(config.GetDBPath()); err != nil { + t.Fatalf("init db: %v", err) + } + t.Cleanup(func() { _ = CloseDB() }) + if err := db.Where("seeder_name = ?", "NodeDownNotified").Delete(&model.HistoryOfSeeders{}).Error; err != nil { + t.Fatalf("clear seeder history: %v", err) + } + offline := model.Node{Name: "offline", Address: "10.0.0.1", Port: 1, Status: "offline"} + online := model.Node{Name: "online", Address: "10.0.0.2", Port: 1, Status: "online"} + for _, n := range []*model.Node{&offline, &online} { + if err := db.Create(n).Error; err != nil { + t.Fatalf("create node %s: %v", n.Name, err) + } + } + + if err := runSeeders(false); err != nil { + t.Fatalf("runSeeders: %v", err) + } + + for _, tc := range []struct { + node *model.Node + want bool + }{{&offline, true}, {&online, false}} { + var got model.Node + if err := db.First(&got, tc.node.Id).Error; err != nil { + t.Fatalf("reload node %s: %v", tc.node.Name, err) + } + if got.DownNotified != tc.want { + t.Errorf("node %s DownNotified = %v, want %v", tc.node.Name, got.DownNotified, tc.want) + } + } +} diff --git a/internal/web/entity/entity.go b/internal/web/entity/entity.go index 3437a86ad..4a5adb235 100644 --- a/internal/web/entity/entity.go +++ b/internal/web/entity/entity.go @@ -80,6 +80,7 @@ type AllSetting struct { DiscordEnabledEvents string `json:"discordEnabledEvents" form:"discordEnabledEvents"` OutboundDownThreshold int `json:"outboundDownThreshold" form:"outboundDownThreshold" validate:"gte=1,lte=100"` + NodeDownThreshold int `json:"nodeDownThreshold" form:"nodeDownThreshold" validate:"gte=1,lte=100"` TimeLocation string `json:"timeLocation" form:"timeLocation"` TwoFactorEnable bool `json:"twoFactorEnable" form:"twoFactorEnable"` diff --git a/internal/web/job/node_down_threshold_test.go b/internal/web/job/node_down_threshold_test.go new file mode 100644 index 000000000..8503d9428 --- /dev/null +++ b/internal/web/job/node_down_threshold_test.go @@ -0,0 +1,118 @@ +package job + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/op/go-logging" + + "github.com/mhsanaei/3x-ui/v3/internal/database" + "github.com/mhsanaei/3x-ui/v3/internal/database/dbtest" + "github.com/mhsanaei/3x-ui/v3/internal/database/model" + xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger" + "github.com/mhsanaei/3x-ui/v3/internal/web/runtime" + "github.com/mhsanaei/3x-ui/v3/internal/web/service" +) + +func switchableNode(t *testing.T) *atomic.Bool { + t.Helper() + xuilogger.InitLogger(logging.ERROR) + dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db")) + runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}})) + t.Cleanup(func() { runtime.SetManager(nil) }) + + var up atomic.Bool + up.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !up.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "server/status") { + _, _ = w.Write([]byte(`{"success":true,"obj":{"panelGuid":"flaky-guid","xray":{"state":"running"}}}`)) + return + } + _, _ = w.Write([]byte(`{"success":true,"obj":[]}`)) + })) + t.Cleanup(srv.Close) + host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":") + portNum, _ := strconv.Atoi(port) + node := &model.Node{ + Name: "flaky", Scheme: "http", Address: host, Port: portNum, BasePath: "/", ApiToken: "tok", + Enable: true, Status: "online", AllowPrivateAddress: true, TlsVerifyMode: "verify", + } + if err := database.GetDB().Create(node).Error; err != nil { + t.Fatalf("create node: %v", err) + } + return &up +} + +func TestHeartbeatHoldsNodeDownUntilThreshold(t *testing.T) { + cases := []struct { + name string + probes []bool + restartBefore int + want []string + }{ + { + name: "flapping below threshold", + probes: []bool{false, true, false, false, true, false, true}, + }, + { + name: "outage reaching threshold", + probes: []bool{false, false, false, false, true}, + want: []string{"probe 3: node.down", "probe 5: node.up"}, + }, + { + name: "restart mid-streak still announces the outage", + probes: []bool{false, false, false, false, true}, + restartBefore: 2, + want: []string{"probe 4: node.down", "probe 5: node.up"}, + }, + { + name: "restart after announced outage does not repeat it", + probes: []bool{false, false, false, false, false, false, true}, + restartBefore: 5, + want: []string{"probe 3: node.down", "probe 7: node.up"}, + }, + { + name: "restart after recovery announces the next outage", + probes: []bool{false, false, false, true, false, false, false, true}, + restartBefore: 5, + want: []string{"probe 3: node.down", "probe 4: node.up", "probe 7: node.down", "probe 8: node.up"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + up := switchableNode(t) + if err := (&service.SettingService{}).SetNodeDownThreshold(3); err != nil { + t.Fatalf("SetNodeDownThreshold: %v", err) + } + events := collectNodeEvents(t) + job := NewNodeHeartbeatJob() + + var got []string + for i, probeOK := range tc.probes { + if i+1 == tc.restartBefore { + job = NewNodeHeartbeatJob() + } + up.Store(probeOK) + job.Run() + for _, e := range events()[len(got):] { + got = append(got, fmt.Sprintf("probe %d: %s", i+1, e.Type)) + } + } + + if strings.Join(got, ", ") != strings.Join(tc.want, ", ") { + t.Fatalf("notifications = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/internal/web/job/node_heartbeat_job.go b/internal/web/job/node_heartbeat_job.go index 1fafd7ea8..ca6c6e1fc 100644 --- a/internal/web/job/node_heartbeat_job.go +++ b/internal/web/job/node_heartbeat_job.go @@ -27,8 +27,20 @@ const ( ) type NodeHeartbeatJob struct { - nodeService service.NodeService - running sync.Mutex + nodeService service.NodeService + settingService service.SettingService + running sync.Mutex + health map[int]nodeHealth +} + +type nodeHealth struct { + failStreak int + notified bool +} + +type nodeProbe struct { + node *model.Node + patch service.HeartbeatPatch } func NewNodeHeartbeatJob() *NodeHeartbeatJob { @@ -53,8 +65,8 @@ func (j *NodeHeartbeatJob) Run() { sem := make(chan struct{}, nodeHeartbeatConcurrency) var wg sync.WaitGroup - var transitionsMu sync.Mutex - var transitions []eventbus.Event + var probesMu sync.Mutex + var probes []nodeProbe for _, n := range nodes { if !n.Enable { continue @@ -65,15 +77,14 @@ func (j *NodeHeartbeatJob) Run() { common.GoRecover("node-heartbeat:"+n.Name, func() { defer wg.Done() defer func() { <-sem }() - if event := j.probeOne(n); event != nil { - transitionsMu.Lock() - transitions = append(transitions, *event) - transitionsMu.Unlock() - } + patch := j.probeOne(n) + probesMu.Lock() + probes = append(probes, nodeProbe{node: n, patch: patch}) + probesMu.Unlock() }) } wg.Wait() - publishNodeTransitions(transitions) + publishNodeTransitions(j.healthTransitions(probes)) if !websocket.HasClients() { return @@ -86,10 +97,9 @@ func (j *NodeHeartbeatJob) Run() { websocket.BroadcastNodes(updated) } -func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event { +func (j *NodeHeartbeatJob) probeOne(n *model.Node) service.HeartbeatPatch { ctx, cancel := context.WithTimeout(context.Background(), nodeHeartbeatRequestTimeout) defer cancel() - prevStatus := n.Status patch, err := j.nodeService.Probe(ctx, n) if err != nil { patch.Status = "offline" @@ -109,26 +119,53 @@ func (j *NodeHeartbeatJob) probeOne(n *model.Node) *eventbus.Event { } else { j.nodeService.ClearDescendants(n.Id) } - return nodeTransitionEvent(n, prevStatus, patch) + return patch } -// nodeTransitionEvent is node.down / node.up on a genuine state change only; an unknown -// previous status (fresh start) counts as not-online, so it never yields node.down. -func nodeTransitionEvent(n *model.Node, prevStatus string, patch service.HeartbeatPatch) *eventbus.Event { - var eventType eventbus.EventType - switch { - case prevStatus == "online" && patch.Status == "offline": - eventType = eventbus.EventNodeDown - case prevStatus != "online" && patch.Status == "online": - eventType = eventbus.EventNodeUp - default: - return nil +func (j *NodeHeartbeatJob) healthTransitions(probes []nodeProbe) []eventbus.Event { + threshold := 1 + if v, err := j.settingService.GetNodeDownThreshold(); err == nil && v > 0 { + threshold = v } + prev := j.health + j.health = make(map[int]nodeHealth, len(probes)) + var events []eventbus.Event + for _, p := range probes { + h, seen := prev[p.node.Id] + if !seen { + h.notified = p.node.DownNotified + } + if p.patch.Status == "online" { + if h.notified { + events = append(events, nodeHealthEvent(eventbus.EventNodeUp, p.node, p.patch)) + j.saveDownNotified(p.node.Id, false) + } + h = nodeHealth{} + } else { + h.failStreak++ + if h.failStreak >= threshold && !h.notified { + events = append(events, nodeHealthEvent(eventbus.EventNodeDown, p.node, p.patch)) + h.notified = true + j.saveDownNotified(p.node.Id, true) + } + } + j.health[p.node.Id] = h + } + return events +} + +func (j *NodeHeartbeatJob) saveDownNotified(id int, notified bool) { + if err := j.nodeService.SetDownNotified(id, notified); err != nil { + logger.Warning("node heartbeat: save down-notified flag for node", id, "failed:", err) + } +} + +func nodeHealthEvent(eventType eventbus.EventType, n *model.Node, patch service.HeartbeatPatch) eventbus.Event { source := n.Name if source == "" { source = "node-" + strconv.Itoa(n.Id) } - return &eventbus.Event{ + return eventbus.Event{ Type: eventType, Source: source, Data: &eventbus.NodeHealthData{ diff --git a/internal/web/job/node_transition_burst_test.go b/internal/web/job/node_transition_burst_test.go index 0b31a8c4b..4f0fe2056 100644 --- a/internal/web/job/node_transition_burst_test.go +++ b/internal/web/job/node_transition_burst_test.go @@ -18,6 +18,7 @@ import ( "github.com/mhsanaei/3x-ui/v3/internal/eventbus" xuilogger "github.com/mhsanaei/3x-ui/v3/internal/logger" "github.com/mhsanaei/3x-ui/v3/internal/web/runtime" + "github.com/mhsanaei/3x-ui/v3/internal/web/service" ) // goingDownNodes seeds n online nodes whose address refuses connections, so the @@ -28,6 +29,9 @@ func goingDownNodes(t *testing.T, n int) { dbtest.InitDB(t, filepath.Join(t.TempDir(), "x-ui.db")) runtime.SetManager(runtime.NewManager(runtime.LocalDeps{APIPort: func() int { return 0 }, SetNeedRestart: func() {}})) t.Cleanup(func() { runtime.SetManager(nil) }) + if err := (&service.SettingService{}).SetNodeDownThreshold(1); err != nil { + t.Fatalf("SetNodeDownThreshold: %v", err) + } srv := httptest.NewServer(nil) host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":") portNum, _ := strconv.Atoi(port) diff --git a/internal/web/service/node.go b/internal/web/service/node.go index 9a770f0c6..21cd49a94 100644 --- a/internal/web/service/node.go +++ b/internal/web/service/node.go @@ -1064,6 +1064,12 @@ func (s *NodeService) ClearNodeDirty(id int, dirtyAt int64) error { Update("config_dirty", false).Error } +func (s *NodeService) SetDownNotified(id int, notified bool) error { + return database.GetDB().Model(model.Node{}). + Where("id = ?", id). + Update("down_notified", notified).Error +} + func (s *NodeService) MarkNodeInboundsAdopted(id int) error { if id <= 0 { return nil diff --git a/internal/web/service/setting.go b/internal/web/service/setting.go index c309726c5..4677daf7f 100644 --- a/internal/web/service/setting.go +++ b/internal/web/service/setting.go @@ -238,6 +238,8 @@ var defaultValueMap = map[string]string{ // Consecutive failed observatory probes before an outbound.down event fires "outboundDownThreshold": "3", + "nodeDownThreshold": "1", + // Email (SMTP) notifications "smtpEnable": "false", "smtpHost": "", @@ -1641,6 +1643,14 @@ func (s *SettingService) SetOutboundDownThreshold(value int) error { return s.setInt("outboundDownThreshold", value) } +func (s *SettingService) GetNodeDownThreshold() (int, error) { + return s.getInt("nodeDownThreshold") +} + +func (s *SettingService) SetNodeDownThreshold(value int) error { + return s.setInt("nodeDownThreshold", value) +} + // SecretClears marks redacted secrets the user explicitly emptied. Without a // flag, a blank submitted secret means "unchanged" (the field is always served // blank to the browser) and the stored value is preserved. diff --git a/internal/web/translation/ar-EG.json b/internal/web/translation/ar-EG.json index 66addd333..f0fa5d5cd 100644 --- a/internal/web/translation/ar-EG.json +++ b/internal/web/translation/ar-EG.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "العقد", "eventOutboundDown": "غير متصل", "eventOutboundUp": "متصل", + "eventDownThresholdHint": "فحوصات فاشلة متتالية", "eventXrayCrash": "تعطّل", "eventNodeDown": "غير متصلة", "eventNodeUp": "متصلة", diff --git a/internal/web/translation/en-US.json b/internal/web/translation/en-US.json index 6f520bd98..923475e76 100644 --- a/internal/web/translation/en-US.json +++ b/internal/web/translation/en-US.json @@ -1610,6 +1610,7 @@ "eventGroupNode": "Nodes", "eventOutboundDown": "Down", "eventOutboundUp": "Up", + "eventDownThresholdHint": "failed checks in a row", "eventXrayCrash": "Crash", "eventNodeDown": "Down", "eventNodeUp": "Up", diff --git a/internal/web/translation/es-ES.json b/internal/web/translation/es-ES.json index 9c3cd8815..5c66a0c4d 100644 --- a/internal/web/translation/es-ES.json +++ b/internal/web/translation/es-ES.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Nodos", "eventOutboundDown": "Caído", "eventOutboundUp": "Activo", + "eventDownThresholdHint": "comprobaciones fallidas seguidas", "eventXrayCrash": "Caída", "eventNodeDown": "Caído", "eventNodeUp": "Activo", diff --git a/internal/web/translation/fa-IR.json b/internal/web/translation/fa-IR.json index 2865ad109..a4726af21 100644 --- a/internal/web/translation/fa-IR.json +++ b/internal/web/translation/fa-IR.json @@ -1492,6 +1492,7 @@ "eventGroupNode": "نودها", "eventOutboundDown": "قطع", "eventOutboundUp": "وصل", + "eventDownThresholdHint": "بررسی ناموفق پیاپی", "eventXrayCrash": "کرش", "eventNodeDown": "قطع", "eventNodeUp": "وصل", diff --git a/internal/web/translation/id-ID.json b/internal/web/translation/id-ID.json index ff143679b..9854cc354 100644 --- a/internal/web/translation/id-ID.json +++ b/internal/web/translation/id-ID.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Node", "eventOutboundDown": "Mati", "eventOutboundUp": "Aktif", + "eventDownThresholdHint": "pemeriksaan gagal berturut-turut", "eventXrayCrash": "Crash", "eventNodeDown": "Mati", "eventNodeUp": "Aktif", diff --git a/internal/web/translation/ja-JP.json b/internal/web/translation/ja-JP.json index 9f78705c7..7f1be3ddb 100644 --- a/internal/web/translation/ja-JP.json +++ b/internal/web/translation/ja-JP.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "ノード", "eventOutboundDown": "ダウン", "eventOutboundUp": "アップ", + "eventDownThresholdHint": "回連続でチェック失敗", "eventXrayCrash": "クラッシュ", "eventNodeDown": "ダウン", "eventNodeUp": "アップ", diff --git a/internal/web/translation/pt-BR.json b/internal/web/translation/pt-BR.json index f2a2d4e9e..39d880e13 100644 --- a/internal/web/translation/pt-BR.json +++ b/internal/web/translation/pt-BR.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Nós", "eventOutboundDown": "Inativo", "eventOutboundUp": "Ativo", + "eventDownThresholdHint": "verificações com falha seguidas", "eventXrayCrash": "Falha", "eventNodeDown": "Inativo", "eventNodeUp": "Ativo", diff --git a/internal/web/translation/ru-RU.json b/internal/web/translation/ru-RU.json index a9e06a42f..656b063ff 100644 --- a/internal/web/translation/ru-RU.json +++ b/internal/web/translation/ru-RU.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Узлы", "eventOutboundDown": "Недоступен", "eventOutboundUp": "Работает", + "eventDownThresholdHint": "неудачных проверок подряд", "eventXrayCrash": "Сбой", "eventNodeDown": "Недоступен", "eventNodeUp": "В сети", diff --git a/internal/web/translation/tr-TR.json b/internal/web/translation/tr-TR.json index d46d1a32c..4e997f03c 100644 --- a/internal/web/translation/tr-TR.json +++ b/internal/web/translation/tr-TR.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Düğümler", "eventOutboundDown": "Çevrimdışı", "eventOutboundUp": "Çevrimiçi", + "eventDownThresholdHint": "art arda başarısız kontrol", "eventXrayCrash": "Çökme", "eventNodeDown": "Çevrimdışı", "eventNodeUp": "Çevrimiçi", diff --git a/internal/web/translation/uk-UA.json b/internal/web/translation/uk-UA.json index 61d6d8edd..02169e8be 100644 --- a/internal/web/translation/uk-UA.json +++ b/internal/web/translation/uk-UA.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Вузли", "eventOutboundDown": "Недоступне", "eventOutboundUp": "Доступне", + "eventDownThresholdHint": "невдалих перевірок поспіль", "eventXrayCrash": "Збій", "eventNodeDown": "Недоступний", "eventNodeUp": "Доступний", diff --git a/internal/web/translation/vi-VN.json b/internal/web/translation/vi-VN.json index c0c8d92ff..b9e269fbc 100644 --- a/internal/web/translation/vi-VN.json +++ b/internal/web/translation/vi-VN.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "Node", "eventOutboundDown": "Ngừng hoạt động", "eventOutboundUp": "Hoạt động", + "eventDownThresholdHint": "lần kiểm tra thất bại liên tiếp", "eventXrayCrash": "Sự cố", "eventNodeDown": "Ngừng hoạt động", "eventNodeUp": "Hoạt động", diff --git a/internal/web/translation/zh-CN.json b/internal/web/translation/zh-CN.json index 22bf4cc7a..f9bcf5295 100644 --- a/internal/web/translation/zh-CN.json +++ b/internal/web/translation/zh-CN.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "节点", "eventOutboundDown": "断开", "eventOutboundUp": "恢复", + "eventDownThresholdHint": "次连续检查失败", "eventXrayCrash": "崩溃", "eventNodeDown": "离线", "eventNodeUp": "上线", diff --git a/internal/web/translation/zh-TW.json b/internal/web/translation/zh-TW.json index 979acc5e8..7213233d0 100644 --- a/internal/web/translation/zh-TW.json +++ b/internal/web/translation/zh-TW.json @@ -1488,6 +1488,7 @@ "eventGroupNode": "節點", "eventOutboundDown": "中斷", "eventOutboundUp": "恢復", + "eventDownThresholdHint": "次連續檢查失敗", "eventXrayCrash": "當機", "eventNodeDown": "離線", "eventNodeUp": "上線",