feat(notifications): add a failed-check threshold for node.down alerts (#6768)

One lost heartbeat on a lossy link flipped a node offline and sent node.down,
and the next probe sent node.up, so a flaky node flooded Telegram, email and
Discord with alert pairs.

A new nodeDownThreshold setting (default 1, today's behaviour) works like
outboundDownThreshold: node.down fires once the failed-check streak reaches it,
once per outage, and node.up only after a node.down was sent. The announcement
is persisted in nodes.down_notified, so a restart inside the threshold window
neither drops the outage's node.down nor re-announces nodes already reported
offline; nodes offline at upgrade are seeded as announced.
This commit is contained in:
Grigoriy
2026-10-24 05:26:13 +03:00
committed by GitHub
parent c3037fc716
commit 2f62dadd25
34 changed files with 379 additions and 35 deletions
+12
View File
@@ -152,6 +152,11 @@
"ldapVlessField": {
"type": "string"
},
"nodeDownThreshold": {
"maximum": 100,
"minimum": 1,
"type": "integer"
},
"outboundDownThreshold": {
"maximum": 100,
"minimum": 1,
@@ -639,6 +644,7 @@
"ldapUserAttr",
"ldapUserFilter",
"ldapVlessField",
"nodeDownThreshold",
"outboundDownThreshold",
"pageSize",
"panelOutbound",
@@ -932,6 +938,11 @@
"ldapVlessField": {
"type": "string"
},
"nodeDownThreshold": {
"maximum": 100,
"minimum": 1,
"type": "integer"
},
"outboundDownThreshold": {
"maximum": 100,
"minimum": 1,
@@ -1427,6 +1438,7 @@
"ldapUserAttr",
"ldapUserFilter",
"ldapVlessField",
"nodeDownThreshold",
"outboundDownThreshold",
"pageSize",
"panelOutbound",
@@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [
key: 'outbound.down',
label: 'eventOutboundDown',
settingKey: 'outboundDownThreshold',
hint: 'eventDownThresholdHint',
extra: ({ value, onChange, ariaLabel }) => (
<InputNumber
size="small"
@@ -44,7 +45,23 @@ const GROUPS: NotificationGroupConfig[] = [
icon: <DesktopOutlined aria-hidden />,
title: 'eventGroupNode',
events: [
{ key: 'node.down', label: 'eventNodeDown', settingKey: '' },
{
key: 'node.down',
label: 'eventNodeDown',
settingKey: 'nodeDownThreshold',
hint: 'eventDownThresholdHint',
extra: ({ value, onChange, ariaLabel }) => (
<InputNumber
size="small"
min={1}
max={100}
value={value}
onChange={onChange}
aria-label={ariaLabel}
style={{ width: 80 }}
/>
),
},
{ key: 'node.up', label: 'eventNodeUp', settingKey: '' },
],
},
@@ -1,4 +1,4 @@
import { Space } from 'antd';
import { Space, Typography } from 'antd';
import { useTranslation } from 'react-i18next';
import type { AllSetting } from '@/models/setting';
import type { NotificationGroupConfig } from './types';
@@ -55,12 +55,22 @@ export function NotificationGroup({
checked={selected.includes(event.key)}
onToggle={() => onToggle(event.key)}
>
{event.extra?.({
value:
Number((allSetting as unknown as Record<string, unknown>)[event.settingKey]) || 0,
onChange: (v) => updateSetting({ [event.settingKey]: v }),
ariaLabel: t(`pages.settings.${event.label}`),
})}
{event.extra && (
<Space size={8}>
{event.extra({
value:
Number((allSetting as unknown as Record<string, unknown>)[event.settingKey]) ||
0,
onChange: (v) => updateSetting({ [event.settingKey]: v }),
ariaLabel: t(`pages.settings.${event.label}`),
})}
{event.hint && (
<Typography.Text type="secondary">
{t(`pages.settings.${event.hint}`)}
</Typography.Text>
)}
</Space>
)}
</NotificationEvent>
))}
</Space>
@@ -20,6 +20,7 @@ const GROUPS: NotificationGroupConfig[] = [
key: 'outbound.down',
label: 'eventOutboundDown',
settingKey: 'outboundDownThreshold',
hint: 'eventDownThresholdHint',
extra: ({ value, onChange, ariaLabel }) => (
<InputNumber
size="small"
@@ -44,7 +45,23 @@ const GROUPS: NotificationGroupConfig[] = [
icon: <DesktopOutlined aria-hidden />,
title: 'eventGroupNode',
events: [
{ key: 'node.down', label: 'eventNodeDown', settingKey: '' },
{
key: 'node.down',
label: 'eventNodeDown',
settingKey: 'nodeDownThreshold',
hint: 'eventDownThresholdHint',
extra: ({ value, onChange, ariaLabel }) => (
<InputNumber
size="small"
min={1}
max={100}
value={value}
onChange={onChange}
aria-label={ariaLabel}
style={{ width: 80 }}
/>
),
},
{ key: 'node.up', label: 'eventNodeUp', settingKey: '' },
],
},
@@ -4,6 +4,7 @@ export interface NotificationEventConfig {
key: string;
label: string;
settingKey: string;
hint?: string;
extra?: (props: {
value: number;
onChange: (v: number | null) => void;
+2
View File
@@ -39,6 +39,7 @@ export const EXAMPLES: Record<string, unknown> = {
"ldapUserAttr": "",
"ldapUserFilter": "",
"ldapVlessField": "",
"nodeDownThreshold": 1,
"outboundDownThreshold": 1,
"pageSize": 0,
"panelOutbound": "",
@@ -227,6 +228,7 @@ export const EXAMPLES: Record<string, unknown> = {
"ldapUserAttr": "",
"ldapUserFilter": "",
"ldapVlessField": "",
"nodeDownThreshold": 1,
"outboundDownThreshold": 1,
"pageSize": 0,
"panelOutbound": "",
+12
View File
@@ -126,6 +126,11 @@ export const SCHEMAS: Record<string, unknown> = {
"ldapVlessField": {
"type": "string"
},
"nodeDownThreshold": {
"maximum": 100,
"minimum": 1,
"type": "integer"
},
"outboundDownThreshold": {
"maximum": 100,
"minimum": 1,
@@ -613,6 +618,7 @@ export const SCHEMAS: Record<string, unknown> = {
"ldapUserAttr",
"ldapUserFilter",
"ldapVlessField",
"nodeDownThreshold",
"outboundDownThreshold",
"pageSize",
"panelOutbound",
@@ -906,6 +912,11 @@ export const SCHEMAS: Record<string, unknown> = {
"ldapVlessField": {
"type": "string"
},
"nodeDownThreshold": {
"maximum": 100,
"minimum": 1,
"type": "integer"
},
"outboundDownThreshold": {
"maximum": 100,
"minimum": 1,
@@ -1401,6 +1412,7 @@ export const SCHEMAS: Record<string, unknown> = {
"ldapUserAttr",
"ldapUserFilter",
"ldapVlessField",
"nodeDownThreshold",
"outboundDownThreshold",
"pageSize",
"panelOutbound",
+2
View File
@@ -47,6 +47,7 @@ export interface AllSetting {
ldapUserAttr: string;
ldapUserFilter: string;
ldapVlessField: string;
nodeDownThreshold: number;
outboundDownThreshold: number;
pageSize: number;
panelOutbound: string;
@@ -236,6 +237,7 @@ export interface AllSettingView {
ldapUserAttr: string;
ldapUserFilter: string;
ldapVlessField: string;
nodeDownThreshold: number;
outboundDownThreshold: number;
pageSize: number;
panelOutbound: string;
+2
View File
@@ -63,6 +63,7 @@ export const AllSettingSchema = z.object({
ldapUserAttr: z.string(),
ldapUserFilter: z.string(),
ldapVlessField: z.string(),
nodeDownThreshold: z.number().int().min(1).max(100),
outboundDownThreshold: z.number().int().min(1).max(100),
pageSize: z.number().int().min(0).max(1000),
panelOutbound: z.string(),
@@ -253,6 +254,7 @@ export const AllSettingViewSchema = z.object({
ldapUserAttr: z.string(),
ldapUserFilter: z.string(),
ldapVlessField: z.string(),
nodeDownThreshold: z.number().int().min(1).max(100),
outboundDownThreshold: z.number().int().min(1).max(100),
pageSize: z.number().int().min(0).max(1000),
panelOutbound: z.string(),
+5
View File
@@ -174,6 +174,7 @@ export class AllSetting {
smtpCpu = 80;
smtpMemory = 80;
outboundDownThreshold = 3;
nodeDownThreshold = 1;
hasTgBotToken = false;
hasTwoFactorToken = false;
hasLdapPassword = false;
@@ -216,6 +217,10 @@ export class AllSetting {
this.outboundDownThreshold = Number.isFinite(threshold)
? Math.min(100, Math.max(1, threshold))
: 3;
const nodeThreshold = Math.round(Number(this.nodeDownThreshold));
this.nodeDownThreshold = Number.isFinite(nodeThreshold)
? Math.min(100, Math.max(1, nodeThreshold))
: 1;
}
equals(other: AllSetting): boolean {
+1
View File
@@ -39,6 +39,7 @@ export const AllSettingSchema = z
tgBotBackup: z.boolean().optional(),
tgCpu: z.number().int().min(0).max(100).optional(),
outboundDownThreshold: z.number().int().min(1).max(100).optional(),
nodeDownThreshold: z.number().int().min(1).max(100).optional(),
tgLang: z.string().optional(),
twoFactorEnable: z.boolean().optional(),
twoFactorToken: z.string().optional(),