mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-11-17 18:45:03 +03:00
feat(notifications): add a consecutive-failure threshold for outbound.down alerts (#5968)
Problem: a flaky outbound produces hundreds of false-positive "outbound down" notifications overnight — each fires the moment xray's observatory reports a single failed probe, and the next successful probe fires an "up". applyObservatory forwarded every raw alive:true->false transition straight to EventOutboundDown; xray's observatory has effectively no hysteresis, and nothing on the panel side debounced it (the email/Telegram subscribers are pure formatters). Fix: debounce per outbound. outbound.down now fires only after outboundDownThreshold consecutive FAILED probes (new setting, default 3); outbound.up fires immediately on the first successful probe and only when a down was actually notified. The threshold gates the event itself, so email and Telegram share one knob (exposed next to the outbound.down toggle). The streak counts genuinely new probes (last_try_time advancing), not sampler polls — the sampler runs every 2s but the observatory re-probes per its probeInterval, so counting samples would trip the threshold instantly. outboundDownThreshold=1 reproduces the legacy notify-on-first-failure behaviour. Tuning the observatory's probe interval/timeout is not a workaround: those probes also drive the load balancer's outbound selection, so loosening them to quiet notifications would slow real failover away from a genuinely dead outbound. Notifications don't need observatory-grade latency, so the tolerance belongs at the notification layer, leaving the observatory (and balancer) untouched. Adds TestApplyObservatoryDebounce covering the threshold, probe-vs-sample counting, single-blip suppression and the legacy path. Co-authored-by: Yuriy Khachaturian <y.khachaturian@souzmult.ru> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
committed by
GitHub
parent
9e117bbdd3
commit
2b1308ca29
@@ -0,0 +1,122 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
|
||||
)
|
||||
|
||||
// probe is one observatory sample: whether the outbound is alive and the
|
||||
// last_try_time xray reports for it (a new probe advances lastTry).
|
||||
type probe struct {
|
||||
alive bool
|
||||
lastTry int64
|
||||
}
|
||||
|
||||
const testSentinel eventbus.EventType = "test.sentinel"
|
||||
|
||||
// runObservatory feeds a probe sequence through applyObservatory with the given
|
||||
// threshold and returns the outbound.* events it published, in order.
|
||||
func runObservatory(t *testing.T, threshold int, seq []probe) []eventbus.EventType {
|
||||
t.Helper()
|
||||
|
||||
ss := SettingService{}
|
||||
if err := ss.SetOutboundDownThreshold(threshold); err != nil {
|
||||
t.Fatalf("set threshold: %v", err)
|
||||
}
|
||||
|
||||
bus := eventbus.New(256)
|
||||
events := make(chan eventbus.Event, 256)
|
||||
bus.Subscribe("test", func(e eventbus.Event) { events <- e })
|
||||
SetEventBus(bus)
|
||||
t.Cleanup(func() {
|
||||
SetEventBus(nil)
|
||||
bus.Stop()
|
||||
})
|
||||
|
||||
s := &XrayMetricsService{settingService: ss}
|
||||
for _, p := range seq {
|
||||
s.applyObservatory(time.Unix(p.lastTry, 0), map[string]rawObsEntry{
|
||||
"proxy": {Alive: p.alive, Delay: 10, LastTryTime: p.lastTry, OutboundTag: "proxy"},
|
||||
})
|
||||
}
|
||||
|
||||
bus.Publish(eventbus.Event{Type: testSentinel, Source: "x"})
|
||||
var got []eventbus.EventType
|
||||
for {
|
||||
select {
|
||||
case e := <-events:
|
||||
if e.Type == testSentinel {
|
||||
return got
|
||||
}
|
||||
got = append(got, e.Type)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for events to drain")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyObservatoryDebounce(t *testing.T) {
|
||||
if err := database.InitDB(filepath.Join(t.TempDir(), "x-ui.db")); err != nil {
|
||||
t.Fatalf("init db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = database.CloseDB() })
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
threshold int
|
||||
seq []probe
|
||||
want []eventbus.EventType
|
||||
}{
|
||||
{
|
||||
name: "notifies only after threshold consecutive failed probes",
|
||||
threshold: 3,
|
||||
seq: []probe{
|
||||
{true, 1},
|
||||
{false, 2},
|
||||
{false, 3},
|
||||
{false, 4},
|
||||
{false, 5},
|
||||
{true, 6},
|
||||
{false, 7},
|
||||
{true, 8},
|
||||
},
|
||||
want: []eventbus.EventType{eventbus.EventOutboundDown, eventbus.EventOutboundUp},
|
||||
},
|
||||
{
|
||||
name: "repeated samples of the same probe do not advance the streak",
|
||||
threshold: 3,
|
||||
seq: []probe{{false, 2}, {false, 2}, {false, 2}, {false, 2}, {false, 2}},
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "single-probe blip never notifies",
|
||||
threshold: 3,
|
||||
seq: []probe{{true, 1}, {false, 2}, {true, 3}},
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "threshold 1 keeps the legacy notify-on-first-failure behaviour",
|
||||
threshold: 1,
|
||||
seq: []probe{{true, 1}, {false, 2}},
|
||||
want: []eventbus.EventType{eventbus.EventOutboundDown},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := runObservatory(t, tt.threshold, tt.seq)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("events = %v, want %v", got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("event[%d] = %q, want %q (full: %v)", i, got[i], tt.want[i], got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user