fix: keep panel restarts from hanging on the CPU job and Telegram (#7029)

Restarting the panel left it unreachable for up to a minute. CheckCpuJob sampled cpu.Percent over its whole one-minute period, so cron.Stop nearly always waited on it; it now diffs cpu.Times against its previous run (iowait counted as idle, as before). With Telegram unreachable, bus.Stop waited on an in-flight send and tgbot.Start ran getMe/setMyCommands inline; each bot call now returns once StopBot closes the bot's stop channel, the bot stops before the bus, and the artwork/commands warm-up runs in the background.

internal/web/job/check_cpu_usage.go

@@ -1,25 +1,38 @@
 package job
 
 import (
-	"time"
-
 	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
+	"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
 
 	"github.com/shirou/gopsutil/v4/cpu"
 )
 
 // CheckCpuJob monitors CPU usage and publishes events when threshold is exceeded.
-type CheckCpuJob struct{}
+type CheckCpuJob struct {
+	readTimes func(percpu bool) ([]cpu.TimesStat, error)
+	last      cpu.TimesStat
+	hasLast   bool
+}
 
 // NewCheckCpuJob creates a new CPU monitoring job instance.
 func NewCheckCpuJob() *CheckCpuJob {
-	return new(CheckCpuJob)
+	return &CheckCpuJob{readTimes: cpu.Times}
 }
 
-// Run checks CPU usage and publishes a cpu.high event with raw metric data.
+// Run publishes a cpu.high event with the CPU usage since the previous run. It
+// must not sample in place: cron.Stop, and so every panel restart, waits on it.
 func (j *CheckCpuJob) Run() {
-	percent, err := cpu.Percent(1*time.Minute, false)
-	if err != nil || len(percent) == 0 {
+	times, err := j.readTimes(false)
+	if err != nil || len(times) == 0 {
+		return
+	}
+	prev, hadPrev := j.last, j.hasLast
+	j.last, j.hasLast = times[0], true
+	if !hadPrev {
+		return
+	}
+	percent, ok := sys.CPUWorkPercent(prev, times[0])
+	if !ok {
 		return
 	}
 
@@ -27,7 +40,7 @@ func (j *CheckCpuJob) Run() {
 		EventBus.Publish(eventbus.Event{
 			Type: eventbus.EventCPUHigh,
 			Data: &eventbus.SystemMetricData{
-				Percent: percent[0],
+				Percent: percent,
 			},
 		})
 	}

internal/web/job/check_cpu_usage_test.go

@@ -0,0 +1,104 @@
+package job
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
+
+	"github.com/shirou/gopsutil/v4/cpu"
+)
+
+// The alarm must report usage over the minute between runs, not since boot:
+// cumulative counters alone would hide a spike behind days of idle time.
+func TestCheckCpuJobPublishesUsageSinceThePreviousRun(t *testing.T) {
+	samples := []cpu.TimesStat{
+		{User: 1000, System: 500, Idle: 98500},
+		{User: 1060, System: 515, Idle: 98525}, // 60+15 busy, 25 idle: 75%
+	}
+	j := &CheckCpuJob{readTimes: func(bool) ([]cpu.TimesStat, error) {
+		s := samples[0]
+		samples = samples[1:]
+		return []cpu.TimesStat{s}, nil
+	}}
+
+	bus := eventbus.New(0)
+	t.Cleanup(bus.Stop)
+	got := make(chan float64, 2)
+	bus.Subscribe("test", func(e eventbus.Event) {
+		got <- e.Data.(*eventbus.SystemMetricData).Percent
+	})
+	prevBus := EventBus
+	EventBus = bus
+	t.Cleanup(func() { EventBus = prevBus })
+
+	j.Run()
+	j.Run()
+
+	select {
+	case p := <-got:
+		if p != 75 {
+			t.Fatalf("published %v%%, want 75%% (busy share between the two runs)", p)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("no cpu.high event published by the second run")
+	}
+	select {
+	case p := <-got:
+		t.Fatalf("first run has no baseline yet but published %v%%", p)
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+// gopsutil's cpu.Percent, which the alarm used before, counts iowait as idle; counting it busy
+// would page operators every minute of a backup on a slow-disk host.
+func TestCheckCpuJobDoesNotCountIowaitAsBusy(t *testing.T) {
+	samples := []cpu.TimesStat{
+		{User: 1000, Iowait: 2000, Idle: 97000},
+		{User: 1030, Iowait: 2060, Idle: 97010}, // 30 user, 60 iowait, 10 idle: 30%
+	}
+	j := &CheckCpuJob{readTimes: func(bool) ([]cpu.TimesStat, error) {
+		s := samples[0]
+		samples = samples[1:]
+		return []cpu.TimesStat{s}, nil
+	}}
+
+	bus := eventbus.New(0)
+	t.Cleanup(bus.Stop)
+	got := make(chan float64, 2)
+	bus.Subscribe("test", func(e eventbus.Event) {
+		got <- e.Data.(*eventbus.SystemMetricData).Percent
+	})
+	prevBus := EventBus
+	EventBus = bus
+	t.Cleanup(func() { EventBus = prevBus })
+
+	j.Run()
+	j.Run()
+
+	select {
+	case p := <-got:
+		if p != 30 {
+			t.Fatalf("published %v%%, want 30%% (iowait is waiting, not CPU work)", p)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("no cpu.high event published by the second run")
+	}
+}
+
+// A Run that samples for its whole period keeps cron.Stop waiting, and with it
+// every panel restart, for up to a minute behind a closed listener.
+func TestCheckCpuJobRunDoesNotBlockForTheSamplePeriod(t *testing.T) {
+	j := NewCheckCpuJob()
+	done := make(chan struct{})
+	go func() {
+		j.Run()
+		j.Run()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("CheckCpuJob.Run still blocking after 5s; it must read CPU time since its previous run")
+	}
+}