fix: keep panel restarts from hanging on the CPU job and Telegram (#7029)

Restarting the panel left it unreachable for up to a minute. CheckCpuJob sampled cpu.Percent over its whole one-minute period, so cron.Stop nearly always waited on it; it now diffs cpu.Times against its previous run (iowait counted as idle, as before). With Telegram unreachable, bus.Stop waited on an in-flight send and tgbot.Start ran getMe/setMyCommands inline; each bot call now returns once StopBot closes the bot's stop channel, the bot stops before the bus, and the artwork/commands warm-up runs in the background.

internal/util/sys/cpu_times.go

@@ -0,0 +1,36 @@
+package sys
+
+import "github.com/shirou/gopsutil/v4/cpu"
+
+// CPUBusyPercent is the busy share of CPU time between two cumulative samples,
+// counting iowait as busy; ok is false when no time elapsed or the counters reset.
+func CPUBusyPercent(prev, cur cpu.TimesStat) (percent float64, ok bool) {
+	return cpuPercent(prev, cur, true)
+}
+
+// CPUWorkPercent is CPUBusyPercent with iowait counted as idle, as gopsutil's
+// cpu.Percent does, so a host waiting on disk is not reported as CPU-bound.
+func CPUWorkPercent(prev, cur cpu.TimesStat) (percent float64, ok bool) {
+	return cpuPercent(prev, cur, false)
+}
+
+func cpuPercent(prev, cur cpu.TimesStat, iowaitIsBusy bool) (float64, bool) {
+	// Guest and GuestNice are already counted inside User and Nice on Linux.
+	idle := cur.Idle - prev.Idle
+	busy := (cur.User - prev.User) +
+		(cur.System - prev.System) +
+		(cur.Nice - prev.Nice) +
+		(cur.Irq - prev.Irq) +
+		(cur.Softirq - prev.Softirq) +
+		(cur.Steal - prev.Steal)
+	if iowait := cur.Iowait - prev.Iowait; iowaitIsBusy {
+		busy += iowait
+	} else {
+		idle += iowait
+	}
+	total := busy + idle
+	if total <= 0 {
+		return 0, false
+	}
+	return min(max(100*busy/total, 0), 100), true
+}

internal/web/job/check_cpu_usage.go

@@ -1,25 +1,38 @@
 package job
 
 import (
-	"time"
-
 	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
+	"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
 
 	"github.com/shirou/gopsutil/v4/cpu"
 )
 
 // CheckCpuJob monitors CPU usage and publishes events when threshold is exceeded.
-type CheckCpuJob struct{}
+type CheckCpuJob struct {
+	readTimes func(percpu bool) ([]cpu.TimesStat, error)
+	last      cpu.TimesStat
+	hasLast   bool
+}
 
 // NewCheckCpuJob creates a new CPU monitoring job instance.
 func NewCheckCpuJob() *CheckCpuJob {
-	return new(CheckCpuJob)
+	return &CheckCpuJob{readTimes: cpu.Times}
 }
 
-// Run checks CPU usage and publishes a cpu.high event with raw metric data.
+// Run publishes a cpu.high event with the CPU usage since the previous run. It
+// must not sample in place: cron.Stop, and so every panel restart, waits on it.
 func (j *CheckCpuJob) Run() {
-	percent, err := cpu.Percent(1*time.Minute, false)
-	if err != nil || len(percent) == 0 {
+	times, err := j.readTimes(false)
+	if err != nil || len(times) == 0 {
+		return
+	}
+	prev, hadPrev := j.last, j.hasLast
+	j.last, j.hasLast = times[0], true
+	if !hadPrev {
+		return
+	}
+	percent, ok := sys.CPUWorkPercent(prev, times[0])
+	if !ok {
 		return
 	}
 
@@ -27,7 +40,7 @@ func (j *CheckCpuJob) Run() {
 		EventBus.Publish(eventbus.Event{
 			Type: eventbus.EventCPUHigh,
 			Data: &eventbus.SystemMetricData{
-				Percent: percent[0],
+				Percent: percent,
 			},
 		})
 	}

internal/web/job/check_cpu_usage_test.go

@@ -0,0 +1,104 @@
+package job
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mhsanaei/3x-ui/v3/internal/eventbus"
+
+	"github.com/shirou/gopsutil/v4/cpu"
+)
+
+// The alarm must report usage over the minute between runs, not since boot:
+// cumulative counters alone would hide a spike behind days of idle time.
+func TestCheckCpuJobPublishesUsageSinceThePreviousRun(t *testing.T) {
+	samples := []cpu.TimesStat{
+		{User: 1000, System: 500, Idle: 98500},
+		{User: 1060, System: 515, Idle: 98525}, // 60+15 busy, 25 idle: 75%
+	}
+	j := &CheckCpuJob{readTimes: func(bool) ([]cpu.TimesStat, error) {
+		s := samples[0]
+		samples = samples[1:]
+		return []cpu.TimesStat{s}, nil
+	}}
+
+	bus := eventbus.New(0)
+	t.Cleanup(bus.Stop)
+	got := make(chan float64, 2)
+	bus.Subscribe("test", func(e eventbus.Event) {
+		got <- e.Data.(*eventbus.SystemMetricData).Percent
+	})
+	prevBus := EventBus
+	EventBus = bus
+	t.Cleanup(func() { EventBus = prevBus })
+
+	j.Run()
+	j.Run()
+
+	select {
+	case p := <-got:
+		if p != 75 {
+			t.Fatalf("published %v%%, want 75%% (busy share between the two runs)", p)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("no cpu.high event published by the second run")
+	}
+	select {
+	case p := <-got:
+		t.Fatalf("first run has no baseline yet but published %v%%", p)
+	case <-time.After(100 * time.Millisecond):
+	}
+}
+
+// gopsutil's cpu.Percent, which the alarm used before, counts iowait as idle; counting it busy
+// would page operators every minute of a backup on a slow-disk host.
+func TestCheckCpuJobDoesNotCountIowaitAsBusy(t *testing.T) {
+	samples := []cpu.TimesStat{
+		{User: 1000, Iowait: 2000, Idle: 97000},
+		{User: 1030, Iowait: 2060, Idle: 97010}, // 30 user, 60 iowait, 10 idle: 30%
+	}
+	j := &CheckCpuJob{readTimes: func(bool) ([]cpu.TimesStat, error) {
+		s := samples[0]
+		samples = samples[1:]
+		return []cpu.TimesStat{s}, nil
+	}}
+
+	bus := eventbus.New(0)
+	t.Cleanup(bus.Stop)
+	got := make(chan float64, 2)
+	bus.Subscribe("test", func(e eventbus.Event) {
+		got <- e.Data.(*eventbus.SystemMetricData).Percent
+	})
+	prevBus := EventBus
+	EventBus = bus
+	t.Cleanup(func() { EventBus = prevBus })
+
+	j.Run()
+	j.Run()
+
+	select {
+	case p := <-got:
+		if p != 30 {
+			t.Fatalf("published %v%%, want 30%% (iowait is waiting, not CPU work)", p)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("no cpu.high event published by the second run")
+	}
+}
+
+// A Run that samples for its whole period keeps cron.Stop waiting, and with it
+// every panel restart, for up to a minute behind a closed listener.
+func TestCheckCpuJobRunDoesNotBlockForTheSamplePeriod(t *testing.T) {
+	j := NewCheckCpuJob()
+	done := make(chan struct{})
+	go func() {
+		j.Run()
+		j.Run()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("CheckCpuJob.Run still blocking after 5s; it must read CPU time since its previous run")
+	}
+}

internal/web/service/server.go

@@ -880,36 +880,12 @@ func (s *ServerService) sampleCPUUtilization() (float64, error) {
 		return s.emaCPU, nil
 	}
 
-	// Compute busy and total deltas
-	// Note: Guest and GuestNice times are already included in User and Nice respectively,
-	// so we exclude them to avoid double-counting (Linux kernel accounting)
-	idleDelta := cur.Idle - s.lastCPUTimes.Idle
-	busyDelta := (cur.User - s.lastCPUTimes.User) +
-		(cur.System - s.lastCPUTimes.System) +
-		(cur.Nice - s.lastCPUTimes.Nice) +
-		(cur.Iowait - s.lastCPUTimes.Iowait) +
-		(cur.Irq - s.lastCPUTimes.Irq) +
-		(cur.Softirq - s.lastCPUTimes.Softirq) +
-		(cur.Steal - s.lastCPUTimes.Steal)
-
-	totalDelta := busyDelta + idleDelta
-
-	// Update last sample for next time
+	raw, ok := sys.CPUBusyPercent(s.lastCPUTimes, cur)
 	s.lastCPUTimes = cur
-
-	// Guard against division by zero or negative deltas (e.g., counter resets)
-	if totalDelta <= 0 {
+	if !ok {
 		return s.emaCPU, nil
 	}
 
-	raw := 100.0 * (busyDelta / totalDelta)
-	if raw < 0 {
-		raw = 0
-	}
-	if raw > 100 {
-		raw = 100
-	}
-
 	// Exponential moving average to smooth spikes
 	const alpha = 0.3 // smoothing factor (0<alpha<=1). Higher = more responsive, lower = smoother
 	if s.emaCPU == 0 {

internal/web/service/tgbot/stoppable_caller.go

@@ -0,0 +1,43 @@
+package tgbot
+
+import (
+	"context"
+	"errors"
+
+	ta "github.com/mymmrac/telego/telegoapi"
+)
+
+var errBotStopped = errors.New("telegram bot stopped")
+
+// stoppableCaller returns as soon as its bot is stopped or the call's context
+// ends; fasthttp honours only deadlines, so a blocked route held panel restarts.
+type stoppableCaller struct {
+	inner   ta.Caller
+	stopped <-chan struct{}
+}
+
+type callResult struct {
+	resp *ta.Response
+	err  error
+}
+
+func (c stoppableCaller) Call(ctx context.Context, url string, data *ta.RequestData) (*ta.Response, error) {
+	select {
+	case <-c.stopped:
+		return nil, errBotStopped
+	default:
+	}
+	done := make(chan callResult, 1)
+	go func() {
+		resp, err := c.inner.Call(ctx, url, data)
+		done <- callResult{resp, err}
+	}()
+	select {
+	case r := <-done:
+		return r.resp, r.err
+	case <-c.stopped:
+		return nil, errBotStopped
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}

internal/web/service/tgbot/stoppable_caller_test.go

@@ -0,0 +1,68 @@
+package tgbot
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/mymmrac/telego"
+	ta "github.com/mymmrac/telego/telegoapi"
+	"github.com/valyala/fasthttp"
+)
+
+// With Telegram unreachable a send hangs for its whole fasthttp timeout, and
+// the event bus, so the panel restart behind it, waited that long.
+func TestStopBotEndsInFlightTelegramCall(t *testing.T) {
+	arrived := make(chan struct{}, 1)
+	release := make(chan struct{})
+	hang := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+		arrived <- struct{}{}
+		<-release
+	}))
+	t.Cleanup(func() { close(release); hang.Close() })
+
+	stop := make(chan struct{})
+	b, err := telego.NewBot(testBotToken, telego.WithAPIServer(hang.URL),
+		telego.WithAPICaller(stoppableCaller{inner: ta.FastHTTPCaller{Client: &fasthttp.Client{}}, stopped: stop}))
+	if err != nil {
+		t.Fatalf("NewBot: %v", err)
+	}
+	origBot := liveBot()
+	botMu.Lock()
+	bot, botStop = b, stop
+	botMu.Unlock()
+	t.Cleanup(func() { setTestBot(origBot) })
+
+	errc := make(chan error, 1)
+	go func() {
+		_, err := b.GetMe(context.Background())
+		errc <- err
+	}()
+	select {
+	case <-arrived:
+	case <-time.After(5 * time.Second):
+		t.Fatal("getMe never reached the fake Telegram API")
+	}
+
+	StopBot()
+
+	select {
+	case err := <-errc:
+		if !errors.Is(err, errBotStopped) {
+			t.Fatalf("getMe returned %v, want errBotStopped", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("getMe still blocked 2s after StopBot")
+	}
+	if _, err := b.GetMe(context.Background()); !errors.Is(err, errBotStopped) {
+		t.Fatalf("call after StopBot returned %v, want errBotStopped", err)
+	}
+	select {
+	case <-arrived:
+		t.Fatal("a call after StopBot still went out to Telegram")
+	default:
+	}
+}

internal/web/service/tgbot/tgbot.go

@@ -25,6 +25,7 @@ import (
 	"github.com/mhsanaei/3x-ui/v3/internal/web/service"
 
 	"github.com/mymmrac/telego"
+	ta "github.com/mymmrac/telego/telegoapi"
 	th "github.com/mymmrac/telego/telegohandler"
 	"github.com/valyala/fasthttp"
 	"github.com/valyala/fasthttp/fasthttpproxy"
@@ -35,6 +36,8 @@ var (
 	// botMu guards bot: EnsureBot may replace it while a background goroutine
 	// (the screen artwork warm-up) is still using the previous liveBot().
 	botMu sync.RWMutex
+	// botStop is closed by StopBot to end every in-flight call of the live bot.
+	botStop chan struct{}
 
 	// botCancel stores the function to cancel the context, stopping Long Polling gracefully.
 	botCancel context.CancelFunc
@@ -367,16 +370,24 @@ func (t *Tgbot) Start(i18nFS embed.FS) error {
 	}
 
 	// Create new Telegram bot instance
+	stop := make(chan struct{})
 	botMu.Lock()
-	bot, err = t.NewBot(tgBotToken, tgBotProxy, tgBotAPIServer)
+	bot, err = t.NewBot(tgBotToken, tgBotProxy, tgBotAPIServer, stop)
+	if err == nil {
+		botStop = stop
+	}
 	botMu.Unlock()
 	if err != nil {
 		logger.Error("Failed to initialize Telegram bot API:", err)
 		return err
 	}
 
-	t.resolveScreenArt()
-	t.trySetBotCommands(liveBot())
+	// Off the start path: with Telegram unreachable these block for a minute,
+	// holding the panel's signal loop and the sub server behind them.
+	go func(b *telego.Bot) {
+		t.resolveScreenArt()
+		t.trySetBotCommands(b)
+	}(liveBot())
 
 	// Start receiving Telegram bot messages
 	tgBotMutex.Lock()
@@ -462,7 +473,7 @@ func panelEgressDial(resolve func() string) fasthttp.DialFunc {
 }
 
 // NewBot creates a new Telegram bot instance with optional proxy and API server settings.
-func (t *Tgbot) NewBot(token string, proxyUrl string, apiServerUrl string) (*telego.Bot, error) {
+func (t *Tgbot) NewBot(token string, proxyUrl string, apiServerUrl string, stopped <-chan struct{}) (*telego.Bot, error) {
 	// Validate proxy URL if provided
 	if proxyUrl != "" {
 		if !isSupportedBotProxyScheme(proxyUrl) {
@@ -490,7 +501,10 @@ func (t *Tgbot) NewBot(token string, proxyUrl string, apiServerUrl string) (*tel
 
 	// Build bot options
 	var options []telego.BotOption
-	options = append(options, telego.WithFastHTTPClient(client))
+	options = append(options, telego.WithAPICaller(stoppableCaller{
+		inner:   ta.FastHTTPCaller{Client: client},
+		stopped: stopped,
+	}))
 
 	if apiServerUrl != "" {
 		options = append(options, telego.WithAPIServer(apiServerUrl))
@@ -603,6 +617,13 @@ func StopBot() {
 	isRunning = false
 	tgBotMutex.Unlock()
 
+	botMu.Lock()
+	if botStop != nil {
+		close(botStop)
+		botStop = nil
+	}
+	botMu.Unlock()
+
 	userStateMgr.reset()
 	addClientDrafts.resetAll()
 	broadcastResetAll()

internal/web/web.go

@@ -814,6 +814,10 @@ func (s *Server) stop(stopXray bool, stopTgBot bool) error {
 			err2 = common.Combine(err2, err)
 		}
 	}
+	// Before the bus: its Telegram notifier may sit in a send only StopBot ends.
+	if stopTgBot && s.tgbotService.IsRunning() {
+		s.tgbotService.Stop()
+	}
 	if s.bus != nil {
 		s.bus.Stop()
 	}
@@ -823,9 +827,6 @@ func (s *Server) stop(stopXray bool, stopTgBot bool) error {
 	if stopXray {
 		service.StopTrafficWriter()
 	}
-	if stopTgBot && s.tgbotService.IsRunning() {
-		s.tgbotService.Stop()
-	}
 	if s.discordGateway != nil && s.discordGateway.IsRunning() {
 		s.discordGateway.Stop()
 	}