fix(monitoring): read LXC CPU, swap and uptime from the container (#6920)

In an LXC container with LXCFS the Overview showed the container's RAM
beside the host's swap, the host's CPU topology and a CPU percentage from
the host's view. Swap and uptime now come from LXCFS's virtualised files
and CPU from the container's cgroup v2 counter, normalised by the visible
vCPUs and any quota; container CPU and swap history is kept apart from
the earlier host samples.

A value the reader cannot measure (the first CPU sample, or a layout it
cannot verify such as cgroup v1) is reported as unavailable instead of 0:
the dashboard, the node list, node CPU history and the Telegram and
Discord uptime show a dash, and nodes keep the flags in a new
unavailable_metrics column.
This commit is contained in:
VAIO73
2026-10-24 20:24:34 +02:00
committed by GitHub
parent 2c18cbe1cd
commit 746bdbe9f4
34 changed files with 1710 additions and 131 deletions
+1 -4
View File
@@ -487,9 +487,6 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
hostname = "3x-ui"
}
days := status.Uptime / 86400
hours := (status.Uptime % 86400) / 3600
var onlines []string
if process := service.XrayProcess(); process != nil && process.IsRunning() {
onlines = process.GetOnlineClients()
@@ -515,7 +512,7 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
Fields: []EmbedField{
{Name: tr("discord.fields.panelVersion"), Value: config.GetPanelVersion(), Inline: true},
{Name: tr("discord.fields.xrayCore"), Value: fmt.Sprintf("%s (%s)", status.Xray.Version, status.Xray.State), Inline: true},
{Name: tr("pages.index.uptime"), Value: tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours)), Inline: true},
{Name: tr("pages.index.uptime"), Value: formatStatusUptime(status, tr), Inline: true},
{Name: tr("discord.fields.systemLoad"), Value: fmt.Sprintf("%.2f, %.2f, %.2f", load1, load2, load3), Inline: true},
{Name: tr("pages.index.memory"), Value: fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total))), Inline: true},
{Name: tr("pages.index.historyTitleOnline"), Value: strconv.Itoa(len(onlines)), Inline: true},
+9 -3
View File
@@ -102,9 +102,7 @@ func (s *DiscordService) BuildReport(ctx context.Context, server ServerProvider,
}
tr := translator(s.settingService)
days := status.Uptime / 86400
hours := (status.Uptime % 86400) / 3600
uptimeStr := tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours))
uptimeStr := formatStatusUptime(status, tr)
ramStr := fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total)))
trafficStr := tr("discord.values.traffic",
@@ -239,3 +237,11 @@ func getInterfaceIPs() (ipv4, ipv6 string) {
}
return strings.Join(v4s, ", "), strings.Join(v6s, ", ")
}
// formatStatusUptime is shared by scheduled reports and the status command.
func formatStatusUptime(status *service.Status, tr func(string, ...string) string) string {
if !status.MetricAvailable("uptime") {
return "—"
}
return tr("discord.values.uptime", "Days=="+fmt.Sprint(status.Uptime/86400), "Hours=="+fmt.Sprint((status.Uptime%86400)/3600))
}
@@ -229,3 +229,61 @@ func TestSendReport_DeliversEmbedWhenBackupUploadIsRejected(t *testing.T) {
t.Fatal("report embed was never delivered: it rode on the rejected backup upload")
}
}
func TestBuildReportShowsUnavailableUptime(t *testing.T) {
for _, unavailable := range []bool{true, false} {
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
settings := setupTestDB(t)
_ = settings.SetDiscordLang("en-US")
_ = settings.SetDiscordBotBackup(false)
status := &service.Status{Loads: []float64{0, 0, 0}}
if unavailable {
status.UnavailableMetrics = []string{"uptime"}
}
server := &mockServerProvider{status: status}
svc := NewDiscordService(settings)
report, _, err := svc.BuildReport(context.Background(), server, nil)
if err != nil {
t.Fatal(err)
}
want := "0d 0h"
if unavailable {
want = "—"
}
check := func(fields []EmbedField) {
t.Helper()
for _, f := range fields {
if f.Name == "Uptime" {
if f.Value != want {
t.Fatalf("uptime=%q want %q", f.Value, want)
}
return
}
}
t.Fatal("uptime field missing")
}
check(report.Embeds[0].Fields)
_ = settings.SetDiscordBotToken("synthetic-test")
_ = settings.SetDiscordChannelId("synthetic-channel")
sent := make(chan MessagePayload, 1)
rest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload MessagePayload
_ = json.NewDecoder(r.Body).Decode(&payload)
sent <- payload
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"synthetic-message"}`))
}))
defer rest.Close()
svc.SetBaseURL(rest.URL)
svc.SetHTTPClient(rest.Client())
g := NewGatewayClient(svc, settings, server, nil, nil)
g.sendStatus(context.Background())
select {
case msg := <-sent:
check(msg.Embeds[0].Fields)
default:
t.Fatal("no local status payload")
}
})
}
}
+56 -42
View File
@@ -32,15 +32,16 @@ import (
)
type HeartbeatPatch struct {
Status string
LastHeartbeat int64
LatencyMs int
XrayVersion string
PanelVersion string
Guid string
CpuPct float64
MemPct float64
UptimeSecs uint64
Status string
LastHeartbeat int64
LatencyMs int
XrayVersion string
PanelVersion string
Guid string
CpuPct float64
MemPct float64
UptimeSecs uint64
UnavailableMetrics []string
// NetUp/NetDown are the node's current interface throughput (bytes/sec),
// summed over non-virtual interfaces, read from its status response.
NetUp uint64
@@ -978,20 +979,27 @@ func (s *NodeService) UpdatePanels(ids []int, dev bool) ([]NodeUpdateResult, err
func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
db := database.GetDB()
// Map updates bypass GORM field serializers. Encode the availability list
// explicitly so it survives list requests and panel restarts.
unavailable, err := json.Marshal(p.UnavailableMetrics)
if err != nil {
return err
}
updates := map[string]any{
"status": p.Status,
"last_heartbeat": p.LastHeartbeat,
"latency_ms": p.LatencyMs,
"xray_version": p.XrayVersion,
"panel_version": p.PanelVersion,
"cpu_pct": p.CpuPct,
"mem_pct": p.MemPct,
"uptime_secs": p.UptimeSecs,
"net_up": p.NetUp,
"net_down": p.NetDown,
"last_error": p.LastError,
"xray_state": p.XrayState,
"xray_error": p.XrayError,
"unavailable_metrics": string(unavailable),
"status": p.Status,
"last_heartbeat": p.LastHeartbeat,
"latency_ms": p.LatencyMs,
"xray_version": p.XrayVersion,
"panel_version": p.PanelVersion,
"cpu_pct": p.CpuPct,
"mem_pct": p.MemPct,
"uptime_secs": p.UptimeSecs,
"net_up": p.NetUp,
"net_down": p.NetDown,
"last_error": p.LastError,
"xray_state": p.XrayState,
"xray_error": p.XrayError,
}
// Only learn the GUID; never clear a known one if an old-build node (or a
// failed probe) reports none, so the stable identity survives blips.
@@ -1004,7 +1012,9 @@ func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
}
if p.Status == "online" {
now := time.Unix(p.LastHeartbeat, 0)
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
if !slices.Contains(p.UnavailableMetrics, "cpu") {
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
}
nodeMetrics.append(nodeMetricKey(id, "mem"), now, p.MemPct)
nodeMetrics.append(nodeMetricKey(id, "netUp"), now, float64(p.NetUp))
nodeMetrics.append(nodeMetricKey(id, "netDown"), now, float64(p.NetDown))
@@ -1335,8 +1345,9 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
Success bool `json:"success"`
Msg string `json:"msg"`
Obj *struct {
CpuPct float64 `json:"cpu"`
Mem struct {
CpuPct float64 `json:"cpu"`
UnavailableMetrics []string `json:"unavailableMetrics"`
Mem struct {
Current uint64 `json:"current"`
Total uint64 `json:"total"`
} `json:"mem"`
@@ -1370,6 +1381,7 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
}
o := envelope.Obj
patch.CpuPct = o.CpuPct
patch.UnavailableMetrics = o.UnavailableMetrics
if o.Mem.Total > 0 {
patch.MemPct = float64(o.Mem.Current) * 100.0 / float64(o.Mem.Total)
}
@@ -1385,14 +1397,15 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
}
type ProbeResultUI struct {
Status string `json:"status" example:"online"`
LatencyMs int `json:"latencyMs" example:"42"`
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
Error string `json:"error"`
Status string `json:"status" example:"online"`
LatencyMs int `json:"latencyMs" example:"42"`
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
Error string `json:"error"`
// XrayState/XrayError are populated on successful probes even when the node's
// Xray core is not healthy. The UI uses them for a distinct "panel ok, xray failed" indicator.
XrayState string `json:"xrayState"`
@@ -1401,15 +1414,16 @@ type ProbeResultUI struct {
func (p HeartbeatPatch) ToUI(ok bool) ProbeResultUI {
r := ProbeResultUI{
LatencyMs: p.LatencyMs,
XrayVersion: p.XrayVersion,
PanelVersion: p.PanelVersion,
CpuPct: p.CpuPct,
MemPct: p.MemPct,
UptimeSecs: p.UptimeSecs,
Error: FriendlyProbeError(p.LastError),
XrayState: p.XrayState,
XrayError: p.XrayError,
LatencyMs: p.LatencyMs,
XrayVersion: p.XrayVersion,
PanelVersion: p.PanelVersion,
CpuPct: p.CpuPct,
MemPct: p.MemPct,
UptimeSecs: p.UptimeSecs,
UnavailableMetrics: p.UnavailableMetrics,
Error: FriendlyProbeError(p.LastError),
XrayState: p.XrayState,
XrayError: p.XrayError,
}
if ok {
r.Status = "online"
+2
View File
@@ -34,6 +34,7 @@ type NodeView struct {
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
NetUp uint64 `json:"netUp" example:"2097152"`
NetDown uint64 `json:"netDown" example:"1048576"`
LastError string `json:"lastError" example:""`
@@ -82,6 +83,7 @@ func toNodeView(n *model.Node) *NodeView {
CpuPct: n.CpuPct,
MemPct: n.MemPct,
UptimeSecs: n.UptimeSecs,
UnavailableMetrics: n.UnavailableMetrics,
NetUp: n.NetUp,
NetDown: n.NetDown,
LastError: n.LastError,
@@ -2,15 +2,20 @@ package service
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"reflect"
"slices"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
)
@@ -70,3 +75,131 @@ func TestNodeMetricKeysIncludesNet(t *testing.T) {
}
}
}
func availabilityNodeFixture(t *testing.T, body string) (*NodeService, *model.Node, func(string)) {
t.Helper()
dbtest.InitDB(t, filepath.Join(t.TempDir(), "node.db"))
var response atomic.Value
response.Store(body)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(response.Load().(string)))
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
port, err := strconv.Atoi(u.Port())
if err != nil {
t.Fatal(err)
}
n := &model.Node{Name: "sample", Scheme: "http", Address: u.Hostname(), Port: port, BasePath: "/", Enable: true, AllowPrivateAddress: true, TlsVerifyMode: "skip"}
if err := database.GetDB().Create(n).Error; err != nil {
t.Fatal(err)
}
old := nodeMetrics
nodeMetrics = newMetricHistory()
t.Cleanup(func() { nodeMetrics = old })
return &NodeService{}, n, func(s string) { response.Store(s) }
}
func unavailableMetricsOf(t *testing.T, value any) []string {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var out struct {
Unavailable []string `json:"unavailableMetrics"`
}
if err := json.Unmarshal(data, &out); err != nil {
t.Fatal(err)
}
return out.Unavailable
}
const (
unavailableStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"unavailableMetrics":["cpu","uptime"],"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
zeroStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
)
func TestProbeKeepsMetricAvailability(t *testing.T) {
svc, n, _ := availabilityNodeFixture(t, unavailableStatusBody)
patch, err := svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, patch.ToUI(true)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
t.Fatalf("probe availability=%v, want cpu+uptime unavailable", got)
}
}
func TestUpdateHeartbeatKeepsUnavailableCPUOutOfHistory(t *testing.T) {
svc, n, setBody := availabilityNodeFixture(t, unavailableStatusBody)
patch, err := svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
patch.Status = "online"
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
t.Fatal(err)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 0 {
t.Fatalf("unavailable CPU became history: %v", points)
}
var stored model.Node
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, toNodeView(&stored)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
t.Fatalf("stored availability=%v", got)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "mem"), 2, 60); len(points) != 1 || points[0]["v"] != float64(50) {
t.Fatalf("memory history changed: %v", points)
}
setBody(zeroStatusBody)
patch, err = svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
patch.Status = "online"
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
t.Fatal(err)
}
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, toNodeView(&stored)); len(got) != 0 {
t.Fatalf("valid zero did not recover: %v", got)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
t.Fatalf("valid zero lost: %v", points)
}
}
func TestNodeAvailabilityColumnKeepsLegacyRows(t *testing.T) {
path := filepath.Join(t.TempDir(), "legacy.db")
dbtest.InitDB(t, path)
n := &model.Node{Name: "legacy", CpuPct: 0, UptimeSecs: 0}
if err := database.GetDB().Create(n).Error; err != nil {
t.Fatal(err)
}
// Recreate the pre-change schema, then let the normal startup migration run.
if err := database.GetDB().Migrator().DropColumn(&model.Node{}, "UnavailableMetrics"); err != nil {
t.Fatal(err)
}
if err := database.CloseDB(); err != nil {
t.Fatal(err)
}
if err := database.InitDB(path); err != nil {
t.Fatal(err)
}
var stored model.Node
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if stored.Name != "legacy" || stored.CpuPct != 0 || stored.UptimeSecs != 0 || len(stored.UnavailableMetrics) != 0 {
t.Fatalf("legacy row changed: id=%d availability=%v", stored.Id, stored.UnavailableMetrics)
}
}
+71 -56
View File
@@ -66,6 +66,9 @@ const (
// Status represents comprehensive system and application status information.
// It includes CPU, memory, disk, network statistics, and Xray process status.
type Status struct {
ResourceScope string `json:"resourceScope,omitempty"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
T time.Time `json:"-"`
Cpu float64 `json:"cpu"`
CpuCores int `json:"cpuCores"`
@@ -143,6 +146,8 @@ type Release struct {
// ServerService provides business logic for server monitoring and management.
// It handles system status collection, IP detection, and application statistics.
type ServerService struct {
lxcMetrics sys.LXCReader
xrayService XrayService
inboundService InboundService
settingService SettingService
@@ -355,7 +360,7 @@ func (s *ServerService) GetDefaultLogOutboundTags() (freedoms, blackholes []stri
// Kept for back-compat with the original /panel/api/server/cpuHistory/:bucket route;
// the response key is "cpu" (not "v") so legacy consumers parse unchanged.
func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []map[string]any {
out := systemMetrics.aggregate("cpu", bucketSeconds, maxPoints)
out := systemMetrics.aggregate(s.systemMetricKey("cpu"), bucketSeconds, maxPoints)
for _, p := range out {
p["cpu"] = p["v"]
delete(p, "v")
@@ -367,7 +372,7 @@ func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []
// known system metric (see SystemMetricKeys). Output points have keys
// {"t": unixSec, "v": value}; the caller decides how to format the value.
func (s *ServerService) AggregateSystemMetric(metric string, bucketSeconds int, maxPoints int) []map[string]any {
return systemMetrics.aggregate(metric, bucketSeconds, maxPoints)
return systemMetrics.aggregate(s.systemMetricKey(metric), bucketSeconds, maxPoints)
}
type LogEntry struct {
@@ -526,53 +531,61 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
T: now,
}
// CPU stats
util, err := s.sampleCPUUtilization()
if err != nil {
logger.Warning("get cpu percent failed:", err)
scoped, scopeErr := s.lxcMetrics.Read(now)
if scopeErr != nil {
logger.Warning("get scoped metrics failed:", scopeErr)
}
if scoped != nil {
applyLXCStats(status, scoped)
} else {
status.Cpu = util
}
status.CpuCores, err = cpu.Counts(false)
if err != nil {
logger.Warning("get cpu cores count failed:", err)
}
status.LogicalPro = runtime.NumCPU()
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
s.lastCpuInfoAttempt = time.Now()
done := make(chan struct{})
go func() {
defer close(done)
cpuInfos, err := cpu.Info()
if err != nil {
logger.Warning("get cpu info failed:", err)
return
}
if len(cpuInfos) > 0 {
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
} else {
logger.Warning("could not find cpu info")
}
}()
select {
case <-done:
case <-time.After(1500 * time.Millisecond):
logger.Warning("cpu info query timed out; will retry later")
// CPU stats
util, err := s.sampleCPUUtilization()
if err != nil {
logger.Warning("get cpu percent failed:", err)
} else {
status.Cpu = util
}
} else if s.cachedCpuSpeedMhz != 0 {
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
}
// Uptime
upTime, err := host.Uptime()
if err != nil {
logger.Warning("get uptime failed:", err)
} else {
status.Uptime = upTime
status.CpuCores, err = cpu.Counts(false)
if err != nil {
logger.Warning("get cpu cores count failed:", err)
}
status.LogicalPro = runtime.NumCPU()
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
s.lastCpuInfoAttempt = time.Now()
done := make(chan struct{})
go func() {
defer close(done)
cpuInfos, err := cpu.Info()
if err != nil {
logger.Warning("get cpu info failed:", err)
return
}
if len(cpuInfos) > 0 {
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
} else {
logger.Warning("could not find cpu info")
}
}()
select {
case <-done:
case <-time.After(1500 * time.Millisecond):
logger.Warning("cpu info query timed out; will retry later")
}
} else if s.cachedCpuSpeedMhz != 0 {
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
}
// Uptime
upTime, err := host.Uptime()
if err != nil {
logger.Warning("get uptime failed:", err)
} else {
status.Uptime = upTime
}
}
// Memory stats
@@ -584,12 +597,14 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
status.Mem.Total = memInfo.Total
}
swapInfo, err := mem.SwapMemory()
if err != nil {
logger.Warning("get swap memory failed:", err)
} else {
status.Swap.Current = swapInfo.Used
status.Swap.Total = swapInfo.Total
if scoped == nil {
swapInfo, err := mem.SwapMemory()
if err != nil {
logger.Warning("get swap memory failed:", err)
} else {
status.Swap.Current = swapInfo.Used
status.Swap.Total = swapInfo.Total
}
}
// Disk stats
@@ -736,7 +751,7 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
// AppendCpuSample is preserved for callers that only have the CPU number.
// New callers should prefer AppendStatusSample which writes the full set.
func (s *ServerService) AppendCpuSample(t time.Time, v float64) {
systemMetrics.append("cpu", t, v)
systemMetrics.append(s.systemMetricKey("cpu"), t, v)
}
// AppendStatusSample writes one tick of every metric we keep — CPU, memory
@@ -747,14 +762,14 @@ func (s *ServerService) AppendStatusSample(t time.Time, status *Status) {
if status == nil {
return
}
systemMetrics.append("cpu", t, status.Cpu)
appendScopedMetric(status, "cpu", t, status.Cpu)
if status.Mem.Total > 0 {
systemMetrics.append("mem", t, float64(status.Mem.Current)*100.0/float64(status.Mem.Total))
}
if status.Swap.Total > 0 {
systemMetrics.append("swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
appendScopedMetric(status, "swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
} else {
systemMetrics.append("swap", t, 0)
appendScopedMetric(status, "swap", t, 0)
}
systemMetrics.append("netUp", t, float64(status.NetIO.Up))
systemMetrics.append("netDown", t, float64(status.NetIO.Down))
+58
View File
@@ -0,0 +1,58 @@
package service
import (
"slices"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
)
const containerHistoryPrefix = "lxc-v1/"
func scopedMetric(metric string) bool {
return metric == "cpu" || metric == "swap"
}
func (s *ServerService) systemMetricKey(metric string) string {
if !scopedMetric(metric) {
return metric
}
if status := s.LastStatus(); status != nil {
if status.ResourceScope != "" {
return containerHistoryPrefix + metric
}
return metric
}
if s.lxcMetrics.UseScopedHistory() {
return containerHistoryPrefix + metric
}
return metric
}
// Scope metadata applies only to CPU, swap and uptime.
func applyLXCStats(status *Status, sample *sys.LXCStats) {
status.ResourceScope = sample.Scope
status.UnavailableMetrics = append([]string(nil), sample.Unavailable...)
status.Cpu = sample.CPU
status.CpuCores = sample.CPUs
status.LogicalPro = sample.CPUs
status.Swap.Current = sample.SwapUsed
status.Swap.Total = sample.SwapTotal
status.Uptime = sample.Uptime
}
// MetricAvailable distinguishes an unavailable metric from a measured zero.
// Legacy status responses omit metadata and retain their previous semantics.
func (s *Status) MetricAvailable(metric string) bool {
return s != nil && !slices.Contains(s.UnavailableMetrics, metric)
}
func appendScopedMetric(status *Status, metric string, t time.Time, value float64) {
if !status.MetricAvailable(metric) {
return
}
if status.ResourceScope != "" {
metric = containerHistoryPrefix + metric
}
systemMetrics.append(metric, t, value)
}
+145
View File
@@ -0,0 +1,145 @@
//go:build linux
package service
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"slices"
"testing"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
)
func TestContainerStatusOnlyOverridesCPUSwapAndUptime(t *testing.T) {
root := t.TempDir()
t.Setenv("XUI_DB_FOLDER", root)
dbtest.InitDB(t, filepath.Join(root, "test.db"))
sys.SelfRSS()
proc, cg := filepath.Join(root, "proc"), filepath.Join(root, "cg")
fixtures := map[string]string{
filepath.Join(proc, "self/mountinfo"): "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n4 0 0:28 / " + cg + " rw - cgroup2 cgroup rw\n",
filepath.Join(proc, "1/cgroup"): "0::/init.scope\n",
filepath.Join(proc, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n",
filepath.Join(proc, "cpuinfo"): "processor : 0\ncpu cores : 16\nprocessor : 1\ncpu cores : 16\n",
filepath.Join(proc, "uptime"): "52624.93 52624.93\n",
filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/5 1\n",
filepath.Join(proc, "diskstats"): "8 0 vda 1 0 2 0 1 0 4 0 0 0 0\n",
filepath.Join(proc, "stat"): "cpu 100 0 50 1000 0 0 0 0\n",
filepath.Join(proc, "net/dev"): "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n",
filepath.Join(cg, "cpu.stat"): "usage_usec 222772076\n",
filepath.Join(cg, "cpu.max"): "max 100000\n",
filepath.Join(cg, "io.stat"): "259:0 rbytes=999999999 wbytes=999999999\n",
}
for path, content := range fixtures {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
t.Setenv("HOST_PROC", proc)
t.Setenv("HOST_SYS", filepath.Join(root, "sys"))
svc := &ServerService{cachedIPv4: "192.0.2.1", noIPv6: true, cachedCpuSpeedMhz: 5000, lxcMetrics: sys.LXCReader{ProcRoot: proc, CgroupRoot: cg}}
previous := &Status{T: time.Now().Add(-time.Second)}
got := svc.GetStatus(previous)
if got.ResourceScope != "container" || got.CpuCores != 2 || got.LogicalPro != 2 || got.CpuSpeedMhz != 0 || got.Swap.Total != 512<<20 || got.Swap.Current != 19357696 || got.Uptime != 52624 {
t.Fatalf("incorrect corrected resources: %+v", got)
}
if got.Mem.Total != 2<<30 || got.Mem.Current != 128<<20 || !reflect.DeepEqual(got.Loads, []float64{1.25, 2.5, 3.75}) || got.DiskTraffic.Read != 1024 || got.DiskTraffic.Write != 2048 || got.DiskIO.Read == 0 || got.DiskIO.Write == 0 {
t.Fatalf("unrelated metric path changed: %+v", got)
}
if !slices.Equal(got.UnavailableMetrics, []string{"cpu"}) {
t.Fatalf("unexpected unavailable metrics: %v", got.UnavailableMetrics)
}
raw, err := json.Marshal(got)
if err != nil {
t.Fatal(err)
}
var wire map[string]json.RawMessage
if err = json.Unmarshal(raw, &wire); err != nil {
t.Fatal(err)
}
for _, key := range []string{"cpu", "cpuCores", "logicalPro", "swap", "uptime", "mem", "loads", "diskIO", "diskTraffic", "resourceScope", "unavailableMetrics"} {
if _, ok := wire[key]; !ok {
t.Fatalf("API field lost: %s", key)
}
}
if err = os.WriteFile(filepath.Join(proc, "self/mountinfo"), []byte("1 0 0:1 / / rw - ext4 /dev/root rw\n"), 0o600); err != nil {
t.Fatal(err)
}
native := svc.GetStatus(previous)
if native.ResourceScope != "" || len(native.UnavailableMetrics) != 0 || native.CpuSpeedMhz != 5000 || native.Mem != got.Mem || !reflect.DeepEqual(native.Loads, got.Loads) || native.DiskTraffic != got.DiskTraffic {
t.Fatalf("native regression: %+v", native)
}
}
func TestScopedResourcesLeaveUnrelatedStatusUntouched(t *testing.T) {
status := &Status{Loads: []float64{1, 2, 3}, CpuSpeedMhz: 0}
status.Mem.Current, status.Mem.Total = 123, 456
status.DiskTraffic.Read, status.DiskTraffic.Write = 99, 101
status.DiskIO.Read, status.DiskIO.Write = 11, 12
before := *status
applyLXCStats(status, &sys.LXCStats{Scope: "container", CPUs: 2, SwapUsed: 8, SwapTotal: 512, Uptime: 15})
if status.Mem != before.Mem || status.DiskTraffic != before.DiskTraffic || status.DiskIO != before.DiskIO || !reflect.DeepEqual(status.Loads, before.Loads) {
t.Fatalf("unrelated status overwritten: %+v", status)
}
}
func TestContainerHistorySeparatesOnlyCPUAndSwap(t *testing.T) {
old := systemMetrics
systemMetrics = newMetricHistory()
defer func() { systemMetrics = old }()
now := time.Now()
systemMetrics.append("cpu", now, 90)
systemMetrics.append("swap", now, 62)
for key, value := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
systemMetrics.append(key, now, value)
}
status := &Status{T: now, ResourceScope: "container", Cpu: 10, Loads: []float64{1.25, 2.5, 3.75}, UnavailableMetrics: []string{"swap"}}
status.Mem.Total, status.Mem.Current = 100, 6
status.DiskIO.Read, status.NetIO.Up = 1024, 7
svc := &ServerService{lastStatus: status}
svc.AppendStatusSample(now, status)
cpu := svc.AggregateCpuHistory(2, 60)
if len(cpu) != 1 || cpu[0]["cpu"] != float64(10) {
t.Fatalf("CPU history=%v", cpu)
}
native := &ServerService{lastStatus: &Status{}}
if host := native.AggregateCpuHistory(2, 60); len(host) != 1 || host[0]["cpu"] != float64(90) {
t.Fatalf("legacy CPU changed: %v", host)
}
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 0 {
t.Fatalf("missing swap became zero: %v", points)
}
for key, want := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
if svc.systemMetricKey(key) != key {
t.Fatalf("unrelated history renamed: %s", key)
}
points := svc.AggregateSystemMetric(key, 2, 60)
if len(points) != 1 || points[0]["v"] != want {
t.Fatalf("unrelated %s history=%v", key, points)
}
}
status.UnavailableMetrics = nil
svc.AppendStatusSample(now, status)
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
t.Fatalf("disabled swap lost: %v", points)
}
snapshot := systemMetrics.snapshot()
for _, key := range []string{"cpu", "swap", containerHistoryPrefix + "cpu", containerHistoryPrefix + "swap"} {
if _, ok := snapshot[key]; !ok {
t.Fatalf("missing preserved series: %s", key)
}
}
for key := range snapshot {
if len(key) > len(containerHistoryPrefix) && key[:len(containerHistoryPrefix)] == containerHistoryPrefix && key != containerHistoryPrefix+"cpu" && key != containerHistoryPrefix+"swap" {
t.Fatalf("unexpected scoped metric: %s", key)
}
}
}
+6 -1
View File
@@ -187,7 +187,12 @@ func (t *Tgbot) prepareServerUsageInfo() string {
info += t.I18nBot("tgbot.messages.ipv6", "IPv6=="+ipv6)
}
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+strconv.FormatUint(t.lastStatus.Uptime/86400, 10), "Unit=="+t.I18nBot("tgbot.days"))
uptime, uptimeUnit := "—", ""
if t.lastStatus.MetricAvailable("uptime") {
uptime = strconv.FormatUint(t.lastStatus.Uptime/86400, 10)
uptimeUnit = t.I18nBot("tgbot.days")
}
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+uptime, "Unit=="+uptimeUnit)
info += t.I18nBot("tgbot.messages.serverLoad", "Load1=="+strconv.FormatFloat(t.lastStatus.Loads[0], 'f', 2, 64), "Load2=="+strconv.FormatFloat(t.lastStatus.Loads[1], 'f', 2, 64), "Load3=="+strconv.FormatFloat(t.lastStatus.Loads[2], 'f', 2, 64))
info += t.I18nBot("tgbot.messages.serverMemory", "Current=="+common.FormatTraffic(int64(t.lastStatus.Mem.Current)), "Total=="+common.FormatTraffic(int64(t.lastStatus.Mem.Total)))
info += t.I18nBot("tgbot.messages.onlinesCount", "Count=="+fmt.Sprint(len(onlines)))
@@ -14,11 +14,14 @@ import (
"time"
"github.com/mymmrac/telego"
"github.com/nicksnyder/go-i18n/v2/i18n"
"golang.org/x/text/language"
"gorm.io/gorm"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/web/locale"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
"github.com/mhsanaei/3x-ui/v3/internal/xray"
)
@@ -284,3 +287,44 @@ func TestNotifyExhaustedLooksUpEachIDOnce(t *testing.T) {
})
}
}
func TestServerUsageInfoShowsUnavailableUptime(t *testing.T) {
bundle := i18n.NewBundle(language.English)
_ = bundle.AddMessages(language.English, &i18n.Message{ID: "tgbot.messages.serverUpTime", Other: "Uptime: {{.UpTime}} {{.Unit}}\n"}, &i18n.Message{ID: "tgbot.days", Other: "days"})
oldLocale := locale.LocalizerBot
locale.LocalizerBot = i18n.NewLocalizer(bundle, "en")
t.Cleanup(func() { locale.LocalizerBot = oldLocale })
statusCache.mutex.Lock()
oldStatus, oldTime := statusCache.data, statusCache.timestamp
statusCache.mutex.Unlock()
serverStatsCache.mutex.Lock()
oldStats, oldStatsTime := serverStatsCache.data, serverStatsCache.timestamp
serverStatsCache.mutex.Unlock()
t.Cleanup(func() {
statusCache.mutex.Lock()
statusCache.data, statusCache.timestamp = oldStatus, oldTime
statusCache.mutex.Unlock()
serverStatsCache.mutex.Lock()
serverStatsCache.data, serverStatsCache.timestamp = oldStats, oldStatsTime
serverStatsCache.mutex.Unlock()
})
for _, unavailable := range []bool{true, false} {
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
bot := &Tgbot{}
status := &service.Status{Loads: []float64{0, 0, 0}}
if unavailable {
status.UnavailableMetrics = []string{"uptime"}
}
bot.setCachedStatus(status)
bot.setCachedServerStats("")
info := bot.prepareServerUsageInfo()
want := "Uptime: 0 days"
if unavailable {
want = "Uptime: —"
}
if !strings.Contains(info, want) {
t.Fatalf("report did not contain %q: %s", want, info)
}
})
}
}