fix(monitoring): read LXC CPU, swap and uptime from the container (#6920)

In an LXC container with LXCFS the Overview showed the container's RAM
beside the host's swap, the host's CPU topology and a CPU percentage from
the host's view. Swap and uptime now come from LXCFS's virtualised files
and CPU from the container's cgroup v2 counter, normalised by the visible
vCPUs and any quota; container CPU and swap history is kept apart from
the earlier host samples.

A value the reader cannot measure (the first CPU sample, or a layout it
cannot verify such as cgroup v1) is reported as unavailable instead of 0:
the dashboard, the node list, node CPU history and the Telegram and
Discord uptime show a dash, and nodes keep the flags in a new
unavailable_metrics column.
This commit is contained in:
VAIO73
2026-10-24 20:24:34 +02:00
committed by GitHub
parent 2c18cbe1cd
commit 746bdbe9f4
34 changed files with 1710 additions and 131 deletions
+5 -3
View File
@@ -864,9 +864,11 @@ type Node struct {
CpuPct float64 `json:"cpuPct" example:"23.5"`
MemPct float64 `json:"memPct" example:"45.1"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"`
NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"`
LastError string `json:"lastError"`
// Nil on legacy rows and responses: zero remains a valid measured value.
UnavailableMetrics []string `json:"unavailableMetrics,omitempty" gorm:"serializer:json;column:unavailable_metrics"`
NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"`
NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"`
LastError string `json:"lastError"`
// XrayState and XrayError are captured from the remote node's /panel/api/server/status
// during heartbeats. They let the central panel distinguish "panel API reachable"
+10
View File
@@ -0,0 +1,10 @@
package sys
type LXCStats struct {
Scope string
CPU float64
CPUs int
SwapUsed, SwapTotal uint64
Uptime uint64
Unavailable []string
}
+274
View File
@@ -0,0 +1,274 @@
//go:build linux
package sys
import (
"errors"
"fmt"
"math"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
type LXCReader struct {
ProcRoot, CgroupRoot string
mu sync.Mutex
lastCPU uint64
lastAt time.Time
lastCapacity float64
sampled bool
}
func (r *LXCReader) roots() (string, string) {
proc := r.ProcRoot
if proc == "" {
proc = HostProc()
}
cg := r.CgroupRoot
if cg == "" {
cg = "/sys/fs/cgroup"
}
return proc, cg
}
type lxcMounts struct {
sources map[string]bool
cgroupRoot bool
}
func (r *LXCReader) detect() (lxcMounts, error) {
proc, cg := r.roots()
data, err := os.ReadFile(filepath.Join(proc, "self/mountinfo"))
if err != nil {
return lxcMounts{}, err
}
mounts := lxcMounts{sources: map[string]bool{}}
valid := false
for _, line := range strings.Split(string(data), "\n") {
parts := strings.SplitN(line, " - ", 2)
if len(parts) != 2 {
continue
}
mount, fs := strings.Fields(parts[0]), strings.Fields(parts[1])
if len(mount) <= 4 || len(fs) == 0 {
continue
}
valid = true
if fs[0] == "cgroup2" && mount[3] == "/" && mount[4] == cg {
mounts.cgroupRoot = true
}
if fs[0] == "fuse.lxcfs" {
switch mount[4] {
case "/proc/meminfo", "/proc/cpuinfo", "/proc/uptime":
mounts.sources[mount[4]] = true
}
}
}
if !valid {
return lxcMounts{}, errors.New("invalid mountinfo")
}
return mounts, nil
}
func (r *LXCReader) UseScopedHistory() bool {
mounts, err := r.detect()
return len(mounts.sources) > 0 || err != nil
}
func (r *LXCReader) resetCPU() {
r.mu.Lock()
r.sampled = false
r.mu.Unlock()
}
// LXCFS supplies container swap and uptime; sysinfo bypasses those mounts.
func (r *LXCReader) Read(now time.Time) (*LXCStats, error) {
mounts, err := r.detect()
if len(mounts.sources) == 0 && err == nil {
r.resetCPU()
return nil, nil
}
out := &LXCStats{Scope: "container"}
if err != nil {
r.resetCPU()
out.Scope = "unknown"
out.Unavailable = []string{"cpu", "swap", "uptime"}
return out, fmt.Errorf("cannot determine metric scope: %w", err)
}
proc, cg := r.roots()
if mounts.sources["/proc/meminfo"] {
mem, err := readLXCBytes(filepath.Join(proc, "meminfo"))
total, tok := mem["SwapTotal"]
free, fok := mem["SwapFree"]
if err == nil && tok && fok && free <= total {
out.SwapTotal = total
out.SwapUsed = total - free
} else {
out.Unavailable = append(out.Unavailable, "swap")
}
} else {
out.Unavailable = append(out.Unavailable, "swap")
}
if mounts.sources["/proc/uptime"] {
data, err := os.ReadFile(filepath.Join(proc, "uptime"))
fields := strings.Fields(string(data))
if err == nil && len(fields) > 0 {
value, e := strconv.ParseFloat(fields[0], 64)
if e == nil && !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 && value < float64(math.MaxUint64) {
out.Uptime = uint64(value)
} else {
err = errors.New("invalid uptime")
}
} else {
err = errors.New("missing uptime")
}
if err != nil {
out.Unavailable = append(out.Unavailable, "uptime")
}
} else {
out.Unavailable = append(out.Unavailable, "uptime")
}
if !mounts.sources["/proc/cpuinfo"] || !mounts.cgroupRoot || !containerInitAtRoot(proc) {
r.resetCPU()
out.Unavailable = append(out.Unavailable, "cpu")
return out, nil
}
capacity, cpus, err := r.cpuCapacity(proc, cg)
out.CPUs = cpus
usage, usageErr := readUsage(filepath.Join(cg, "cpu.stat"))
r.mu.Lock()
defer r.mu.Unlock()
if err != nil || usageErr != nil {
r.sampled = false
out.Unavailable = append(out.Unavailable, "cpu")
} else {
elapsed := now.Sub(r.lastAt).Seconds()
if r.sampled && elapsed > 0 && usage >= r.lastCPU && capacity == r.lastCapacity {
out.CPU = math.Min(100, float64(usage-r.lastCPU)/1e6/elapsed/capacity*100)
} else {
out.Unavailable = append(out.Unavailable, "cpu")
}
r.lastCPU, r.lastAt, r.lastCapacity, r.sampled = usage, now, capacity, true
}
return out, nil
}
// PID 1 must belong to this namespace root, not a host-visible LXC subtree.
func containerInitAtRoot(proc string) bool {
data, err := os.ReadFile(filepath.Join(proc, "1/cgroup"))
if err != nil {
return false
}
switch strings.TrimSpace(string(data)) {
case "0::/", "0::/init.scope":
return true
default:
return false
}
}
func readLXCBytes(path string) (map[string]uint64, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
wanted := map[string]bool{"SwapTotal": true, "SwapFree": true}
values := map[string]uint64{}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
key := strings.TrimSuffix(fields[0], ":")
if !wanted[key] {
continue
}
if len(fields) != 3 || fields[2] != "kB" {
return nil, errors.New("invalid meminfo unit")
}
if _, exists := values[key]; exists {
return nil, errors.New("duplicate meminfo field")
}
value, e := strconv.ParseUint(fields[1], 10, 64)
if e != nil || value > math.MaxUint64/1024 {
return nil, errors.New("invalid meminfo value")
}
values[key] = value * 1024
}
return values, nil
}
func (r *LXCReader) cpuCapacity(proc, cg string) (float64, int, error) {
data, err := os.ReadFile(filepath.Join(proc, "cpuinfo"))
if err != nil {
return 0, 0, err
}
ids := map[uint64]bool{}
for _, line := range strings.Split(string(data), "\n") {
key, value, ok := strings.Cut(line, ":")
if !ok || strings.TrimSpace(key) != "processor" {
continue
}
id, e := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
if e != nil || ids[id] {
return 0, 0, errors.New("invalid CPU list")
}
ids[id] = true
}
cpus := len(ids)
if cpus == 0 {
return 0, 0, errors.New("empty CPU list")
}
capacity := float64(cpus)
data, err = os.ReadFile(filepath.Join(cg, "cpu.max"))
if err != nil {
return 0, cpus, err
}
fields := strings.Fields(string(data))
if len(fields) != 2 {
return 0, cpus, errors.New("invalid CPU quota")
}
period, e := strconv.ParseUint(fields[1], 10, 64)
if e != nil || period == 0 {
return 0, cpus, errors.New("invalid CPU period")
}
if fields[0] != "max" {
quota, e := strconv.ParseUint(fields[0], 10, 64)
if e != nil || quota == 0 {
return 0, cpus, errors.New("invalid CPU quota")
}
capacity = math.Min(capacity, float64(quota)/float64(period))
}
return capacity, cpus, nil
}
func readUsage(path string) (uint64, error) {
data, err := os.ReadFile(path)
if err != nil {
return 0, err
}
var usage uint64
found := false
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) == 0 || fields[0] != "usage_usec" {
continue
}
if len(fields) != 2 || found {
return 0, errors.New("invalid cgroup CPU usage")
}
usage, err = strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return 0, err
}
found = true
}
if !found {
return 0, errors.New("missing cgroup CPU usage")
}
return usage, nil
}
+274
View File
@@ -0,0 +1,274 @@
//go:build linux
package sys
import (
"math"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
"testing"
"time"
)
const lxcMountFixture = "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n"
func lxcFixture(t *testing.T) *LXCReader {
t.Helper()
root := t.TempDir()
r := &LXCReader{ProcRoot: filepath.Join(root, "proc"), CgroupRoot: filepath.Join(root, "cgroup")}
fixtures := map[string]string{
filepath.Join(r.ProcRoot, "self/mountinfo"): lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n",
filepath.Join(r.ProcRoot, "1/cgroup"): "0::/init.scope\n",
filepath.Join(r.ProcRoot, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n",
filepath.Join(r.ProcRoot, "cpuinfo"): "processor : 0\ncpu cores : 16\n\nprocessor : 1\ncpu cores : 16\n",
filepath.Join(r.ProcRoot, "uptime"): "52624.93 52624.93\n",
filepath.Join(r.CgroupRoot, "cpu.stat"): "usage_usec 222772076\nuser_usec 101515374\n",
filepath.Join(r.CgroupRoot, "cpu.max"): "max 100000\n",
}
for path, content := range fixtures {
fixtureWrite(t, path, content)
}
return r
}
func fixtureWrite(t *testing.T, path, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
func TestLXCResourcesIgnoreHostTopologyAndInheritedMax(t *testing.T) {
r := lxcFixture(t)
got, err := r.Read(time.Unix(100, 0))
if err != nil || got == nil {
t.Fatalf("sample=%+v err=%v", got, err)
}
if got.CPUs != 2 || got.SwapTotal != 512<<20 || got.SwapUsed != 19357696 || got.Uptime != 52624 {
t.Fatalf("wrong container resources: %+v", got)
}
if !slices.Equal(got.Unavailable, []string{"cpu"}) {
t.Fatalf("first sample=%+v", got)
}
}
func TestLXCCPUUsesWholeContainerDeltaAndCapacity(t *testing.T) {
for _, tc := range []struct {
name, quota string
delta uint64
want float64
}{
{"one CPU of two", "max 100000\n", 1000000, 50},
{"idle is a valid zero", "max 100000\n", 0, 0},
{"half CPU quota", "50000 100000\n", 125000, 25},
{"quota above visible CPUs", "800000 100000\n", 1000000, 50},
{"saturated interval bounded", "max 100000\n", 2100000, 100},
} {
t.Run(tc.name, func(t *testing.T) {
r := lxcFixture(t)
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), tc.quota)
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
r.Read(time.Unix(100, 0))
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(1000000+tc.delta, 10)+"\n")
got, err := r.Read(time.Unix(101, 0))
if err != nil || got.CPU != tc.want || slices.Contains(got.Unavailable, "cpu") {
t.Fatalf("sample=%+v err=%v want=%v", got, err, tc.want)
}
})
}
}
func TestLXCCounterClockAndCapacityChangesResetBaseline(t *testing.T) {
for _, scenario := range []string{"counter reset", "zero interval", "backward interval", "capacity change", "missing counter", "scope change"} {
t.Run(scenario, func(t *testing.T) {
r := lxcFixture(t)
r.Read(time.Unix(100, 0))
at := time.Unix(101, 0)
switch scenario {
case "counter reset":
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1\n")
case "zero interval":
at = time.Unix(100, 0)
case "backward interval":
at = time.Unix(99, 0)
case "capacity change":
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), "100000 100000\n")
case "missing counter":
os.Remove(filepath.Join(r.CgroupRoot, "cpu.stat"))
r.Read(at)
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 222772076\n")
case "scope change":
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n")
r.Read(at)
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), lxcMountFixture+"4 0 0:28 / "+r.CgroupRoot+" rw - cgroup2 cgroup rw\n")
}
got, err := r.Read(at)
if err != nil || !slices.Contains(got.Unavailable, "cpu") {
t.Fatalf("invalid interval=%+v err=%v", got, err)
}
usage, _ := readUsage(filepath.Join(r.CgroupRoot, "cpu.stat"))
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(usage+500000, 10)+"\n")
got, err = r.Read(at.Add(time.Second))
want := 25.0
if scenario == "capacity change" {
want = 50
}
if err != nil || got.CPU != want || slices.Contains(got.Unavailable, "cpu") {
t.Fatalf("recovery=%+v err=%v", got, err)
}
})
}
}
func TestLXCInvalidMetricsNeverUseHostFallback(t *testing.T) {
for _, tc := range []struct{ path, content, metric string }{
{"meminfo", "SwapTotal: 1 kB\nSwapFree: 2 kB\n", "swap"},
{"meminfo", "SwapTotal: 18446744073709551615 kB\nSwapFree: 0 kB\n", "swap"},
{"meminfo", "SwapTotal: 1 MB\nSwapFree: 0 kB\n", "swap"},
{"meminfo", "SwapTotal: 1 kB\nSwapTotal: 1 kB\nSwapFree: 0 kB\n", "swap"},
{"meminfo", "SwapTotal: 1 kB\n", "swap"},
{"uptime", "NaN 0\n", "uptime"},
{"uptime", "+Inf 0\n", "uptime"},
{"uptime", "-1 0\n", "uptime"},
{"uptime", "18446744073709551616 0\n", "uptime"},
{"uptime", "", "uptime"},
{"cpuinfo", "cpu cores : 16\n", "cpu"},
{"cpuinfo", "processor : 0\nprocessor : 0\n", "cpu"},
} {
t.Run(tc.metric+"/"+tc.content, func(t *testing.T) {
r := lxcFixture(t)
fixtureWrite(t, filepath.Join(r.ProcRoot, tc.path), tc.content)
got, err := r.Read(time.Unix(100, 0))
if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) {
t.Fatalf("invalid sample=%+v err=%v", got, err)
}
for _, v := range got.Unavailable {
if v != "cpu" && v != "swap" && v != "uptime" {
t.Fatalf("unrelated metric changed: %s", v)
}
}
})
}
}
func TestLXCCPURejectsMalformedCgroupData(t *testing.T) {
for _, tc := range []struct{ path, value string }{
{"cpu.max", "max 0"},
{"cpu.max", "0 100000"},
{"cpu.max", "NaN 100000"},
{"cpu.max", "max"},
{"cpu.stat", "user_usec 123"},
{"cpu.stat", "usage_usec -1"},
{"cpu.stat", "usage_usec 1\nusage_usec 2"},
} {
t.Run(tc.path+"/"+tc.value, func(t *testing.T) {
r := lxcFixture(t)
r.Read(time.Unix(100, 0))
fixtureWrite(t, filepath.Join(r.CgroupRoot, tc.path), tc.value)
got, err := r.Read(time.Unix(101, 0))
if err != nil || !slices.Contains(got.Unavailable, "cpu") || math.IsNaN(got.CPU) || math.IsInf(got.CPU, 0) {
t.Fatalf("bad CPU=%+v err=%v", got, err)
}
})
}
}
func TestLXCZeroSwapIsValidAndRAMIsNotParsed(t *testing.T) {
r := lxcFixture(t)
fixtureWrite(t, filepath.Join(r.ProcRoot, "meminfo"), "MemTotal: not-a-number kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n")
got, err := r.Read(time.Unix(100, 0))
if err != nil || got.SwapTotal != 0 || got.SwapUsed != 0 || slices.Contains(got.Unavailable, "swap") {
t.Fatalf("disabled swap=%+v err=%v", got, err)
}
}
func TestLXCEachProcSourceRequiresItsOwnMount(t *testing.T) {
for _, tc := range []struct{ path, metric string }{{"meminfo", "swap"}, {"cpuinfo", "cpu"}, {"uptime", "uptime"}} {
t.Run(tc.path, func(t *testing.T) {
r := lxcFixture(t)
var lines []string
for _, line := range strings.Split(lxcMountFixture, "\n") {
if !strings.Contains(line, "/proc/"+tc.path) {
lines = append(lines, line)
}
}
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), strings.Join(lines, "\n"))
got, err := r.Read(time.Unix(100, 0))
if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) {
t.Fatalf("unverified mount=%+v err=%v", got, err)
}
})
}
}
func TestNativeMetricsKeepOriginalPath(t *testing.T) {
r := lxcFixture(t)
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n")
os.RemoveAll(r.CgroupRoot)
got, err := r.Read(time.Unix(100, 0))
if err != nil || got != nil || r.UseScopedHistory() {
t.Fatalf("native=%+v err=%v", got, err)
}
}
func TestUnreadableScopeNeverSelectsNativeHostMetrics(t *testing.T) {
for _, bad := range []string{"missing", "malformed"} {
t.Run(bad, func(t *testing.T) {
r := lxcFixture(t)
if bad == "missing" {
os.Remove(filepath.Join(r.ProcRoot, "self/mountinfo"))
} else {
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "invalid")
}
got, err := r.Read(time.Unix(100, 0))
if err == nil || got == nil || got.Scope != "unknown" || !slices.Equal(got.Unavailable, []string{"cpu", "swap", "uptime"}) || !r.UseScopedHistory() {
t.Fatalf("unknown=%+v err=%v", got, err)
}
})
}
}
func TestLXCCPURejectsUnverifiedCgroupRoot(t *testing.T) {
for _, scenario := range []string{"missing mount", "subtree mount", "legacy controller", "host namespace", "missing init membership"} {
t.Run(scenario, func(t *testing.T) {
r := lxcFixture(t)
r.Read(time.Unix(100, 0))
mounts := lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n"
switch scenario {
case "missing mount":
mounts = lxcMountFixture
case "subtree mount":
mounts = lxcMountFixture + "4 0 0:28 /system.slice " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n"
case "legacy controller":
mounts = lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup cgroup rw\n"
case "host namespace":
fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/lxc/example/init.scope\n")
case "missing init membership":
os.Remove(filepath.Join(r.ProcRoot, "1/cgroup"))
}
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), mounts)
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n")
got, err := r.Read(time.Unix(101, 0))
if err != nil || got == nil || !slices.Contains(got.Unavailable, "cpu") || got.CPU != 0 || got.SwapTotal != 512<<20 || got.Uptime != 52624 {
t.Fatalf("unverified cgroup leaked CPU or changed proc metrics: %+v err=%v", got, err)
}
})
}
}
func TestLXCCPUAcceptsContainerInitAtNamespaceRoot(t *testing.T) {
r := lxcFixture(t)
fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/\n")
r.Read(time.Unix(100, 0))
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n")
got, err := r.Read(time.Unix(101, 0))
if err != nil || got.CPU != 50 || slices.Contains(got.Unavailable, "cpu") {
t.Fatalf("root init sample=%+v err=%v", got, err)
}
}
+10
View File
@@ -0,0 +1,10 @@
//go:build !linux
package sys
import "time"
type LXCReader struct{}
func (r *LXCReader) Read(now time.Time) (*LXCStats, error) { return nil, nil }
func (r *LXCReader) UseScopedHistory() bool { return false }
+1 -4
View File
@@ -487,9 +487,6 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
hostname = "3x-ui"
}
days := status.Uptime / 86400
hours := (status.Uptime % 86400) / 3600
var onlines []string
if process := service.XrayProcess(); process != nil && process.IsRunning() {
onlines = process.GetOnlineClients()
@@ -515,7 +512,7 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
Fields: []EmbedField{
{Name: tr("discord.fields.panelVersion"), Value: config.GetPanelVersion(), Inline: true},
{Name: tr("discord.fields.xrayCore"), Value: fmt.Sprintf("%s (%s)", status.Xray.Version, status.Xray.State), Inline: true},
{Name: tr("pages.index.uptime"), Value: tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours)), Inline: true},
{Name: tr("pages.index.uptime"), Value: formatStatusUptime(status, tr), Inline: true},
{Name: tr("discord.fields.systemLoad"), Value: fmt.Sprintf("%.2f, %.2f, %.2f", load1, load2, load3), Inline: true},
{Name: tr("pages.index.memory"), Value: fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total))), Inline: true},
{Name: tr("pages.index.historyTitleOnline"), Value: strconv.Itoa(len(onlines)), Inline: true},
+9 -3
View File
@@ -102,9 +102,7 @@ func (s *DiscordService) BuildReport(ctx context.Context, server ServerProvider,
}
tr := translator(s.settingService)
days := status.Uptime / 86400
hours := (status.Uptime % 86400) / 3600
uptimeStr := tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours))
uptimeStr := formatStatusUptime(status, tr)
ramStr := fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total)))
trafficStr := tr("discord.values.traffic",
@@ -239,3 +237,11 @@ func getInterfaceIPs() (ipv4, ipv6 string) {
}
return strings.Join(v4s, ", "), strings.Join(v6s, ", ")
}
// formatStatusUptime is shared by scheduled reports and the status command.
func formatStatusUptime(status *service.Status, tr func(string, ...string) string) string {
if !status.MetricAvailable("uptime") {
return "—"
}
return tr("discord.values.uptime", "Days=="+fmt.Sprint(status.Uptime/86400), "Hours=="+fmt.Sprint((status.Uptime%86400)/3600))
}
@@ -229,3 +229,61 @@ func TestSendReport_DeliversEmbedWhenBackupUploadIsRejected(t *testing.T) {
t.Fatal("report embed was never delivered: it rode on the rejected backup upload")
}
}
func TestBuildReportShowsUnavailableUptime(t *testing.T) {
for _, unavailable := range []bool{true, false} {
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
settings := setupTestDB(t)
_ = settings.SetDiscordLang("en-US")
_ = settings.SetDiscordBotBackup(false)
status := &service.Status{Loads: []float64{0, 0, 0}}
if unavailable {
status.UnavailableMetrics = []string{"uptime"}
}
server := &mockServerProvider{status: status}
svc := NewDiscordService(settings)
report, _, err := svc.BuildReport(context.Background(), server, nil)
if err != nil {
t.Fatal(err)
}
want := "0d 0h"
if unavailable {
want = "—"
}
check := func(fields []EmbedField) {
t.Helper()
for _, f := range fields {
if f.Name == "Uptime" {
if f.Value != want {
t.Fatalf("uptime=%q want %q", f.Value, want)
}
return
}
}
t.Fatal("uptime field missing")
}
check(report.Embeds[0].Fields)
_ = settings.SetDiscordBotToken("synthetic-test")
_ = settings.SetDiscordChannelId("synthetic-channel")
sent := make(chan MessagePayload, 1)
rest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload MessagePayload
_ = json.NewDecoder(r.Body).Decode(&payload)
sent <- payload
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"synthetic-message"}`))
}))
defer rest.Close()
svc.SetBaseURL(rest.URL)
svc.SetHTTPClient(rest.Client())
g := NewGatewayClient(svc, settings, server, nil, nil)
g.sendStatus(context.Background())
select {
case msg := <-sent:
check(msg.Embeds[0].Fields)
default:
t.Fatal("no local status payload")
}
})
}
}
+56 -42
View File
@@ -32,15 +32,16 @@ import (
)
type HeartbeatPatch struct {
Status string
LastHeartbeat int64
LatencyMs int
XrayVersion string
PanelVersion string
Guid string
CpuPct float64
MemPct float64
UptimeSecs uint64
Status string
LastHeartbeat int64
LatencyMs int
XrayVersion string
PanelVersion string
Guid string
CpuPct float64
MemPct float64
UptimeSecs uint64
UnavailableMetrics []string
// NetUp/NetDown are the node's current interface throughput (bytes/sec),
// summed over non-virtual interfaces, read from its status response.
NetUp uint64
@@ -978,20 +979,27 @@ func (s *NodeService) UpdatePanels(ids []int, dev bool) ([]NodeUpdateResult, err
func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
db := database.GetDB()
// Map updates bypass GORM field serializers. Encode the availability list
// explicitly so it survives list requests and panel restarts.
unavailable, err := json.Marshal(p.UnavailableMetrics)
if err != nil {
return err
}
updates := map[string]any{
"status": p.Status,
"last_heartbeat": p.LastHeartbeat,
"latency_ms": p.LatencyMs,
"xray_version": p.XrayVersion,
"panel_version": p.PanelVersion,
"cpu_pct": p.CpuPct,
"mem_pct": p.MemPct,
"uptime_secs": p.UptimeSecs,
"net_up": p.NetUp,
"net_down": p.NetDown,
"last_error": p.LastError,
"xray_state": p.XrayState,
"xray_error": p.XrayError,
"unavailable_metrics": string(unavailable),
"status": p.Status,
"last_heartbeat": p.LastHeartbeat,
"latency_ms": p.LatencyMs,
"xray_version": p.XrayVersion,
"panel_version": p.PanelVersion,
"cpu_pct": p.CpuPct,
"mem_pct": p.MemPct,
"uptime_secs": p.UptimeSecs,
"net_up": p.NetUp,
"net_down": p.NetDown,
"last_error": p.LastError,
"xray_state": p.XrayState,
"xray_error": p.XrayError,
}
// Only learn the GUID; never clear a known one if an old-build node (or a
// failed probe) reports none, so the stable identity survives blips.
@@ -1004,7 +1012,9 @@ func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
}
if p.Status == "online" {
now := time.Unix(p.LastHeartbeat, 0)
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
if !slices.Contains(p.UnavailableMetrics, "cpu") {
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
}
nodeMetrics.append(nodeMetricKey(id, "mem"), now, p.MemPct)
nodeMetrics.append(nodeMetricKey(id, "netUp"), now, float64(p.NetUp))
nodeMetrics.append(nodeMetricKey(id, "netDown"), now, float64(p.NetDown))
@@ -1335,8 +1345,9 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
Success bool `json:"success"`
Msg string `json:"msg"`
Obj *struct {
CpuPct float64 `json:"cpu"`
Mem struct {
CpuPct float64 `json:"cpu"`
UnavailableMetrics []string `json:"unavailableMetrics"`
Mem struct {
Current uint64 `json:"current"`
Total uint64 `json:"total"`
} `json:"mem"`
@@ -1370,6 +1381,7 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
}
o := envelope.Obj
patch.CpuPct = o.CpuPct
patch.UnavailableMetrics = o.UnavailableMetrics
if o.Mem.Total > 0 {
patch.MemPct = float64(o.Mem.Current) * 100.0 / float64(o.Mem.Total)
}
@@ -1385,14 +1397,15 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
}
type ProbeResultUI struct {
Status string `json:"status" example:"online"`
LatencyMs int `json:"latencyMs" example:"42"`
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
Error string `json:"error"`
Status string `json:"status" example:"online"`
LatencyMs int `json:"latencyMs" example:"42"`
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
Error string `json:"error"`
// XrayState/XrayError are populated on successful probes even when the node's
// Xray core is not healthy. The UI uses them for a distinct "panel ok, xray failed" indicator.
XrayState string `json:"xrayState"`
@@ -1401,15 +1414,16 @@ type ProbeResultUI struct {
func (p HeartbeatPatch) ToUI(ok bool) ProbeResultUI {
r := ProbeResultUI{
LatencyMs: p.LatencyMs,
XrayVersion: p.XrayVersion,
PanelVersion: p.PanelVersion,
CpuPct: p.CpuPct,
MemPct: p.MemPct,
UptimeSecs: p.UptimeSecs,
Error: FriendlyProbeError(p.LastError),
XrayState: p.XrayState,
XrayError: p.XrayError,
LatencyMs: p.LatencyMs,
XrayVersion: p.XrayVersion,
PanelVersion: p.PanelVersion,
CpuPct: p.CpuPct,
MemPct: p.MemPct,
UptimeSecs: p.UptimeSecs,
UnavailableMetrics: p.UnavailableMetrics,
Error: FriendlyProbeError(p.LastError),
XrayState: p.XrayState,
XrayError: p.XrayError,
}
if ok {
r.Status = "online"
+2
View File
@@ -34,6 +34,7 @@ type NodeView struct {
CpuPct float64 `json:"cpuPct" example:"12.5"`
MemPct float64 `json:"memPct" example:"45.2"`
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
NetUp uint64 `json:"netUp" example:"2097152"`
NetDown uint64 `json:"netDown" example:"1048576"`
LastError string `json:"lastError" example:""`
@@ -82,6 +83,7 @@ func toNodeView(n *model.Node) *NodeView {
CpuPct: n.CpuPct,
MemPct: n.MemPct,
UptimeSecs: n.UptimeSecs,
UnavailableMetrics: n.UnavailableMetrics,
NetUp: n.NetUp,
NetDown: n.NetDown,
LastError: n.LastError,
@@ -2,15 +2,20 @@ package service
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"reflect"
"slices"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
)
@@ -70,3 +75,131 @@ func TestNodeMetricKeysIncludesNet(t *testing.T) {
}
}
}
func availabilityNodeFixture(t *testing.T, body string) (*NodeService, *model.Node, func(string)) {
t.Helper()
dbtest.InitDB(t, filepath.Join(t.TempDir(), "node.db"))
var response atomic.Value
response.Store(body)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(response.Load().(string)))
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
port, err := strconv.Atoi(u.Port())
if err != nil {
t.Fatal(err)
}
n := &model.Node{Name: "sample", Scheme: "http", Address: u.Hostname(), Port: port, BasePath: "/", Enable: true, AllowPrivateAddress: true, TlsVerifyMode: "skip"}
if err := database.GetDB().Create(n).Error; err != nil {
t.Fatal(err)
}
old := nodeMetrics
nodeMetrics = newMetricHistory()
t.Cleanup(func() { nodeMetrics = old })
return &NodeService{}, n, func(s string) { response.Store(s) }
}
func unavailableMetricsOf(t *testing.T, value any) []string {
t.Helper()
data, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
var out struct {
Unavailable []string `json:"unavailableMetrics"`
}
if err := json.Unmarshal(data, &out); err != nil {
t.Fatal(err)
}
return out.Unavailable
}
const (
unavailableStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"unavailableMetrics":["cpu","uptime"],"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
zeroStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
)
func TestProbeKeepsMetricAvailability(t *testing.T) {
svc, n, _ := availabilityNodeFixture(t, unavailableStatusBody)
patch, err := svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, patch.ToUI(true)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
t.Fatalf("probe availability=%v, want cpu+uptime unavailable", got)
}
}
func TestUpdateHeartbeatKeepsUnavailableCPUOutOfHistory(t *testing.T) {
svc, n, setBody := availabilityNodeFixture(t, unavailableStatusBody)
patch, err := svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
patch.Status = "online"
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
t.Fatal(err)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 0 {
t.Fatalf("unavailable CPU became history: %v", points)
}
var stored model.Node
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, toNodeView(&stored)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
t.Fatalf("stored availability=%v", got)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "mem"), 2, 60); len(points) != 1 || points[0]["v"] != float64(50) {
t.Fatalf("memory history changed: %v", points)
}
setBody(zeroStatusBody)
patch, err = svc.Probe(context.Background(), n)
if err != nil {
t.Fatal(err)
}
patch.Status = "online"
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
t.Fatal(err)
}
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if got := unavailableMetricsOf(t, toNodeView(&stored)); len(got) != 0 {
t.Fatalf("valid zero did not recover: %v", got)
}
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
t.Fatalf("valid zero lost: %v", points)
}
}
func TestNodeAvailabilityColumnKeepsLegacyRows(t *testing.T) {
path := filepath.Join(t.TempDir(), "legacy.db")
dbtest.InitDB(t, path)
n := &model.Node{Name: "legacy", CpuPct: 0, UptimeSecs: 0}
if err := database.GetDB().Create(n).Error; err != nil {
t.Fatal(err)
}
// Recreate the pre-change schema, then let the normal startup migration run.
if err := database.GetDB().Migrator().DropColumn(&model.Node{}, "UnavailableMetrics"); err != nil {
t.Fatal(err)
}
if err := database.CloseDB(); err != nil {
t.Fatal(err)
}
if err := database.InitDB(path); err != nil {
t.Fatal(err)
}
var stored model.Node
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
t.Fatal(err)
}
if stored.Name != "legacy" || stored.CpuPct != 0 || stored.UptimeSecs != 0 || len(stored.UnavailableMetrics) != 0 {
t.Fatalf("legacy row changed: id=%d availability=%v", stored.Id, stored.UnavailableMetrics)
}
}
+71 -56
View File
@@ -66,6 +66,9 @@ const (
// Status represents comprehensive system and application status information.
// It includes CPU, memory, disk, network statistics, and Xray process status.
type Status struct {
ResourceScope string `json:"resourceScope,omitempty"`
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
T time.Time `json:"-"`
Cpu float64 `json:"cpu"`
CpuCores int `json:"cpuCores"`
@@ -143,6 +146,8 @@ type Release struct {
// ServerService provides business logic for server monitoring and management.
// It handles system status collection, IP detection, and application statistics.
type ServerService struct {
lxcMetrics sys.LXCReader
xrayService XrayService
inboundService InboundService
settingService SettingService
@@ -355,7 +360,7 @@ func (s *ServerService) GetDefaultLogOutboundTags() (freedoms, blackholes []stri
// Kept for back-compat with the original /panel/api/server/cpuHistory/:bucket route;
// the response key is "cpu" (not "v") so legacy consumers parse unchanged.
func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []map[string]any {
out := systemMetrics.aggregate("cpu", bucketSeconds, maxPoints)
out := systemMetrics.aggregate(s.systemMetricKey("cpu"), bucketSeconds, maxPoints)
for _, p := range out {
p["cpu"] = p["v"]
delete(p, "v")
@@ -367,7 +372,7 @@ func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []
// known system metric (see SystemMetricKeys). Output points have keys
// {"t": unixSec, "v": value}; the caller decides how to format the value.
func (s *ServerService) AggregateSystemMetric(metric string, bucketSeconds int, maxPoints int) []map[string]any {
return systemMetrics.aggregate(metric, bucketSeconds, maxPoints)
return systemMetrics.aggregate(s.systemMetricKey(metric), bucketSeconds, maxPoints)
}
type LogEntry struct {
@@ -526,53 +531,61 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
T: now,
}
// CPU stats
util, err := s.sampleCPUUtilization()
if err != nil {
logger.Warning("get cpu percent failed:", err)
scoped, scopeErr := s.lxcMetrics.Read(now)
if scopeErr != nil {
logger.Warning("get scoped metrics failed:", scopeErr)
}
if scoped != nil {
applyLXCStats(status, scoped)
} else {
status.Cpu = util
}
status.CpuCores, err = cpu.Counts(false)
if err != nil {
logger.Warning("get cpu cores count failed:", err)
}
status.LogicalPro = runtime.NumCPU()
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
s.lastCpuInfoAttempt = time.Now()
done := make(chan struct{})
go func() {
defer close(done)
cpuInfos, err := cpu.Info()
if err != nil {
logger.Warning("get cpu info failed:", err)
return
}
if len(cpuInfos) > 0 {
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
} else {
logger.Warning("could not find cpu info")
}
}()
select {
case <-done:
case <-time.After(1500 * time.Millisecond):
logger.Warning("cpu info query timed out; will retry later")
// CPU stats
util, err := s.sampleCPUUtilization()
if err != nil {
logger.Warning("get cpu percent failed:", err)
} else {
status.Cpu = util
}
} else if s.cachedCpuSpeedMhz != 0 {
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
}
// Uptime
upTime, err := host.Uptime()
if err != nil {
logger.Warning("get uptime failed:", err)
} else {
status.Uptime = upTime
status.CpuCores, err = cpu.Counts(false)
if err != nil {
logger.Warning("get cpu cores count failed:", err)
}
status.LogicalPro = runtime.NumCPU()
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
s.lastCpuInfoAttempt = time.Now()
done := make(chan struct{})
go func() {
defer close(done)
cpuInfos, err := cpu.Info()
if err != nil {
logger.Warning("get cpu info failed:", err)
return
}
if len(cpuInfos) > 0 {
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
} else {
logger.Warning("could not find cpu info")
}
}()
select {
case <-done:
case <-time.After(1500 * time.Millisecond):
logger.Warning("cpu info query timed out; will retry later")
}
} else if s.cachedCpuSpeedMhz != 0 {
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
}
// Uptime
upTime, err := host.Uptime()
if err != nil {
logger.Warning("get uptime failed:", err)
} else {
status.Uptime = upTime
}
}
// Memory stats
@@ -584,12 +597,14 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
status.Mem.Total = memInfo.Total
}
swapInfo, err := mem.SwapMemory()
if err != nil {
logger.Warning("get swap memory failed:", err)
} else {
status.Swap.Current = swapInfo.Used
status.Swap.Total = swapInfo.Total
if scoped == nil {
swapInfo, err := mem.SwapMemory()
if err != nil {
logger.Warning("get swap memory failed:", err)
} else {
status.Swap.Current = swapInfo.Used
status.Swap.Total = swapInfo.Total
}
}
// Disk stats
@@ -736,7 +751,7 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
// AppendCpuSample is preserved for callers that only have the CPU number.
// New callers should prefer AppendStatusSample which writes the full set.
func (s *ServerService) AppendCpuSample(t time.Time, v float64) {
systemMetrics.append("cpu", t, v)
systemMetrics.append(s.systemMetricKey("cpu"), t, v)
}
// AppendStatusSample writes one tick of every metric we keep — CPU, memory
@@ -747,14 +762,14 @@ func (s *ServerService) AppendStatusSample(t time.Time, status *Status) {
if status == nil {
return
}
systemMetrics.append("cpu", t, status.Cpu)
appendScopedMetric(status, "cpu", t, status.Cpu)
if status.Mem.Total > 0 {
systemMetrics.append("mem", t, float64(status.Mem.Current)*100.0/float64(status.Mem.Total))
}
if status.Swap.Total > 0 {
systemMetrics.append("swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
appendScopedMetric(status, "swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
} else {
systemMetrics.append("swap", t, 0)
appendScopedMetric(status, "swap", t, 0)
}
systemMetrics.append("netUp", t, float64(status.NetIO.Up))
systemMetrics.append("netDown", t, float64(status.NetIO.Down))
+58
View File
@@ -0,0 +1,58 @@
package service
import (
"slices"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
)
const containerHistoryPrefix = "lxc-v1/"
func scopedMetric(metric string) bool {
return metric == "cpu" || metric == "swap"
}
func (s *ServerService) systemMetricKey(metric string) string {
if !scopedMetric(metric) {
return metric
}
if status := s.LastStatus(); status != nil {
if status.ResourceScope != "" {
return containerHistoryPrefix + metric
}
return metric
}
if s.lxcMetrics.UseScopedHistory() {
return containerHistoryPrefix + metric
}
return metric
}
// Scope metadata applies only to CPU, swap and uptime.
func applyLXCStats(status *Status, sample *sys.LXCStats) {
status.ResourceScope = sample.Scope
status.UnavailableMetrics = append([]string(nil), sample.Unavailable...)
status.Cpu = sample.CPU
status.CpuCores = sample.CPUs
status.LogicalPro = sample.CPUs
status.Swap.Current = sample.SwapUsed
status.Swap.Total = sample.SwapTotal
status.Uptime = sample.Uptime
}
// MetricAvailable distinguishes an unavailable metric from a measured zero.
// Legacy status responses omit metadata and retain their previous semantics.
func (s *Status) MetricAvailable(metric string) bool {
return s != nil && !slices.Contains(s.UnavailableMetrics, metric)
}
func appendScopedMetric(status *Status, metric string, t time.Time, value float64) {
if !status.MetricAvailable(metric) {
return
}
if status.ResourceScope != "" {
metric = containerHistoryPrefix + metric
}
systemMetrics.append(metric, t, value)
}
+145
View File
@@ -0,0 +1,145 @@
//go:build linux
package service
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"slices"
"testing"
"time"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
)
func TestContainerStatusOnlyOverridesCPUSwapAndUptime(t *testing.T) {
root := t.TempDir()
t.Setenv("XUI_DB_FOLDER", root)
dbtest.InitDB(t, filepath.Join(root, "test.db"))
sys.SelfRSS()
proc, cg := filepath.Join(root, "proc"), filepath.Join(root, "cg")
fixtures := map[string]string{
filepath.Join(proc, "self/mountinfo"): "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n4 0 0:28 / " + cg + " rw - cgroup2 cgroup rw\n",
filepath.Join(proc, "1/cgroup"): "0::/init.scope\n",
filepath.Join(proc, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n",
filepath.Join(proc, "cpuinfo"): "processor : 0\ncpu cores : 16\nprocessor : 1\ncpu cores : 16\n",
filepath.Join(proc, "uptime"): "52624.93 52624.93\n",
filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/5 1\n",
filepath.Join(proc, "diskstats"): "8 0 vda 1 0 2 0 1 0 4 0 0 0 0\n",
filepath.Join(proc, "stat"): "cpu 100 0 50 1000 0 0 0 0\n",
filepath.Join(proc, "net/dev"): "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n",
filepath.Join(cg, "cpu.stat"): "usage_usec 222772076\n",
filepath.Join(cg, "cpu.max"): "max 100000\n",
filepath.Join(cg, "io.stat"): "259:0 rbytes=999999999 wbytes=999999999\n",
}
for path, content := range fixtures {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
t.Setenv("HOST_PROC", proc)
t.Setenv("HOST_SYS", filepath.Join(root, "sys"))
svc := &ServerService{cachedIPv4: "192.0.2.1", noIPv6: true, cachedCpuSpeedMhz: 5000, lxcMetrics: sys.LXCReader{ProcRoot: proc, CgroupRoot: cg}}
previous := &Status{T: time.Now().Add(-time.Second)}
got := svc.GetStatus(previous)
if got.ResourceScope != "container" || got.CpuCores != 2 || got.LogicalPro != 2 || got.CpuSpeedMhz != 0 || got.Swap.Total != 512<<20 || got.Swap.Current != 19357696 || got.Uptime != 52624 {
t.Fatalf("incorrect corrected resources: %+v", got)
}
if got.Mem.Total != 2<<30 || got.Mem.Current != 128<<20 || !reflect.DeepEqual(got.Loads, []float64{1.25, 2.5, 3.75}) || got.DiskTraffic.Read != 1024 || got.DiskTraffic.Write != 2048 || got.DiskIO.Read == 0 || got.DiskIO.Write == 0 {
t.Fatalf("unrelated metric path changed: %+v", got)
}
if !slices.Equal(got.UnavailableMetrics, []string{"cpu"}) {
t.Fatalf("unexpected unavailable metrics: %v", got.UnavailableMetrics)
}
raw, err := json.Marshal(got)
if err != nil {
t.Fatal(err)
}
var wire map[string]json.RawMessage
if err = json.Unmarshal(raw, &wire); err != nil {
t.Fatal(err)
}
for _, key := range []string{"cpu", "cpuCores", "logicalPro", "swap", "uptime", "mem", "loads", "diskIO", "diskTraffic", "resourceScope", "unavailableMetrics"} {
if _, ok := wire[key]; !ok {
t.Fatalf("API field lost: %s", key)
}
}
if err = os.WriteFile(filepath.Join(proc, "self/mountinfo"), []byte("1 0 0:1 / / rw - ext4 /dev/root rw\n"), 0o600); err != nil {
t.Fatal(err)
}
native := svc.GetStatus(previous)
if native.ResourceScope != "" || len(native.UnavailableMetrics) != 0 || native.CpuSpeedMhz != 5000 || native.Mem != got.Mem || !reflect.DeepEqual(native.Loads, got.Loads) || native.DiskTraffic != got.DiskTraffic {
t.Fatalf("native regression: %+v", native)
}
}
func TestScopedResourcesLeaveUnrelatedStatusUntouched(t *testing.T) {
status := &Status{Loads: []float64{1, 2, 3}, CpuSpeedMhz: 0}
status.Mem.Current, status.Mem.Total = 123, 456
status.DiskTraffic.Read, status.DiskTraffic.Write = 99, 101
status.DiskIO.Read, status.DiskIO.Write = 11, 12
before := *status
applyLXCStats(status, &sys.LXCStats{Scope: "container", CPUs: 2, SwapUsed: 8, SwapTotal: 512, Uptime: 15})
if status.Mem != before.Mem || status.DiskTraffic != before.DiskTraffic || status.DiskIO != before.DiskIO || !reflect.DeepEqual(status.Loads, before.Loads) {
t.Fatalf("unrelated status overwritten: %+v", status)
}
}
func TestContainerHistorySeparatesOnlyCPUAndSwap(t *testing.T) {
old := systemMetrics
systemMetrics = newMetricHistory()
defer func() { systemMetrics = old }()
now := time.Now()
systemMetrics.append("cpu", now, 90)
systemMetrics.append("swap", now, 62)
for key, value := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
systemMetrics.append(key, now, value)
}
status := &Status{T: now, ResourceScope: "container", Cpu: 10, Loads: []float64{1.25, 2.5, 3.75}, UnavailableMetrics: []string{"swap"}}
status.Mem.Total, status.Mem.Current = 100, 6
status.DiskIO.Read, status.NetIO.Up = 1024, 7
svc := &ServerService{lastStatus: status}
svc.AppendStatusSample(now, status)
cpu := svc.AggregateCpuHistory(2, 60)
if len(cpu) != 1 || cpu[0]["cpu"] != float64(10) {
t.Fatalf("CPU history=%v", cpu)
}
native := &ServerService{lastStatus: &Status{}}
if host := native.AggregateCpuHistory(2, 60); len(host) != 1 || host[0]["cpu"] != float64(90) {
t.Fatalf("legacy CPU changed: %v", host)
}
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 0 {
t.Fatalf("missing swap became zero: %v", points)
}
for key, want := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
if svc.systemMetricKey(key) != key {
t.Fatalf("unrelated history renamed: %s", key)
}
points := svc.AggregateSystemMetric(key, 2, 60)
if len(points) != 1 || points[0]["v"] != want {
t.Fatalf("unrelated %s history=%v", key, points)
}
}
status.UnavailableMetrics = nil
svc.AppendStatusSample(now, status)
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
t.Fatalf("disabled swap lost: %v", points)
}
snapshot := systemMetrics.snapshot()
for _, key := range []string{"cpu", "swap", containerHistoryPrefix + "cpu", containerHistoryPrefix + "swap"} {
if _, ok := snapshot[key]; !ok {
t.Fatalf("missing preserved series: %s", key)
}
}
for key := range snapshot {
if len(key) > len(containerHistoryPrefix) && key[:len(containerHistoryPrefix)] == containerHistoryPrefix && key != containerHistoryPrefix+"cpu" && key != containerHistoryPrefix+"swap" {
t.Fatalf("unexpected scoped metric: %s", key)
}
}
}
+6 -1
View File
@@ -187,7 +187,12 @@ func (t *Tgbot) prepareServerUsageInfo() string {
info += t.I18nBot("tgbot.messages.ipv6", "IPv6=="+ipv6)
}
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+strconv.FormatUint(t.lastStatus.Uptime/86400, 10), "Unit=="+t.I18nBot("tgbot.days"))
uptime, uptimeUnit := "—", ""
if t.lastStatus.MetricAvailable("uptime") {
uptime = strconv.FormatUint(t.lastStatus.Uptime/86400, 10)
uptimeUnit = t.I18nBot("tgbot.days")
}
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+uptime, "Unit=="+uptimeUnit)
info += t.I18nBot("tgbot.messages.serverLoad", "Load1=="+strconv.FormatFloat(t.lastStatus.Loads[0], 'f', 2, 64), "Load2=="+strconv.FormatFloat(t.lastStatus.Loads[1], 'f', 2, 64), "Load3=="+strconv.FormatFloat(t.lastStatus.Loads[2], 'f', 2, 64))
info += t.I18nBot("tgbot.messages.serverMemory", "Current=="+common.FormatTraffic(int64(t.lastStatus.Mem.Current)), "Total=="+common.FormatTraffic(int64(t.lastStatus.Mem.Total)))
info += t.I18nBot("tgbot.messages.onlinesCount", "Count=="+fmt.Sprint(len(onlines)))
@@ -14,11 +14,14 @@ import (
"time"
"github.com/mymmrac/telego"
"github.com/nicksnyder/go-i18n/v2/i18n"
"golang.org/x/text/language"
"gorm.io/gorm"
"github.com/mhsanaei/3x-ui/v3/internal/database"
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
"github.com/mhsanaei/3x-ui/v3/internal/web/locale"
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
"github.com/mhsanaei/3x-ui/v3/internal/xray"
)
@@ -284,3 +287,44 @@ func TestNotifyExhaustedLooksUpEachIDOnce(t *testing.T) {
})
}
}
func TestServerUsageInfoShowsUnavailableUptime(t *testing.T) {
bundle := i18n.NewBundle(language.English)
_ = bundle.AddMessages(language.English, &i18n.Message{ID: "tgbot.messages.serverUpTime", Other: "Uptime: {{.UpTime}} {{.Unit}}\n"}, &i18n.Message{ID: "tgbot.days", Other: "days"})
oldLocale := locale.LocalizerBot
locale.LocalizerBot = i18n.NewLocalizer(bundle, "en")
t.Cleanup(func() { locale.LocalizerBot = oldLocale })
statusCache.mutex.Lock()
oldStatus, oldTime := statusCache.data, statusCache.timestamp
statusCache.mutex.Unlock()
serverStatsCache.mutex.Lock()
oldStats, oldStatsTime := serverStatsCache.data, serverStatsCache.timestamp
serverStatsCache.mutex.Unlock()
t.Cleanup(func() {
statusCache.mutex.Lock()
statusCache.data, statusCache.timestamp = oldStatus, oldTime
statusCache.mutex.Unlock()
serverStatsCache.mutex.Lock()
serverStatsCache.data, serverStatsCache.timestamp = oldStats, oldStatsTime
serverStatsCache.mutex.Unlock()
})
for _, unavailable := range []bool{true, false} {
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
bot := &Tgbot{}
status := &service.Status{Loads: []float64{0, 0, 0}}
if unavailable {
status.UnavailableMetrics = []string{"uptime"}
}
bot.setCachedStatus(status)
bot.setCachedServerStats("")
info := bot.prepareServerUsageInfo()
want := "Uptime: 0 days"
if unavailable {
want = "Uptime: —"
}
if !strings.Contains(info, want) {
t.Fatalf("report did not contain %q: %s", want, info)
}
})
}
}