mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-10-25 15:25:01 +03:00
fix(monitoring): read LXC CPU, swap and uptime from the container (#6920)
In an LXC container with LXCFS the Overview showed the container's RAM beside the host's swap, the host's CPU topology and a CPU percentage from the host's view. Swap and uptime now come from LXCFS's virtualised files and CPU from the container's cgroup v2 counter, normalised by the visible vCPUs and any quota; container CPU and swap history is kept apart from the earlier host samples. A value the reader cannot measure (the first CPU sample, or a layout it cannot verify such as cgroup v1) is reported as unavailable instead of 0: the dashboard, the node list, node CPU history and the Telegram and Discord uptime show a dash, and nodes keep the flags in a new unavailable_metrics column.
This commit is contained in:
@@ -864,9 +864,11 @@ type Node struct {
|
||||
CpuPct float64 `json:"cpuPct" example:"23.5"`
|
||||
MemPct float64 `json:"memPct" example:"45.1"`
|
||||
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
|
||||
NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"`
|
||||
NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"`
|
||||
LastError string `json:"lastError"`
|
||||
// Nil on legacy rows and responses: zero remains a valid measured value.
|
||||
UnavailableMetrics []string `json:"unavailableMetrics,omitempty" gorm:"serializer:json;column:unavailable_metrics"`
|
||||
NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"`
|
||||
NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"`
|
||||
LastError string `json:"lastError"`
|
||||
|
||||
// XrayState and XrayError are captured from the remote node's /panel/api/server/status
|
||||
// during heartbeats. They let the central panel distinguish "panel API reachable"
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package sys
|
||||
|
||||
type LXCStats struct {
|
||||
Scope string
|
||||
CPU float64
|
||||
CPUs int
|
||||
SwapUsed, SwapTotal uint64
|
||||
Uptime uint64
|
||||
Unavailable []string
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
//go:build linux
|
||||
|
||||
package sys
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type LXCReader struct {
|
||||
ProcRoot, CgroupRoot string
|
||||
mu sync.Mutex
|
||||
lastCPU uint64
|
||||
lastAt time.Time
|
||||
lastCapacity float64
|
||||
sampled bool
|
||||
}
|
||||
|
||||
func (r *LXCReader) roots() (string, string) {
|
||||
proc := r.ProcRoot
|
||||
if proc == "" {
|
||||
proc = HostProc()
|
||||
}
|
||||
cg := r.CgroupRoot
|
||||
if cg == "" {
|
||||
cg = "/sys/fs/cgroup"
|
||||
}
|
||||
return proc, cg
|
||||
}
|
||||
|
||||
type lxcMounts struct {
|
||||
sources map[string]bool
|
||||
cgroupRoot bool
|
||||
}
|
||||
|
||||
func (r *LXCReader) detect() (lxcMounts, error) {
|
||||
proc, cg := r.roots()
|
||||
data, err := os.ReadFile(filepath.Join(proc, "self/mountinfo"))
|
||||
if err != nil {
|
||||
return lxcMounts{}, err
|
||||
}
|
||||
mounts := lxcMounts{sources: map[string]bool{}}
|
||||
valid := false
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
parts := strings.SplitN(line, " - ", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
mount, fs := strings.Fields(parts[0]), strings.Fields(parts[1])
|
||||
if len(mount) <= 4 || len(fs) == 0 {
|
||||
continue
|
||||
}
|
||||
valid = true
|
||||
if fs[0] == "cgroup2" && mount[3] == "/" && mount[4] == cg {
|
||||
mounts.cgroupRoot = true
|
||||
}
|
||||
if fs[0] == "fuse.lxcfs" {
|
||||
switch mount[4] {
|
||||
case "/proc/meminfo", "/proc/cpuinfo", "/proc/uptime":
|
||||
mounts.sources[mount[4]] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
return lxcMounts{}, errors.New("invalid mountinfo")
|
||||
}
|
||||
return mounts, nil
|
||||
}
|
||||
|
||||
func (r *LXCReader) UseScopedHistory() bool {
|
||||
mounts, err := r.detect()
|
||||
return len(mounts.sources) > 0 || err != nil
|
||||
}
|
||||
|
||||
func (r *LXCReader) resetCPU() {
|
||||
r.mu.Lock()
|
||||
r.sampled = false
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// LXCFS supplies container swap and uptime; sysinfo bypasses those mounts.
|
||||
func (r *LXCReader) Read(now time.Time) (*LXCStats, error) {
|
||||
mounts, err := r.detect()
|
||||
if len(mounts.sources) == 0 && err == nil {
|
||||
r.resetCPU()
|
||||
return nil, nil
|
||||
}
|
||||
out := &LXCStats{Scope: "container"}
|
||||
if err != nil {
|
||||
r.resetCPU()
|
||||
out.Scope = "unknown"
|
||||
out.Unavailable = []string{"cpu", "swap", "uptime"}
|
||||
return out, fmt.Errorf("cannot determine metric scope: %w", err)
|
||||
}
|
||||
proc, cg := r.roots()
|
||||
if mounts.sources["/proc/meminfo"] {
|
||||
mem, err := readLXCBytes(filepath.Join(proc, "meminfo"))
|
||||
total, tok := mem["SwapTotal"]
|
||||
free, fok := mem["SwapFree"]
|
||||
if err == nil && tok && fok && free <= total {
|
||||
out.SwapTotal = total
|
||||
out.SwapUsed = total - free
|
||||
} else {
|
||||
out.Unavailable = append(out.Unavailable, "swap")
|
||||
}
|
||||
} else {
|
||||
out.Unavailable = append(out.Unavailable, "swap")
|
||||
}
|
||||
if mounts.sources["/proc/uptime"] {
|
||||
data, err := os.ReadFile(filepath.Join(proc, "uptime"))
|
||||
fields := strings.Fields(string(data))
|
||||
if err == nil && len(fields) > 0 {
|
||||
value, e := strconv.ParseFloat(fields[0], 64)
|
||||
if e == nil && !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 && value < float64(math.MaxUint64) {
|
||||
out.Uptime = uint64(value)
|
||||
} else {
|
||||
err = errors.New("invalid uptime")
|
||||
}
|
||||
} else {
|
||||
err = errors.New("missing uptime")
|
||||
}
|
||||
if err != nil {
|
||||
out.Unavailable = append(out.Unavailable, "uptime")
|
||||
}
|
||||
} else {
|
||||
out.Unavailable = append(out.Unavailable, "uptime")
|
||||
}
|
||||
if !mounts.sources["/proc/cpuinfo"] || !mounts.cgroupRoot || !containerInitAtRoot(proc) {
|
||||
r.resetCPU()
|
||||
out.Unavailable = append(out.Unavailable, "cpu")
|
||||
return out, nil
|
||||
}
|
||||
capacity, cpus, err := r.cpuCapacity(proc, cg)
|
||||
out.CPUs = cpus
|
||||
usage, usageErr := readUsage(filepath.Join(cg, "cpu.stat"))
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if err != nil || usageErr != nil {
|
||||
r.sampled = false
|
||||
out.Unavailable = append(out.Unavailable, "cpu")
|
||||
} else {
|
||||
elapsed := now.Sub(r.lastAt).Seconds()
|
||||
if r.sampled && elapsed > 0 && usage >= r.lastCPU && capacity == r.lastCapacity {
|
||||
out.CPU = math.Min(100, float64(usage-r.lastCPU)/1e6/elapsed/capacity*100)
|
||||
} else {
|
||||
out.Unavailable = append(out.Unavailable, "cpu")
|
||||
}
|
||||
r.lastCPU, r.lastAt, r.lastCapacity, r.sampled = usage, now, capacity, true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PID 1 must belong to this namespace root, not a host-visible LXC subtree.
|
||||
func containerInitAtRoot(proc string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(proc, "1/cgroup"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch strings.TrimSpace(string(data)) {
|
||||
case "0::/", "0::/init.scope":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func readLXCBytes(path string) (map[string]uint64, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wanted := map[string]bool{"SwapTotal": true, "SwapFree": true}
|
||||
values := map[string]uint64{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSuffix(fields[0], ":")
|
||||
if !wanted[key] {
|
||||
continue
|
||||
}
|
||||
if len(fields) != 3 || fields[2] != "kB" {
|
||||
return nil, errors.New("invalid meminfo unit")
|
||||
}
|
||||
if _, exists := values[key]; exists {
|
||||
return nil, errors.New("duplicate meminfo field")
|
||||
}
|
||||
value, e := strconv.ParseUint(fields[1], 10, 64)
|
||||
if e != nil || value > math.MaxUint64/1024 {
|
||||
return nil, errors.New("invalid meminfo value")
|
||||
}
|
||||
values[key] = value * 1024
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (r *LXCReader) cpuCapacity(proc, cg string) (float64, int, error) {
|
||||
data, err := os.ReadFile(filepath.Join(proc, "cpuinfo"))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
ids := map[uint64]bool{}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
key, value, ok := strings.Cut(line, ":")
|
||||
if !ok || strings.TrimSpace(key) != "processor" {
|
||||
continue
|
||||
}
|
||||
id, e := strconv.ParseUint(strings.TrimSpace(value), 10, 64)
|
||||
if e != nil || ids[id] {
|
||||
return 0, 0, errors.New("invalid CPU list")
|
||||
}
|
||||
ids[id] = true
|
||||
}
|
||||
cpus := len(ids)
|
||||
if cpus == 0 {
|
||||
return 0, 0, errors.New("empty CPU list")
|
||||
}
|
||||
capacity := float64(cpus)
|
||||
data, err = os.ReadFile(filepath.Join(cg, "cpu.max"))
|
||||
if err != nil {
|
||||
return 0, cpus, err
|
||||
}
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) != 2 {
|
||||
return 0, cpus, errors.New("invalid CPU quota")
|
||||
}
|
||||
period, e := strconv.ParseUint(fields[1], 10, 64)
|
||||
if e != nil || period == 0 {
|
||||
return 0, cpus, errors.New("invalid CPU period")
|
||||
}
|
||||
if fields[0] != "max" {
|
||||
quota, e := strconv.ParseUint(fields[0], 10, 64)
|
||||
if e != nil || quota == 0 {
|
||||
return 0, cpus, errors.New("invalid CPU quota")
|
||||
}
|
||||
capacity = math.Min(capacity, float64(quota)/float64(period))
|
||||
}
|
||||
return capacity, cpus, nil
|
||||
}
|
||||
|
||||
func readUsage(path string) (uint64, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var usage uint64
|
||||
found := false
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 || fields[0] != "usage_usec" {
|
||||
continue
|
||||
}
|
||||
if len(fields) != 2 || found {
|
||||
return 0, errors.New("invalid cgroup CPU usage")
|
||||
}
|
||||
usage, err = strconv.ParseUint(fields[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
return 0, errors.New("missing cgroup CPU usage")
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
//go:build linux
|
||||
|
||||
package sys
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const lxcMountFixture = "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n"
|
||||
|
||||
func lxcFixture(t *testing.T) *LXCReader {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
r := &LXCReader{ProcRoot: filepath.Join(root, "proc"), CgroupRoot: filepath.Join(root, "cgroup")}
|
||||
fixtures := map[string]string{
|
||||
filepath.Join(r.ProcRoot, "self/mountinfo"): lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n",
|
||||
filepath.Join(r.ProcRoot, "1/cgroup"): "0::/init.scope\n",
|
||||
filepath.Join(r.ProcRoot, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n",
|
||||
filepath.Join(r.ProcRoot, "cpuinfo"): "processor : 0\ncpu cores : 16\n\nprocessor : 1\ncpu cores : 16\n",
|
||||
filepath.Join(r.ProcRoot, "uptime"): "52624.93 52624.93\n",
|
||||
filepath.Join(r.CgroupRoot, "cpu.stat"): "usage_usec 222772076\nuser_usec 101515374\n",
|
||||
filepath.Join(r.CgroupRoot, "cpu.max"): "max 100000\n",
|
||||
}
|
||||
for path, content := range fixtures {
|
||||
fixtureWrite(t, path, content)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func fixtureWrite(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCResourcesIgnoreHostTopologyAndInheritedMax(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("sample=%+v err=%v", got, err)
|
||||
}
|
||||
if got.CPUs != 2 || got.SwapTotal != 512<<20 || got.SwapUsed != 19357696 || got.Uptime != 52624 {
|
||||
t.Fatalf("wrong container resources: %+v", got)
|
||||
}
|
||||
if !slices.Equal(got.Unavailable, []string{"cpu"}) {
|
||||
t.Fatalf("first sample=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCCPUUsesWholeContainerDeltaAndCapacity(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, quota string
|
||||
delta uint64
|
||||
want float64
|
||||
}{
|
||||
{"one CPU of two", "max 100000\n", 1000000, 50},
|
||||
{"idle is a valid zero", "max 100000\n", 0, 0},
|
||||
{"half CPU quota", "50000 100000\n", 125000, 25},
|
||||
{"quota above visible CPUs", "800000 100000\n", 1000000, 50},
|
||||
{"saturated interval bounded", "max 100000\n", 2100000, 100},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), tc.quota)
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1000000\n")
|
||||
r.Read(time.Unix(100, 0))
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(1000000+tc.delta, 10)+"\n")
|
||||
got, err := r.Read(time.Unix(101, 0))
|
||||
if err != nil || got.CPU != tc.want || slices.Contains(got.Unavailable, "cpu") {
|
||||
t.Fatalf("sample=%+v err=%v want=%v", got, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCCounterClockAndCapacityChangesResetBaseline(t *testing.T) {
|
||||
for _, scenario := range []string{"counter reset", "zero interval", "backward interval", "capacity change", "missing counter", "scope change"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
r.Read(time.Unix(100, 0))
|
||||
at := time.Unix(101, 0)
|
||||
switch scenario {
|
||||
case "counter reset":
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1\n")
|
||||
case "zero interval":
|
||||
at = time.Unix(100, 0)
|
||||
case "backward interval":
|
||||
at = time.Unix(99, 0)
|
||||
case "capacity change":
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), "100000 100000\n")
|
||||
case "missing counter":
|
||||
os.Remove(filepath.Join(r.CgroupRoot, "cpu.stat"))
|
||||
r.Read(at)
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 222772076\n")
|
||||
case "scope change":
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n")
|
||||
r.Read(at)
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), lxcMountFixture+"4 0 0:28 / "+r.CgroupRoot+" rw - cgroup2 cgroup rw\n")
|
||||
}
|
||||
got, err := r.Read(at)
|
||||
if err != nil || !slices.Contains(got.Unavailable, "cpu") {
|
||||
t.Fatalf("invalid interval=%+v err=%v", got, err)
|
||||
}
|
||||
usage, _ := readUsage(filepath.Join(r.CgroupRoot, "cpu.stat"))
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(usage+500000, 10)+"\n")
|
||||
got, err = r.Read(at.Add(time.Second))
|
||||
want := 25.0
|
||||
if scenario == "capacity change" {
|
||||
want = 50
|
||||
}
|
||||
if err != nil || got.CPU != want || slices.Contains(got.Unavailable, "cpu") {
|
||||
t.Fatalf("recovery=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCInvalidMetricsNeverUseHostFallback(t *testing.T) {
|
||||
for _, tc := range []struct{ path, content, metric string }{
|
||||
{"meminfo", "SwapTotal: 1 kB\nSwapFree: 2 kB\n", "swap"},
|
||||
{"meminfo", "SwapTotal: 18446744073709551615 kB\nSwapFree: 0 kB\n", "swap"},
|
||||
{"meminfo", "SwapTotal: 1 MB\nSwapFree: 0 kB\n", "swap"},
|
||||
{"meminfo", "SwapTotal: 1 kB\nSwapTotal: 1 kB\nSwapFree: 0 kB\n", "swap"},
|
||||
{"meminfo", "SwapTotal: 1 kB\n", "swap"},
|
||||
{"uptime", "NaN 0\n", "uptime"},
|
||||
{"uptime", "+Inf 0\n", "uptime"},
|
||||
{"uptime", "-1 0\n", "uptime"},
|
||||
{"uptime", "18446744073709551616 0\n", "uptime"},
|
||||
{"uptime", "", "uptime"},
|
||||
{"cpuinfo", "cpu cores : 16\n", "cpu"},
|
||||
{"cpuinfo", "processor : 0\nprocessor : 0\n", "cpu"},
|
||||
} {
|
||||
t.Run(tc.metric+"/"+tc.content, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, tc.path), tc.content)
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) {
|
||||
t.Fatalf("invalid sample=%+v err=%v", got, err)
|
||||
}
|
||||
for _, v := range got.Unavailable {
|
||||
if v != "cpu" && v != "swap" && v != "uptime" {
|
||||
t.Fatalf("unrelated metric changed: %s", v)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCCPURejectsMalformedCgroupData(t *testing.T) {
|
||||
for _, tc := range []struct{ path, value string }{
|
||||
{"cpu.max", "max 0"},
|
||||
{"cpu.max", "0 100000"},
|
||||
{"cpu.max", "NaN 100000"},
|
||||
{"cpu.max", "max"},
|
||||
{"cpu.stat", "user_usec 123"},
|
||||
{"cpu.stat", "usage_usec -1"},
|
||||
{"cpu.stat", "usage_usec 1\nusage_usec 2"},
|
||||
} {
|
||||
t.Run(tc.path+"/"+tc.value, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
r.Read(time.Unix(100, 0))
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, tc.path), tc.value)
|
||||
got, err := r.Read(time.Unix(101, 0))
|
||||
if err != nil || !slices.Contains(got.Unavailable, "cpu") || math.IsNaN(got.CPU) || math.IsInf(got.CPU, 0) {
|
||||
t.Fatalf("bad CPU=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCZeroSwapIsValidAndRAMIsNotParsed(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "meminfo"), "MemTotal: not-a-number kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n")
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err != nil || got.SwapTotal != 0 || got.SwapUsed != 0 || slices.Contains(got.Unavailable, "swap") {
|
||||
t.Fatalf("disabled swap=%+v err=%v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCEachProcSourceRequiresItsOwnMount(t *testing.T) {
|
||||
for _, tc := range []struct{ path, metric string }{{"meminfo", "swap"}, {"cpuinfo", "cpu"}, {"uptime", "uptime"}} {
|
||||
t.Run(tc.path, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
var lines []string
|
||||
for _, line := range strings.Split(lxcMountFixture, "\n") {
|
||||
if !strings.Contains(line, "/proc/"+tc.path) {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), strings.Join(lines, "\n"))
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) {
|
||||
t.Fatalf("unverified mount=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeMetricsKeepOriginalPath(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n")
|
||||
os.RemoveAll(r.CgroupRoot)
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err != nil || got != nil || r.UseScopedHistory() {
|
||||
t.Fatalf("native=%+v err=%v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnreadableScopeNeverSelectsNativeHostMetrics(t *testing.T) {
|
||||
for _, bad := range []string{"missing", "malformed"} {
|
||||
t.Run(bad, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
if bad == "missing" {
|
||||
os.Remove(filepath.Join(r.ProcRoot, "self/mountinfo"))
|
||||
} else {
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "invalid")
|
||||
}
|
||||
got, err := r.Read(time.Unix(100, 0))
|
||||
if err == nil || got == nil || got.Scope != "unknown" || !slices.Equal(got.Unavailable, []string{"cpu", "swap", "uptime"}) || !r.UseScopedHistory() {
|
||||
t.Fatalf("unknown=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCCPURejectsUnverifiedCgroupRoot(t *testing.T) {
|
||||
for _, scenario := range []string{"missing mount", "subtree mount", "legacy controller", "host namespace", "missing init membership"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
r.Read(time.Unix(100, 0))
|
||||
mounts := lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n"
|
||||
switch scenario {
|
||||
case "missing mount":
|
||||
mounts = lxcMountFixture
|
||||
case "subtree mount":
|
||||
mounts = lxcMountFixture + "4 0 0:28 /system.slice " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n"
|
||||
case "legacy controller":
|
||||
mounts = lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup cgroup rw\n"
|
||||
case "host namespace":
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/lxc/example/init.scope\n")
|
||||
case "missing init membership":
|
||||
os.Remove(filepath.Join(r.ProcRoot, "1/cgroup"))
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), mounts)
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n")
|
||||
got, err := r.Read(time.Unix(101, 0))
|
||||
if err != nil || got == nil || !slices.Contains(got.Unavailable, "cpu") || got.CPU != 0 || got.SwapTotal != 512<<20 || got.Uptime != 52624 {
|
||||
t.Fatalf("unverified cgroup leaked CPU or changed proc metrics: %+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXCCPUAcceptsContainerInitAtNamespaceRoot(t *testing.T) {
|
||||
r := lxcFixture(t)
|
||||
fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/\n")
|
||||
r.Read(time.Unix(100, 0))
|
||||
fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n")
|
||||
got, err := r.Read(time.Unix(101, 0))
|
||||
if err != nil || got.CPU != 50 || slices.Contains(got.Unavailable, "cpu") {
|
||||
t.Fatalf("root init sample=%+v err=%v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
//go:build !linux
|
||||
|
||||
package sys
|
||||
|
||||
import "time"
|
||||
|
||||
type LXCReader struct{}
|
||||
|
||||
func (r *LXCReader) Read(now time.Time) (*LXCStats, error) { return nil, nil }
|
||||
func (r *LXCReader) UseScopedHistory() bool { return false }
|
||||
@@ -487,9 +487,6 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
|
||||
hostname = "3x-ui"
|
||||
}
|
||||
|
||||
days := status.Uptime / 86400
|
||||
hours := (status.Uptime % 86400) / 3600
|
||||
|
||||
var onlines []string
|
||||
if process := service.XrayProcess(); process != nil && process.IsRunning() {
|
||||
onlines = process.GetOnlineClients()
|
||||
@@ -515,7 +512,7 @@ func (g *GatewayClient) sendStatus(ctx context.Context) {
|
||||
Fields: []EmbedField{
|
||||
{Name: tr("discord.fields.panelVersion"), Value: config.GetPanelVersion(), Inline: true},
|
||||
{Name: tr("discord.fields.xrayCore"), Value: fmt.Sprintf("%s (%s)", status.Xray.Version, status.Xray.State), Inline: true},
|
||||
{Name: tr("pages.index.uptime"), Value: tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours)), Inline: true},
|
||||
{Name: tr("pages.index.uptime"), Value: formatStatusUptime(status, tr), Inline: true},
|
||||
{Name: tr("discord.fields.systemLoad"), Value: fmt.Sprintf("%.2f, %.2f, %.2f", load1, load2, load3), Inline: true},
|
||||
{Name: tr("pages.index.memory"), Value: fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total))), Inline: true},
|
||||
{Name: tr("pages.index.historyTitleOnline"), Value: strconv.Itoa(len(onlines)), Inline: true},
|
||||
|
||||
@@ -102,9 +102,7 @@ func (s *DiscordService) BuildReport(ctx context.Context, server ServerProvider,
|
||||
}
|
||||
|
||||
tr := translator(s.settingService)
|
||||
days := status.Uptime / 86400
|
||||
hours := (status.Uptime % 86400) / 3600
|
||||
uptimeStr := tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours))
|
||||
uptimeStr := formatStatusUptime(status, tr)
|
||||
|
||||
ramStr := fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total)))
|
||||
trafficStr := tr("discord.values.traffic",
|
||||
@@ -239,3 +237,11 @@ func getInterfaceIPs() (ipv4, ipv6 string) {
|
||||
}
|
||||
return strings.Join(v4s, ", "), strings.Join(v6s, ", ")
|
||||
}
|
||||
|
||||
// formatStatusUptime is shared by scheduled reports and the status command.
|
||||
func formatStatusUptime(status *service.Status, tr func(string, ...string) string) string {
|
||||
if !status.MetricAvailable("uptime") {
|
||||
return "—"
|
||||
}
|
||||
return tr("discord.values.uptime", "Days=="+fmt.Sprint(status.Uptime/86400), "Hours=="+fmt.Sprint((status.Uptime%86400)/3600))
|
||||
}
|
||||
|
||||
@@ -229,3 +229,61 @@ func TestSendReport_DeliversEmbedWhenBackupUploadIsRejected(t *testing.T) {
|
||||
t.Fatal("report embed was never delivered: it rode on the rejected backup upload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildReportShowsUnavailableUptime(t *testing.T) {
|
||||
for _, unavailable := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
|
||||
settings := setupTestDB(t)
|
||||
_ = settings.SetDiscordLang("en-US")
|
||||
_ = settings.SetDiscordBotBackup(false)
|
||||
status := &service.Status{Loads: []float64{0, 0, 0}}
|
||||
if unavailable {
|
||||
status.UnavailableMetrics = []string{"uptime"}
|
||||
}
|
||||
server := &mockServerProvider{status: status}
|
||||
svc := NewDiscordService(settings)
|
||||
report, _, err := svc.BuildReport(context.Background(), server, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "0d 0h"
|
||||
if unavailable {
|
||||
want = "—"
|
||||
}
|
||||
check := func(fields []EmbedField) {
|
||||
t.Helper()
|
||||
for _, f := range fields {
|
||||
if f.Name == "Uptime" {
|
||||
if f.Value != want {
|
||||
t.Fatalf("uptime=%q want %q", f.Value, want)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("uptime field missing")
|
||||
}
|
||||
check(report.Embeds[0].Fields)
|
||||
_ = settings.SetDiscordBotToken("synthetic-test")
|
||||
_ = settings.SetDiscordChannelId("synthetic-channel")
|
||||
sent := make(chan MessagePayload, 1)
|
||||
rest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload MessagePayload
|
||||
_ = json.NewDecoder(r.Body).Decode(&payload)
|
||||
sent <- payload
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"synthetic-message"}`))
|
||||
}))
|
||||
defer rest.Close()
|
||||
svc.SetBaseURL(rest.URL)
|
||||
svc.SetHTTPClient(rest.Client())
|
||||
g := NewGatewayClient(svc, settings, server, nil, nil)
|
||||
g.sendStatus(context.Background())
|
||||
select {
|
||||
case msg := <-sent:
|
||||
check(msg.Embeds[0].Fields)
|
||||
default:
|
||||
t.Fatal("no local status payload")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,15 +32,16 @@ import (
|
||||
)
|
||||
|
||||
type HeartbeatPatch struct {
|
||||
Status string
|
||||
LastHeartbeat int64
|
||||
LatencyMs int
|
||||
XrayVersion string
|
||||
PanelVersion string
|
||||
Guid string
|
||||
CpuPct float64
|
||||
MemPct float64
|
||||
UptimeSecs uint64
|
||||
Status string
|
||||
LastHeartbeat int64
|
||||
LatencyMs int
|
||||
XrayVersion string
|
||||
PanelVersion string
|
||||
Guid string
|
||||
CpuPct float64
|
||||
MemPct float64
|
||||
UptimeSecs uint64
|
||||
UnavailableMetrics []string
|
||||
// NetUp/NetDown are the node's current interface throughput (bytes/sec),
|
||||
// summed over non-virtual interfaces, read from its status response.
|
||||
NetUp uint64
|
||||
@@ -978,20 +979,27 @@ func (s *NodeService) UpdatePanels(ids []int, dev bool) ([]NodeUpdateResult, err
|
||||
|
||||
func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
|
||||
db := database.GetDB()
|
||||
// Map updates bypass GORM field serializers. Encode the availability list
|
||||
// explicitly so it survives list requests and panel restarts.
|
||||
unavailable, err := json.Marshal(p.UnavailableMetrics)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
updates := map[string]any{
|
||||
"status": p.Status,
|
||||
"last_heartbeat": p.LastHeartbeat,
|
||||
"latency_ms": p.LatencyMs,
|
||||
"xray_version": p.XrayVersion,
|
||||
"panel_version": p.PanelVersion,
|
||||
"cpu_pct": p.CpuPct,
|
||||
"mem_pct": p.MemPct,
|
||||
"uptime_secs": p.UptimeSecs,
|
||||
"net_up": p.NetUp,
|
||||
"net_down": p.NetDown,
|
||||
"last_error": p.LastError,
|
||||
"xray_state": p.XrayState,
|
||||
"xray_error": p.XrayError,
|
||||
"unavailable_metrics": string(unavailable),
|
||||
"status": p.Status,
|
||||
"last_heartbeat": p.LastHeartbeat,
|
||||
"latency_ms": p.LatencyMs,
|
||||
"xray_version": p.XrayVersion,
|
||||
"panel_version": p.PanelVersion,
|
||||
"cpu_pct": p.CpuPct,
|
||||
"mem_pct": p.MemPct,
|
||||
"uptime_secs": p.UptimeSecs,
|
||||
"net_up": p.NetUp,
|
||||
"net_down": p.NetDown,
|
||||
"last_error": p.LastError,
|
||||
"xray_state": p.XrayState,
|
||||
"xray_error": p.XrayError,
|
||||
}
|
||||
// Only learn the GUID; never clear a known one if an old-build node (or a
|
||||
// failed probe) reports none, so the stable identity survives blips.
|
||||
@@ -1004,7 +1012,9 @@ func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error {
|
||||
}
|
||||
if p.Status == "online" {
|
||||
now := time.Unix(p.LastHeartbeat, 0)
|
||||
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
|
||||
if !slices.Contains(p.UnavailableMetrics, "cpu") {
|
||||
nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct)
|
||||
}
|
||||
nodeMetrics.append(nodeMetricKey(id, "mem"), now, p.MemPct)
|
||||
nodeMetrics.append(nodeMetricKey(id, "netUp"), now, float64(p.NetUp))
|
||||
nodeMetrics.append(nodeMetricKey(id, "netDown"), now, float64(p.NetDown))
|
||||
@@ -1335,8 +1345,9 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
|
||||
Success bool `json:"success"`
|
||||
Msg string `json:"msg"`
|
||||
Obj *struct {
|
||||
CpuPct float64 `json:"cpu"`
|
||||
Mem struct {
|
||||
CpuPct float64 `json:"cpu"`
|
||||
UnavailableMetrics []string `json:"unavailableMetrics"`
|
||||
Mem struct {
|
||||
Current uint64 `json:"current"`
|
||||
Total uint64 `json:"total"`
|
||||
} `json:"mem"`
|
||||
@@ -1370,6 +1381,7 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
|
||||
}
|
||||
o := envelope.Obj
|
||||
patch.CpuPct = o.CpuPct
|
||||
patch.UnavailableMetrics = o.UnavailableMetrics
|
||||
if o.Mem.Total > 0 {
|
||||
patch.MemPct = float64(o.Mem.Current) * 100.0 / float64(o.Mem.Total)
|
||||
}
|
||||
@@ -1385,14 +1397,15 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string)
|
||||
}
|
||||
|
||||
type ProbeResultUI struct {
|
||||
Status string `json:"status" example:"online"`
|
||||
LatencyMs int `json:"latencyMs" example:"42"`
|
||||
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
|
||||
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
|
||||
CpuPct float64 `json:"cpuPct" example:"12.5"`
|
||||
MemPct float64 `json:"memPct" example:"45.2"`
|
||||
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
|
||||
Error string `json:"error"`
|
||||
Status string `json:"status" example:"online"`
|
||||
LatencyMs int `json:"latencyMs" example:"42"`
|
||||
XrayVersion string `json:"xrayVersion" example:"25.10.31"`
|
||||
PanelVersion string `json:"panelVersion" example:"v3.x.x"`
|
||||
CpuPct float64 `json:"cpuPct" example:"12.5"`
|
||||
MemPct float64 `json:"memPct" example:"45.2"`
|
||||
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
|
||||
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
|
||||
Error string `json:"error"`
|
||||
// XrayState/XrayError are populated on successful probes even when the node's
|
||||
// Xray core is not healthy. The UI uses them for a distinct "panel ok, xray failed" indicator.
|
||||
XrayState string `json:"xrayState"`
|
||||
@@ -1401,15 +1414,16 @@ type ProbeResultUI struct {
|
||||
|
||||
func (p HeartbeatPatch) ToUI(ok bool) ProbeResultUI {
|
||||
r := ProbeResultUI{
|
||||
LatencyMs: p.LatencyMs,
|
||||
XrayVersion: p.XrayVersion,
|
||||
PanelVersion: p.PanelVersion,
|
||||
CpuPct: p.CpuPct,
|
||||
MemPct: p.MemPct,
|
||||
UptimeSecs: p.UptimeSecs,
|
||||
Error: FriendlyProbeError(p.LastError),
|
||||
XrayState: p.XrayState,
|
||||
XrayError: p.XrayError,
|
||||
LatencyMs: p.LatencyMs,
|
||||
XrayVersion: p.XrayVersion,
|
||||
PanelVersion: p.PanelVersion,
|
||||
CpuPct: p.CpuPct,
|
||||
MemPct: p.MemPct,
|
||||
UptimeSecs: p.UptimeSecs,
|
||||
UnavailableMetrics: p.UnavailableMetrics,
|
||||
Error: FriendlyProbeError(p.LastError),
|
||||
XrayState: p.XrayState,
|
||||
XrayError: p.XrayError,
|
||||
}
|
||||
if ok {
|
||||
r.Status = "online"
|
||||
|
||||
@@ -34,6 +34,7 @@ type NodeView struct {
|
||||
CpuPct float64 `json:"cpuPct" example:"12.5"`
|
||||
MemPct float64 `json:"memPct" example:"45.2"`
|
||||
UptimeSecs uint64 `json:"uptimeSecs" example:"86400"`
|
||||
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
|
||||
NetUp uint64 `json:"netUp" example:"2097152"`
|
||||
NetDown uint64 `json:"netDown" example:"1048576"`
|
||||
LastError string `json:"lastError" example:""`
|
||||
@@ -82,6 +83,7 @@ func toNodeView(n *model.Node) *NodeView {
|
||||
CpuPct: n.CpuPct,
|
||||
MemPct: n.MemPct,
|
||||
UptimeSecs: n.UptimeSecs,
|
||||
UnavailableMetrics: n.UnavailableMetrics,
|
||||
NetUp: n.NetUp,
|
||||
NetDown: n.NetDown,
|
||||
LastError: n.LastError,
|
||||
|
||||
@@ -2,15 +2,20 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
|
||||
)
|
||||
|
||||
@@ -70,3 +75,131 @@ func TestNodeMetricKeysIncludesNet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func availabilityNodeFixture(t *testing.T, body string) (*NodeService, *model.Node, func(string)) {
|
||||
t.Helper()
|
||||
dbtest.InitDB(t, filepath.Join(t.TempDir(), "node.db"))
|
||||
var response atomic.Value
|
||||
response.Store(body)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(response.Load().(string)))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port, err := strconv.Atoi(u.Port())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := &model.Node{Name: "sample", Scheme: "http", Address: u.Hostname(), Port: port, BasePath: "/", Enable: true, AllowPrivateAddress: true, TlsVerifyMode: "skip"}
|
||||
if err := database.GetDB().Create(n).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := nodeMetrics
|
||||
nodeMetrics = newMetricHistory()
|
||||
t.Cleanup(func() { nodeMetrics = old })
|
||||
return &NodeService{}, n, func(s string) { response.Store(s) }
|
||||
}
|
||||
|
||||
func unavailableMetricsOf(t *testing.T, value any) []string {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out struct {
|
||||
Unavailable []string `json:"unavailableMetrics"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.Unavailable
|
||||
}
|
||||
|
||||
const (
|
||||
unavailableStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"unavailableMetrics":["cpu","uptime"],"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
|
||||
zeroStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"mem":{"current":1,"total":2},"xray":{"state":"running"}}}`
|
||||
)
|
||||
|
||||
func TestProbeKeepsMetricAvailability(t *testing.T) {
|
||||
svc, n, _ := availabilityNodeFixture(t, unavailableStatusBody)
|
||||
patch, err := svc.Probe(context.Background(), n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := unavailableMetricsOf(t, patch.ToUI(true)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
|
||||
t.Fatalf("probe availability=%v, want cpu+uptime unavailable", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateHeartbeatKeepsUnavailableCPUOutOfHistory(t *testing.T) {
|
||||
svc, n, setBody := availabilityNodeFixture(t, unavailableStatusBody)
|
||||
patch, err := svc.Probe(context.Background(), n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
patch.Status = "online"
|
||||
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 0 {
|
||||
t.Fatalf("unavailable CPU became history: %v", points)
|
||||
}
|
||||
var stored model.Node
|
||||
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := unavailableMetricsOf(t, toNodeView(&stored)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) {
|
||||
t.Fatalf("stored availability=%v", got)
|
||||
}
|
||||
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "mem"), 2, 60); len(points) != 1 || points[0]["v"] != float64(50) {
|
||||
t.Fatalf("memory history changed: %v", points)
|
||||
}
|
||||
setBody(zeroStatusBody)
|
||||
patch, err = svc.Probe(context.Background(), n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
patch.Status = "online"
|
||||
if err := svc.UpdateHeartbeat(n.Id, patch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := unavailableMetricsOf(t, toNodeView(&stored)); len(got) != 0 {
|
||||
t.Fatalf("valid zero did not recover: %v", got)
|
||||
}
|
||||
if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
|
||||
t.Fatalf("valid zero lost: %v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeAvailabilityColumnKeepsLegacyRows(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "legacy.db")
|
||||
dbtest.InitDB(t, path)
|
||||
n := &model.Node{Name: "legacy", CpuPct: 0, UptimeSecs: 0}
|
||||
if err := database.GetDB().Create(n).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Recreate the pre-change schema, then let the normal startup migration run.
|
||||
if err := database.GetDB().Migrator().DropColumn(&model.Node{}, "UnavailableMetrics"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.CloseDB(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := database.InitDB(path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var stored model.Node
|
||||
if err := database.GetDB().First(&stored, n.Id).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stored.Name != "legacy" || stored.CpuPct != 0 || stored.UptimeSecs != 0 || len(stored.UnavailableMetrics) != 0 {
|
||||
t.Fatalf("legacy row changed: id=%d availability=%v", stored.Id, stored.UnavailableMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +66,9 @@ const (
|
||||
// Status represents comprehensive system and application status information.
|
||||
// It includes CPU, memory, disk, network statistics, and Xray process status.
|
||||
type Status struct {
|
||||
ResourceScope string `json:"resourceScope,omitempty"`
|
||||
UnavailableMetrics []string `json:"unavailableMetrics,omitempty"`
|
||||
|
||||
T time.Time `json:"-"`
|
||||
Cpu float64 `json:"cpu"`
|
||||
CpuCores int `json:"cpuCores"`
|
||||
@@ -143,6 +146,8 @@ type Release struct {
|
||||
// ServerService provides business logic for server monitoring and management.
|
||||
// It handles system status collection, IP detection, and application statistics.
|
||||
type ServerService struct {
|
||||
lxcMetrics sys.LXCReader
|
||||
|
||||
xrayService XrayService
|
||||
inboundService InboundService
|
||||
settingService SettingService
|
||||
@@ -355,7 +360,7 @@ func (s *ServerService) GetDefaultLogOutboundTags() (freedoms, blackholes []stri
|
||||
// Kept for back-compat with the original /panel/api/server/cpuHistory/:bucket route;
|
||||
// the response key is "cpu" (not "v") so legacy consumers parse unchanged.
|
||||
func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []map[string]any {
|
||||
out := systemMetrics.aggregate("cpu", bucketSeconds, maxPoints)
|
||||
out := systemMetrics.aggregate(s.systemMetricKey("cpu"), bucketSeconds, maxPoints)
|
||||
for _, p := range out {
|
||||
p["cpu"] = p["v"]
|
||||
delete(p, "v")
|
||||
@@ -367,7 +372,7 @@ func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []
|
||||
// known system metric (see SystemMetricKeys). Output points have keys
|
||||
// {"t": unixSec, "v": value}; the caller decides how to format the value.
|
||||
func (s *ServerService) AggregateSystemMetric(metric string, bucketSeconds int, maxPoints int) []map[string]any {
|
||||
return systemMetrics.aggregate(metric, bucketSeconds, maxPoints)
|
||||
return systemMetrics.aggregate(s.systemMetricKey(metric), bucketSeconds, maxPoints)
|
||||
}
|
||||
|
||||
type LogEntry struct {
|
||||
@@ -526,53 +531,61 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
|
||||
T: now,
|
||||
}
|
||||
|
||||
// CPU stats
|
||||
util, err := s.sampleCPUUtilization()
|
||||
if err != nil {
|
||||
logger.Warning("get cpu percent failed:", err)
|
||||
scoped, scopeErr := s.lxcMetrics.Read(now)
|
||||
if scopeErr != nil {
|
||||
logger.Warning("get scoped metrics failed:", scopeErr)
|
||||
}
|
||||
if scoped != nil {
|
||||
applyLXCStats(status, scoped)
|
||||
} else {
|
||||
status.Cpu = util
|
||||
}
|
||||
|
||||
status.CpuCores, err = cpu.Counts(false)
|
||||
if err != nil {
|
||||
logger.Warning("get cpu cores count failed:", err)
|
||||
}
|
||||
|
||||
status.LogicalPro = runtime.NumCPU()
|
||||
|
||||
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
|
||||
s.lastCpuInfoAttempt = time.Now()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
cpuInfos, err := cpu.Info()
|
||||
if err != nil {
|
||||
logger.Warning("get cpu info failed:", err)
|
||||
return
|
||||
}
|
||||
if len(cpuInfos) > 0 {
|
||||
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
|
||||
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
|
||||
} else {
|
||||
logger.Warning("could not find cpu info")
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(1500 * time.Millisecond):
|
||||
logger.Warning("cpu info query timed out; will retry later")
|
||||
// CPU stats
|
||||
util, err := s.sampleCPUUtilization()
|
||||
if err != nil {
|
||||
logger.Warning("get cpu percent failed:", err)
|
||||
} else {
|
||||
status.Cpu = util
|
||||
}
|
||||
} else if s.cachedCpuSpeedMhz != 0 {
|
||||
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
|
||||
}
|
||||
|
||||
// Uptime
|
||||
upTime, err := host.Uptime()
|
||||
if err != nil {
|
||||
logger.Warning("get uptime failed:", err)
|
||||
} else {
|
||||
status.Uptime = upTime
|
||||
status.CpuCores, err = cpu.Counts(false)
|
||||
if err != nil {
|
||||
logger.Warning("get cpu cores count failed:", err)
|
||||
}
|
||||
|
||||
status.LogicalPro = runtime.NumCPU()
|
||||
|
||||
if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute {
|
||||
s.lastCpuInfoAttempt = time.Now()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
cpuInfos, err := cpu.Info()
|
||||
if err != nil {
|
||||
logger.Warning("get cpu info failed:", err)
|
||||
return
|
||||
}
|
||||
if len(cpuInfos) > 0 {
|
||||
s.cachedCpuSpeedMhz = cpuInfos[0].Mhz
|
||||
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
|
||||
} else {
|
||||
logger.Warning("could not find cpu info")
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(1500 * time.Millisecond):
|
||||
logger.Warning("cpu info query timed out; will retry later")
|
||||
}
|
||||
} else if s.cachedCpuSpeedMhz != 0 {
|
||||
status.CpuSpeedMhz = s.cachedCpuSpeedMhz
|
||||
}
|
||||
|
||||
// Uptime
|
||||
upTime, err := host.Uptime()
|
||||
if err != nil {
|
||||
logger.Warning("get uptime failed:", err)
|
||||
} else {
|
||||
status.Uptime = upTime
|
||||
}
|
||||
}
|
||||
|
||||
// Memory stats
|
||||
@@ -584,12 +597,14 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
|
||||
status.Mem.Total = memInfo.Total
|
||||
}
|
||||
|
||||
swapInfo, err := mem.SwapMemory()
|
||||
if err != nil {
|
||||
logger.Warning("get swap memory failed:", err)
|
||||
} else {
|
||||
status.Swap.Current = swapInfo.Used
|
||||
status.Swap.Total = swapInfo.Total
|
||||
if scoped == nil {
|
||||
swapInfo, err := mem.SwapMemory()
|
||||
if err != nil {
|
||||
logger.Warning("get swap memory failed:", err)
|
||||
} else {
|
||||
status.Swap.Current = swapInfo.Used
|
||||
status.Swap.Total = swapInfo.Total
|
||||
}
|
||||
}
|
||||
|
||||
// Disk stats
|
||||
@@ -736,7 +751,7 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status {
|
||||
// AppendCpuSample is preserved for callers that only have the CPU number.
|
||||
// New callers should prefer AppendStatusSample which writes the full set.
|
||||
func (s *ServerService) AppendCpuSample(t time.Time, v float64) {
|
||||
systemMetrics.append("cpu", t, v)
|
||||
systemMetrics.append(s.systemMetricKey("cpu"), t, v)
|
||||
}
|
||||
|
||||
// AppendStatusSample writes one tick of every metric we keep — CPU, memory
|
||||
@@ -747,14 +762,14 @@ func (s *ServerService) AppendStatusSample(t time.Time, status *Status) {
|
||||
if status == nil {
|
||||
return
|
||||
}
|
||||
systemMetrics.append("cpu", t, status.Cpu)
|
||||
appendScopedMetric(status, "cpu", t, status.Cpu)
|
||||
if status.Mem.Total > 0 {
|
||||
systemMetrics.append("mem", t, float64(status.Mem.Current)*100.0/float64(status.Mem.Total))
|
||||
}
|
||||
if status.Swap.Total > 0 {
|
||||
systemMetrics.append("swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
|
||||
appendScopedMetric(status, "swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total))
|
||||
} else {
|
||||
systemMetrics.append("swap", t, 0)
|
||||
appendScopedMetric(status, "swap", t, 0)
|
||||
}
|
||||
systemMetrics.append("netUp", t, float64(status.NetIO.Up))
|
||||
systemMetrics.append("netDown", t, float64(status.NetIO.Down))
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
|
||||
)
|
||||
|
||||
const containerHistoryPrefix = "lxc-v1/"
|
||||
|
||||
func scopedMetric(metric string) bool {
|
||||
return metric == "cpu" || metric == "swap"
|
||||
}
|
||||
|
||||
func (s *ServerService) systemMetricKey(metric string) string {
|
||||
if !scopedMetric(metric) {
|
||||
return metric
|
||||
}
|
||||
if status := s.LastStatus(); status != nil {
|
||||
if status.ResourceScope != "" {
|
||||
return containerHistoryPrefix + metric
|
||||
}
|
||||
return metric
|
||||
}
|
||||
if s.lxcMetrics.UseScopedHistory() {
|
||||
return containerHistoryPrefix + metric
|
||||
}
|
||||
return metric
|
||||
}
|
||||
|
||||
// Scope metadata applies only to CPU, swap and uptime.
|
||||
func applyLXCStats(status *Status, sample *sys.LXCStats) {
|
||||
status.ResourceScope = sample.Scope
|
||||
status.UnavailableMetrics = append([]string(nil), sample.Unavailable...)
|
||||
status.Cpu = sample.CPU
|
||||
status.CpuCores = sample.CPUs
|
||||
status.LogicalPro = sample.CPUs
|
||||
status.Swap.Current = sample.SwapUsed
|
||||
status.Swap.Total = sample.SwapTotal
|
||||
status.Uptime = sample.Uptime
|
||||
}
|
||||
|
||||
// MetricAvailable distinguishes an unavailable metric from a measured zero.
|
||||
// Legacy status responses omit metadata and retain their previous semantics.
|
||||
func (s *Status) MetricAvailable(metric string) bool {
|
||||
return s != nil && !slices.Contains(s.UnavailableMetrics, metric)
|
||||
}
|
||||
|
||||
func appendScopedMetric(status *Status, metric string, t time.Time, value float64) {
|
||||
if !status.MetricAvailable(metric) {
|
||||
return
|
||||
}
|
||||
if status.ResourceScope != "" {
|
||||
metric = containerHistoryPrefix + metric
|
||||
}
|
||||
systemMetrics.append(metric, t, value)
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//go:build linux
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/util/sys"
|
||||
)
|
||||
|
||||
func TestContainerStatusOnlyOverridesCPUSwapAndUptime(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
t.Setenv("XUI_DB_FOLDER", root)
|
||||
dbtest.InitDB(t, filepath.Join(root, "test.db"))
|
||||
sys.SelfRSS()
|
||||
proc, cg := filepath.Join(root, "proc"), filepath.Join(root, "cg")
|
||||
fixtures := map[string]string{
|
||||
filepath.Join(proc, "self/mountinfo"): "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n4 0 0:28 / " + cg + " rw - cgroup2 cgroup rw\n",
|
||||
filepath.Join(proc, "1/cgroup"): "0::/init.scope\n",
|
||||
filepath.Join(proc, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n",
|
||||
filepath.Join(proc, "cpuinfo"): "processor : 0\ncpu cores : 16\nprocessor : 1\ncpu cores : 16\n",
|
||||
filepath.Join(proc, "uptime"): "52624.93 52624.93\n",
|
||||
filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/5 1\n",
|
||||
filepath.Join(proc, "diskstats"): "8 0 vda 1 0 2 0 1 0 4 0 0 0 0\n",
|
||||
filepath.Join(proc, "stat"): "cpu 100 0 50 1000 0 0 0 0\n",
|
||||
filepath.Join(proc, "net/dev"): "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n",
|
||||
filepath.Join(cg, "cpu.stat"): "usage_usec 222772076\n",
|
||||
filepath.Join(cg, "cpu.max"): "max 100000\n",
|
||||
filepath.Join(cg, "io.stat"): "259:0 rbytes=999999999 wbytes=999999999\n",
|
||||
}
|
||||
for path, content := range fixtures {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
t.Setenv("HOST_PROC", proc)
|
||||
t.Setenv("HOST_SYS", filepath.Join(root, "sys"))
|
||||
svc := &ServerService{cachedIPv4: "192.0.2.1", noIPv6: true, cachedCpuSpeedMhz: 5000, lxcMetrics: sys.LXCReader{ProcRoot: proc, CgroupRoot: cg}}
|
||||
previous := &Status{T: time.Now().Add(-time.Second)}
|
||||
got := svc.GetStatus(previous)
|
||||
if got.ResourceScope != "container" || got.CpuCores != 2 || got.LogicalPro != 2 || got.CpuSpeedMhz != 0 || got.Swap.Total != 512<<20 || got.Swap.Current != 19357696 || got.Uptime != 52624 {
|
||||
t.Fatalf("incorrect corrected resources: %+v", got)
|
||||
}
|
||||
if got.Mem.Total != 2<<30 || got.Mem.Current != 128<<20 || !reflect.DeepEqual(got.Loads, []float64{1.25, 2.5, 3.75}) || got.DiskTraffic.Read != 1024 || got.DiskTraffic.Write != 2048 || got.DiskIO.Read == 0 || got.DiskIO.Write == 0 {
|
||||
t.Fatalf("unrelated metric path changed: %+v", got)
|
||||
}
|
||||
if !slices.Equal(got.UnavailableMetrics, []string{"cpu"}) {
|
||||
t.Fatalf("unexpected unavailable metrics: %v", got.UnavailableMetrics)
|
||||
}
|
||||
raw, err := json.Marshal(got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wire map[string]json.RawMessage
|
||||
if err = json.Unmarshal(raw, &wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"cpu", "cpuCores", "logicalPro", "swap", "uptime", "mem", "loads", "diskIO", "diskTraffic", "resourceScope", "unavailableMetrics"} {
|
||||
if _, ok := wire[key]; !ok {
|
||||
t.Fatalf("API field lost: %s", key)
|
||||
}
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(proc, "self/mountinfo"), []byte("1 0 0:1 / / rw - ext4 /dev/root rw\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
native := svc.GetStatus(previous)
|
||||
if native.ResourceScope != "" || len(native.UnavailableMetrics) != 0 || native.CpuSpeedMhz != 5000 || native.Mem != got.Mem || !reflect.DeepEqual(native.Loads, got.Loads) || native.DiskTraffic != got.DiskTraffic {
|
||||
t.Fatalf("native regression: %+v", native)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScopedResourcesLeaveUnrelatedStatusUntouched(t *testing.T) {
|
||||
status := &Status{Loads: []float64{1, 2, 3}, CpuSpeedMhz: 0}
|
||||
status.Mem.Current, status.Mem.Total = 123, 456
|
||||
status.DiskTraffic.Read, status.DiskTraffic.Write = 99, 101
|
||||
status.DiskIO.Read, status.DiskIO.Write = 11, 12
|
||||
before := *status
|
||||
applyLXCStats(status, &sys.LXCStats{Scope: "container", CPUs: 2, SwapUsed: 8, SwapTotal: 512, Uptime: 15})
|
||||
if status.Mem != before.Mem || status.DiskTraffic != before.DiskTraffic || status.DiskIO != before.DiskIO || !reflect.DeepEqual(status.Loads, before.Loads) {
|
||||
t.Fatalf("unrelated status overwritten: %+v", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerHistorySeparatesOnlyCPUAndSwap(t *testing.T) {
|
||||
old := systemMetrics
|
||||
systemMetrics = newMetricHistory()
|
||||
defer func() { systemMetrics = old }()
|
||||
now := time.Now()
|
||||
systemMetrics.append("cpu", now, 90)
|
||||
systemMetrics.append("swap", now, 62)
|
||||
for key, value := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
|
||||
systemMetrics.append(key, now, value)
|
||||
}
|
||||
status := &Status{T: now, ResourceScope: "container", Cpu: 10, Loads: []float64{1.25, 2.5, 3.75}, UnavailableMetrics: []string{"swap"}}
|
||||
status.Mem.Total, status.Mem.Current = 100, 6
|
||||
status.DiskIO.Read, status.NetIO.Up = 1024, 7
|
||||
svc := &ServerService{lastStatus: status}
|
||||
svc.AppendStatusSample(now, status)
|
||||
cpu := svc.AggregateCpuHistory(2, 60)
|
||||
if len(cpu) != 1 || cpu[0]["cpu"] != float64(10) {
|
||||
t.Fatalf("CPU history=%v", cpu)
|
||||
}
|
||||
native := &ServerService{lastStatus: &Status{}}
|
||||
if host := native.AggregateCpuHistory(2, 60); len(host) != 1 || host[0]["cpu"] != float64(90) {
|
||||
t.Fatalf("legacy CPU changed: %v", host)
|
||||
}
|
||||
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 0 {
|
||||
t.Fatalf("missing swap became zero: %v", points)
|
||||
}
|
||||
for key, want := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} {
|
||||
if svc.systemMetricKey(key) != key {
|
||||
t.Fatalf("unrelated history renamed: %s", key)
|
||||
}
|
||||
points := svc.AggregateSystemMetric(key, 2, 60)
|
||||
if len(points) != 1 || points[0]["v"] != want {
|
||||
t.Fatalf("unrelated %s history=%v", key, points)
|
||||
}
|
||||
}
|
||||
status.UnavailableMetrics = nil
|
||||
svc.AppendStatusSample(now, status)
|
||||
if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 1 || points[0]["v"] != float64(0) {
|
||||
t.Fatalf("disabled swap lost: %v", points)
|
||||
}
|
||||
snapshot := systemMetrics.snapshot()
|
||||
for _, key := range []string{"cpu", "swap", containerHistoryPrefix + "cpu", containerHistoryPrefix + "swap"} {
|
||||
if _, ok := snapshot[key]; !ok {
|
||||
t.Fatalf("missing preserved series: %s", key)
|
||||
}
|
||||
}
|
||||
for key := range snapshot {
|
||||
if len(key) > len(containerHistoryPrefix) && key[:len(containerHistoryPrefix)] == containerHistoryPrefix && key != containerHistoryPrefix+"cpu" && key != containerHistoryPrefix+"swap" {
|
||||
t.Fatalf("unexpected scoped metric: %s", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -187,7 +187,12 @@ func (t *Tgbot) prepareServerUsageInfo() string {
|
||||
info += t.I18nBot("tgbot.messages.ipv6", "IPv6=="+ipv6)
|
||||
}
|
||||
|
||||
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+strconv.FormatUint(t.lastStatus.Uptime/86400, 10), "Unit=="+t.I18nBot("tgbot.days"))
|
||||
uptime, uptimeUnit := "—", ""
|
||||
if t.lastStatus.MetricAvailable("uptime") {
|
||||
uptime = strconv.FormatUint(t.lastStatus.Uptime/86400, 10)
|
||||
uptimeUnit = t.I18nBot("tgbot.days")
|
||||
}
|
||||
info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+uptime, "Unit=="+uptimeUnit)
|
||||
info += t.I18nBot("tgbot.messages.serverLoad", "Load1=="+strconv.FormatFloat(t.lastStatus.Loads[0], 'f', 2, 64), "Load2=="+strconv.FormatFloat(t.lastStatus.Loads[1], 'f', 2, 64), "Load3=="+strconv.FormatFloat(t.lastStatus.Loads[2], 'f', 2, 64))
|
||||
info += t.I18nBot("tgbot.messages.serverMemory", "Current=="+common.FormatTraffic(int64(t.lastStatus.Mem.Current)), "Total=="+common.FormatTraffic(int64(t.lastStatus.Mem.Total)))
|
||||
info += t.I18nBot("tgbot.messages.onlinesCount", "Count=="+fmt.Sprint(len(onlines)))
|
||||
|
||||
@@ -14,11 +14,14 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mymmrac/telego"
|
||||
"github.com/nicksnyder/go-i18n/v2/i18n"
|
||||
"golang.org/x/text/language"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database/dbtest"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/web/locale"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/web/service"
|
||||
"github.com/mhsanaei/3x-ui/v3/internal/xray"
|
||||
)
|
||||
@@ -284,3 +287,44 @@ func TestNotifyExhaustedLooksUpEachIDOnce(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerUsageInfoShowsUnavailableUptime(t *testing.T) {
|
||||
bundle := i18n.NewBundle(language.English)
|
||||
_ = bundle.AddMessages(language.English, &i18n.Message{ID: "tgbot.messages.serverUpTime", Other: "Uptime: {{.UpTime}} {{.Unit}}\n"}, &i18n.Message{ID: "tgbot.days", Other: "days"})
|
||||
oldLocale := locale.LocalizerBot
|
||||
locale.LocalizerBot = i18n.NewLocalizer(bundle, "en")
|
||||
t.Cleanup(func() { locale.LocalizerBot = oldLocale })
|
||||
statusCache.mutex.Lock()
|
||||
oldStatus, oldTime := statusCache.data, statusCache.timestamp
|
||||
statusCache.mutex.Unlock()
|
||||
serverStatsCache.mutex.Lock()
|
||||
oldStats, oldStatsTime := serverStatsCache.data, serverStatsCache.timestamp
|
||||
serverStatsCache.mutex.Unlock()
|
||||
t.Cleanup(func() {
|
||||
statusCache.mutex.Lock()
|
||||
statusCache.data, statusCache.timestamp = oldStatus, oldTime
|
||||
statusCache.mutex.Unlock()
|
||||
serverStatsCache.mutex.Lock()
|
||||
serverStatsCache.data, serverStatsCache.timestamp = oldStats, oldStatsTime
|
||||
serverStatsCache.mutex.Unlock()
|
||||
})
|
||||
for _, unavailable := range []bool{true, false} {
|
||||
t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) {
|
||||
bot := &Tgbot{}
|
||||
status := &service.Status{Loads: []float64{0, 0, 0}}
|
||||
if unavailable {
|
||||
status.UnavailableMetrics = []string{"uptime"}
|
||||
}
|
||||
bot.setCachedStatus(status)
|
||||
bot.setCachedServerStats("")
|
||||
info := bot.prepareServerUsageInfo()
|
||||
want := "Uptime: 0 days"
|
||||
if unavailable {
|
||||
want = "Uptime: —"
|
||||
}
|
||||
if !strings.Contains(info, want) {
|
||||
t.Fatalf("report did not contain %q: %s", want, info)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user