diff --git a/docs/public/openapi.json b/docs/public/openapi.json index d34729d6e..a155f78e4 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -3740,6 +3740,13 @@ "transitive": { "type": "boolean" }, + "unavailableMetrics": { + "description": "Nil on legacy rows and responses: zero remains a valid measured value.", + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700000000, "format": "int64", @@ -4048,6 +4055,12 @@ "example": false, "type": "boolean" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700003600, "format": "int64", @@ -4260,6 +4273,12 @@ "example": "online", "type": "string" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "uptimeSecs": { "example": 86400, "format": "int64", @@ -11163,6 +11182,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11364,6 +11386,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11514,6 +11539,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11737,6 +11765,9 @@ "memPct": 45.2, "panelVersion": "v3.x.x", "status": "online", + "unavailableMetrics": [ + "" + ], "uptimeSecs": 86400, "xrayError": "", "xrayState": "", @@ -16710,6 +16741,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", diff --git a/frontend/public/openapi.json b/frontend/public/openapi.json index d34729d6e..a155f78e4 100644 --- a/frontend/public/openapi.json +++ b/frontend/public/openapi.json @@ -3740,6 +3740,13 @@ "transitive": { "type": "boolean" }, + "unavailableMetrics": { + "description": "Nil on legacy rows and responses: zero remains a valid measured value.", + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700000000, "format": "int64", @@ -4048,6 +4055,12 @@ "example": false, "type": "boolean" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700003600, "format": "int64", @@ -4260,6 +4273,12 @@ "example": "online", "type": "string" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "uptimeSecs": { "example": 86400, "format": "int64", @@ -11163,6 +11182,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11364,6 +11386,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11514,6 +11539,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -11737,6 +11765,9 @@ "memPct": 45.2, "panelVersion": "v3.x.x", "status": "online", + "unavailableMetrics": [ + "" + ], "uptimeSecs": 86400, "xrayError": "", "xrayState": "", @@ -16710,6 +16741,9 @@ "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", diff --git a/frontend/src/generated/examples.ts b/frontend/src/generated/examples.ts index 25b586539..0af9b99c2 100644 --- a/frontend/src/generated/examples.ts +++ b/frontend/src/generated/examples.ts @@ -944,6 +944,9 @@ export const EXAMPLES: Record = { "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700000000, "uptimeSecs": 86400, "xrayError": "", @@ -1009,6 +1012,9 @@ export const EXAMPLES: Record = { "status": "online", "tlsVerifyMode": "verify", "transitive": false, + "unavailableMetrics": [ + "" + ], "updatedAt": 1700003600, "uptimeSecs": 86400, "xrayError": "", @@ -1047,6 +1053,9 @@ export const EXAMPLES: Record = { "memPct": 45.2, "panelVersion": "v3.x.x", "status": "online", + "unavailableMetrics": [ + "" + ], "uptimeSecs": 86400, "xrayError": "", "xrayState": "", diff --git a/frontend/src/generated/schemas.ts b/frontend/src/generated/schemas.ts index 070b32d8c..28cc3f3ac 100644 --- a/frontend/src/generated/schemas.ts +++ b/frontend/src/generated/schemas.ts @@ -3714,6 +3714,13 @@ export const SCHEMAS: Record = { "transitive": { "type": "boolean" }, + "unavailableMetrics": { + "description": "Nil on legacy rows and responses: zero remains a valid measured value.", + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700000000, "format": "int64", @@ -4022,6 +4029,12 @@ export const SCHEMAS: Record = { "example": false, "type": "boolean" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "updatedAt": { "example": 1700003600, "format": "int64", @@ -4234,6 +4247,12 @@ export const SCHEMAS: Record = { "example": "online", "type": "string" }, + "unavailableMetrics": { + "items": { + "type": "string" + }, + "type": "array" + }, "uptimeSecs": { "example": 86400, "format": "int64", diff --git a/frontend/src/generated/types.ts b/frontend/src/generated/types.ts index 5eaff4592..8522f0ce6 100644 --- a/frontend/src/generated/types.ts +++ b/frontend/src/generated/types.ts @@ -855,6 +855,7 @@ export interface Node { status: string; tlsVerifyMode: string; transitive?: boolean; + unavailableMetrics?: string[]; updatedAt: number; uptimeSecs: number; xrayError: string; @@ -918,6 +919,7 @@ export interface NodeView { status: string; tlsVerifyMode: string; transitive?: boolean; + unavailableMetrics?: string[]; updatedAt: number; uptimeSecs: number; xrayError: string; @@ -960,6 +962,7 @@ export interface ProbeResultUI { memPct: number; panelVersion: string; status: string; + unavailableMetrics?: string[]; uptimeSecs: number; xrayError: string; xrayState: string; diff --git a/frontend/src/generated/zod.ts b/frontend/src/generated/zod.ts index cce59a990..550301f97 100644 --- a/frontend/src/generated/zod.ts +++ b/frontend/src/generated/zod.ts @@ -908,6 +908,7 @@ export const NodeSchema = z.object({ status: z.string(), tlsVerifyMode: z.enum(['verify', 'skip', 'pin', 'mtls']), transitive: z.boolean().optional(), + unavailableMetrics: z.array(z.string()).optional(), updatedAt: z.number().int(), uptimeSecs: z.number().int(), xrayError: z.string(), @@ -973,6 +974,7 @@ export const NodeViewSchema = z.object({ status: z.string(), tlsVerifyMode: z.string(), transitive: z.boolean().optional(), + unavailableMetrics: z.array(z.string()).optional(), updatedAt: z.number().int(), uptimeSecs: z.number().int(), xrayError: z.string(), @@ -1019,6 +1021,7 @@ export const ProbeResultUISchema = z.object({ memPct: z.number(), panelVersion: z.string(), status: z.string(), + unavailableMetrics: z.array(z.string()).optional(), uptimeSecs: z.number().int(), xrayError: z.string(), xrayState: z.string(), diff --git a/frontend/src/models/status.ts b/frontend/src/models/status.ts index 6909717db..0fe2c8ab1 100644 --- a/frontend/src/models/status.ts +++ b/frontend/src/models/status.ts @@ -1,4 +1,5 @@ import { NumberFormatter } from '@/utils'; +import type { StatusInput as ParsedStatusInput } from '@/schemas/status'; export const USAGE_WARN_PERCENT = 80; export const USAGE_CRIT_PERCENT = 90; @@ -66,7 +67,7 @@ export interface AmneziaWGInfo { running: boolean; } -interface StatusInput { +interface StatusInput extends Pick { cpu?: number; cpuCores?: number; logicalPro?: number; @@ -88,6 +89,13 @@ interface StatusInput { } export class Status { + resourceScope = ''; + unavailableMetrics: string[] = []; + + metricAvailable(metric: string): boolean { + return !this.unavailableMetrics.includes(metric); + } + cpu: CurTotal = new CurTotal(0, 0); cpuCores = 0; logicalPro = 0; @@ -110,6 +118,8 @@ export class Status { constructor(data?: StatusInput | null) { if (data == null) return; + this.resourceScope = data.resourceScope ?? ''; + this.unavailableMetrics = data.unavailableMetrics ?? []; this.cpu = new CurTotal(data.cpu ?? 0, 100); this.cpuCores = data.cpuCores ?? 0; this.logicalPro = data.logicalPro ?? 0; diff --git a/frontend/src/pages/index/ContainerMetrics.stories.tsx b/frontend/src/pages/index/ContainerMetrics.stories.tsx new file mode 100644 index 000000000..047b196c3 --- /dev/null +++ b/frontend/src/pages/index/ContainerMetrics.stories.tsx @@ -0,0 +1,67 @@ +import type { Meta, StoryObj } from '@storybook/react-vite'; +import { DatabaseOutlined } from '@ant-design/icons'; +import { expect, within } from 'storybook/test'; + +import { Status } from '@/models/status'; +import VitalTile from './VitalTile'; +import SystemStrip from './SystemStrip'; +import './IndexPage.css'; + +function ContainerMetricsPreview({ unavailable = false }: { unavailable?: boolean }) { + const status = new Status({ + resourceScope: 'container', + unavailableMetrics: unavailable ? ['swap', 'uptime'] : [], + swap: { current: 19357696, total: 536870912 }, + uptime: 52624, + appStats: { mem: 104857600, threads: 37, uptime: 50000 }, + }); + return ( +
+
+ } + label="SWAP" + available={status.metricAvailable('swap')} + percent={status.swap.percent} + statusColor={status.swap.color} + detail="18.46 MiB / 512 MiB" + footLeft="Container" + footRight="2 vCPU" + data={unavailable ? [] : [3.5, 3.6, 3.6]} + isMobile={false} + /> +
+ undefined} /> +
+ ); +} + +const meta = { + title: 'Monitoring/Container metrics', + component: ContainerMetricsPreview, + tags: ['autodocs'], + parameters: { layout: 'padded' }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Measured: Story = { + args: { unavailable: false }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.getByText('18.46 MiB / 512 MiB')).toBeVisible(); + await expect(canvas.getByText('CT')).toBeVisible(); + await expect(canvas.queryByText('OS')).not.toBeInTheDocument(); + }, +}; + +export const Unavailable: Story = { + args: { unavailable: true }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await expect(canvas.queryByText('18.46 MiB / 512 MiB')).not.toBeInTheDocument(); + await expect(canvas.queryByText('%')).not.toBeInTheDocument(); + await expect(canvas.getAllByText('—').length).toBeGreaterThan(0); + }, +}; diff --git a/frontend/src/pages/index/IndexPage.tsx b/frontend/src/pages/index/IndexPage.tsx index 6aa0dd1c7..c6d4fefba 100644 --- a/frontend/src/pages/index/IndexPage.tsx +++ b/frontend/src/pages/index/IndexPage.tsx @@ -229,9 +229,14 @@ export default function IndexPage() { } label={t('pages.index.cpu')} + available={status.metricAvailable('cpu')} percent={status.cpu.percent} statusColor={status.cpu.color} - detail={`${CPUFormatter.cpuCoreFormat(status.cpuCores)} / ${status.logicalPro}T · ${CPUFormatter.cpuSpeedFormat(status.cpuSpeedMhz)}`} + detail={ + status.resourceScope === 'container' + ? `${status.logicalPro} vCPU` + : `${CPUFormatter.cpuCoreFormat(status.cpuCores)} / ${status.logicalPro}T · ${CPUFormatter.cpuSpeedFormat(status.cpuSpeedMhz)}` + } footLeft={`${t('pages.index.avg')} ${mean(history.series.cpu).toFixed(0)}%`} footRight={`${t('pages.index.peak')} ${peak(history.series.cpu).toFixed(0)}%`} data={history.series.cpu} @@ -251,6 +256,7 @@ export default function IndexPage() { } label={t('pages.index.swap')} + available={status.metricAvailable('swap')} percent={status.swap.percent} statusColor={status.swap.color} detail={`${SizeFormatter.sizeFormat(status.swap.current)} / ${SizeFormatter.sizeFormat(status.swap.total)}`} diff --git a/frontend/src/pages/index/SystemStrip.tsx b/frontend/src/pages/index/SystemStrip.tsx index c2ce5b78b..511532d96 100644 --- a/frontend/src/pages/index/SystemStrip.tsx +++ b/frontend/src/pages/index/SystemStrip.tsx @@ -38,8 +38,12 @@ export default function SystemStrip({ status, showIp, onToggleIp }: SystemStripP
-
OS
-
{TimeFormatter.formatSecond(status.uptime)}
+
+ {status.resourceScope === 'container' ? 'CT' : 'OS'} +
+
+ {status.metricAvailable('uptime') ? TimeFormatter.formatSecond(status.uptime) : '—'} +
diff --git a/frontend/src/pages/index/VitalTile.tsx b/frontend/src/pages/index/VitalTile.tsx index 24bf49b69..151abf1c9 100644 --- a/frontend/src/pages/index/VitalTile.tsx +++ b/frontend/src/pages/index/VitalTile.tsx @@ -6,6 +6,7 @@ import { Sparkline } from '@/components/viz'; import { mean, peak } from './useOverviewHistory'; interface VitalTileProps { + available?: boolean; icon: ReactNode; label: string; percent: number; @@ -18,6 +19,7 @@ interface VitalTileProps { } export default function VitalTile({ + available = true, icon, label, percent, @@ -44,15 +46,15 @@ export default function VitalTile({
- {percent.toFixed(1)} - % + {available ? percent.toFixed(1) : '—'} + {available && %}
-
{detail}
+
{available ? detail : '—'}
- {footLeft} - {footRight} + {available ? footLeft : '—'} + {available ? footRight : '—'}
diff --git a/frontend/src/pages/index/useOverviewHistory.ts b/frontend/src/pages/index/useOverviewHistory.ts index 6266a24d2..e07865f7d 100644 --- a/frontend/src/pages/index/useOverviewHistory.ts +++ b/frontend/src/pages/index/useOverviewHistory.ts @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import { HttpUtil, TimeFormatter } from '@/utils'; import type { Status } from '@/models/status'; @@ -80,13 +80,27 @@ export function peak(values: number[]): number { export function useOverviewHistory(status: Status, hasData: boolean): OverviewHistory { const [trend, setTrend] = useState(emptyWindow); + const seededUnscoped = useRef(false); + const seedState = useRef<{ scope: string; complete: boolean } | null>(null); + useEffect(() => { + // The placeholder has no resource scope. Wait for the first real status, + // then seed each scope once even if polling temporarily loses data. + if ( + !hasData || + (seedState.current?.scope === status.resourceScope && seedState.current.complete) + ) { + return; + } + seedState.current = { scope: status.resourceScope, complete: false }; let cancelled = false; + const seedUnscoped = !seededUnscoped.current; + const keys = seedUnscoped ? SERIES_KEYS : (['cpu', 'swap'] as const); const seed = async () => { const responses = new Map(); await Promise.all( - SERIES_KEYS.map(async (key) => { + keys.map(async (key) => { const msg = await HttpUtil.get( `/panel/api/server/history/${key}/${SEED_BUCKET_SECONDS}`, undefined, @@ -96,6 +110,8 @@ export function useOverviewHistory(status: Status, hasData: boolean): OverviewHi }), ); if (cancelled || responses.size === 0) return; + seedState.current = { scope: status.resourceScope, complete: true }; + seededUnscoped.current = true; let axis: HistoryPoint[] = []; for (const points of responses.values()) { @@ -109,14 +125,20 @@ export function useOverviewHistory(status: Status, hasData: boolean): OverviewHi for (const key of SERIES_KEYS) { const byTs = new Map(); for (const p of responses.get(key) ?? []) byTs.set(Number(p.t) || 0, Number(p.v) || 0); - seedSeries[key] = seedTimes.map((ts) => byTs.get(ts) ?? 0); + seedSeries[key] = + status.resourceScope && (key === 'cpu' || key === 'swap') + ? seedTimes.flatMap((ts) => (byTs.has(ts) ? [byTs.get(ts)!] : [])) + : seedTimes.map((ts) => byTs.get(ts) ?? 0); } setTrend((prev) => { const merged = emptyWindow(); - merged.times = tailWindow(seedTimes.concat(prev.times)); + merged.times = seedUnscoped ? tailWindow(seedTimes.concat(prev.times)) : prev.times; for (const key of SERIES_KEYS) { - merged.series[key] = tailWindow(seedSeries[key].concat(prev.series[key])); + merged.series[key] = + seedUnscoped || key === 'cpu' || key === 'swap' + ? tailWindow(seedSeries[key].concat(prev.series[key])) + : prev.series[key]; } return merged; }); @@ -126,7 +148,7 @@ export function useOverviewHistory(status: Status, hasData: boolean): OverviewHi return () => { cancelled = true; }; - }, []); + }, [hasData, status.resourceScope]); // Each polled status is appended during render; an effect would show the // chart one sample behind the numbers beside it. @@ -134,11 +156,17 @@ export function useOverviewHistory(status: Status, hasData: boolean): OverviewHi if (hasData && status !== sampledStatus) { setSampledStatus(status); setTrend((prev) => { + const scopeChanged = sampledStatus && sampledStatus.resourceScope !== status.resourceScope; const point = sampleOf(status); const next = emptyWindow(); next.times = tailWindow(prev.times.concat(Math.floor(Date.now() / 1000))); for (const key of SERIES_KEYS) { - next.series[key] = tailWindow(prev.series[key].concat(point[key])); + const scoped = key === 'cpu' || key === 'swap'; + const previous = scoped && scopeChanged ? [] : prev.series[key]; + next.series[key] = + !scoped || status.metricAvailable(key) + ? tailWindow(previous.concat(point[key])) + : previous; } return next; }); diff --git a/frontend/src/pages/nodes/NodeList.tsx b/frontend/src/pages/nodes/NodeList.tsx index e2090e26c..b8cf089a3 100644 --- a/frontend/src/pages/nodes/NodeList.tsx +++ b/frontend/src/pages/nodes/NodeList.tsx @@ -130,12 +130,14 @@ function StatusLabel({ status, xrayState }: HealthProps) { return {t(`pages.nodes.statusValues.${status || 'unknown'}`)}; } -function formatPct(p?: number): string { +function formatPct(p?: number, available = true): string { + if (!available) return '—'; if (typeof p !== 'number' || Number.isNaN(p)) return '-'; return `${p.toFixed(1)}%`; } -function formatUptime(secs?: number): string { +function formatUptime(secs?: number, available = true): string { + if (!available) return '—'; if (!secs) return '-'; const days = Math.floor(secs / 86400); const hours = Math.floor((secs % 86400) / 3600); @@ -404,7 +406,8 @@ export default function NodeList({ dataIndex: 'cpuPct', align: 'center', width: 90, - render: (_value, record) => formatPct(record.cpuPct), + render: (_value, record) => + formatPct(record.cpuPct, !record.unavailableMetrics?.includes('cpu')), }, { title: t('pages.nodes.mem'), @@ -452,7 +455,8 @@ export default function NodeList({ title: t('pages.nodes.uptime'), dataIndex: 'uptimeSecs', align: 'center', - render: (_value, record) => formatUptime(record.uptimeSecs), + render: (_value, record) => + formatUptime(record.uptimeSecs, !record.unavailableMetrics?.includes('uptime')), }, { title: t('clients'), @@ -792,7 +796,9 @@ export default function NodeList({
{t('pages.nodes.cpu')} - {formatPct(statsNode.cpuPct)} + + {formatPct(statsNode.cpuPct, !statsNode.unavailableMetrics?.includes('cpu'))} +
{t('pages.nodes.mem')} @@ -810,7 +816,12 @@ export default function NodeList({
{t('pages.nodes.uptime')} - {formatUptime(statsNode.uptimeSecs)} + + {formatUptime( + statsNode.uptimeSecs, + !statsNode.unavailableMetrics?.includes('uptime'), + )} +
{t('pages.nodes.latency')} diff --git a/frontend/src/schemas/node.ts b/frontend/src/schemas/node.ts index bc8583cf2..10efb8242 100644 --- a/frontend/src/schemas/node.ts +++ b/frontend/src/schemas/node.ts @@ -19,6 +19,7 @@ export const NodeRecordSchema = z xrayVersion: z.string().optional(), panelVersion: z.string().optional(), uptimeSecs: z.number().optional(), + unavailableMetrics: z.array(z.string()).optional(), inboundCount: z.number().optional(), clientCount: z.number().optional(), onlineCount: z.number().optional(), @@ -52,6 +53,7 @@ export const NodeListSchema = z.array(NodeRecordSchema); export const ProbeResultSchema = z .object({ status: z.string(), + unavailableMetrics: z.array(z.string()).optional(), latencyMs: z.number().optional(), xrayVersion: z.string().optional(), error: z.string().optional(), diff --git a/frontend/src/schemas/status.ts b/frontend/src/schemas/status.ts index f5943d7d6..406a4e95c 100644 --- a/frontend/src/schemas/status.ts +++ b/frontend/src/schemas/status.ts @@ -36,6 +36,8 @@ export const XrayInfoSchema = z .partial(); export const StatusSchema = z.object({ + resourceScope: z.string().optional(), + unavailableMetrics: z.array(z.string()).optional(), cpu: z.number().optional(), cpuCores: z.number().optional(), logicalPro: z.number().optional(), diff --git a/frontend/src/test/container-history.test.tsx b/frontend/src/test/container-history.test.tsx new file mode 100644 index 000000000..45cdc0e72 --- /dev/null +++ b/frontend/src/test/container-history.test.tsx @@ -0,0 +1,186 @@ +import { act, renderHook, waitFor } from '@testing-library/react'; +import { beforeEach, expect, it, vi } from 'vitest'; + +import { Status } from '@/models/status'; +import { useOverviewHistory } from '@/pages/index/useOverviewHistory'; + +const { get } = vi.hoisted(() => ({ get: vi.fn() })); +vi.mock('@/utils', async (importOriginal) => { + const original = await importOriginal(); + return { ...original, HttpUtil: { get } }; +}); + +beforeEach(() => { + get.mockReset(); +}); + +it('keeps missing samples out of trends and resets a changed metric scope', async () => { + get.mockImplementation(async (path: string) => ({ + success: true, + obj: path.includes('/cpu/') + ? [{ t: 1, v: 10 }] + : [ + { t: 1, v: 5 }, + { t: 2, v: 6 }, + ], + })); + const unavailable = new Status({ + resourceScope: 'container', + cpu: 0, + unavailableMetrics: ['cpu'], + }); + const { result, rerender } = renderHook(({ status }) => useOverviewHistory(status, true), { + initialProps: { status: unavailable }, + }); + await waitFor(() => expect(result.current.series.cpu).toEqual([10])); + rerender({ status: new Status({ resourceScope: 'container', cpu: 20 }) }); + expect(result.current.series.cpu).toEqual([10, 20]); + rerender({ status: new Status({ cpu: 50 }) }); + expect(result.current.series.cpu).toEqual([50]); +}); + +it('ignores a legacy seed that resolves after the scope changes', async () => { + let release!: (value: { success: boolean; obj: { t: number; v: number }[] }) => void; + const legacy = new Promise<{ success: boolean; obj: { t: number; v: number }[] }>((resolve) => { + release = resolve; + }); + let calls = 0; + get.mockImplementation(() => { + calls += 1; + return calls <= 8 ? legacy : Promise.resolve({ success: true, obj: [] }); + }); + const { result, rerender } = renderHook(({ status }) => useOverviewHistory(status, true), { + initialProps: { status: new Status({ cpu: 90 }) }, + }); + rerender({ status: new Status({ resourceScope: 'container', cpu: 10 }) }); + await act(async () => release({ success: true, obj: [{ t: 1, v: 90 }] })); + expect(result.current.series.cpu).toEqual([10]); +}); + +it('preserves unrelated samples and their seed when only CPU/swap scope changes', async () => { + get.mockImplementation(async (path: string) => ({ + success: true, + obj: [{ t: 1, v: path.includes('/netUp/') ? 7 : 5 }], + })); + const { result, rerender } = renderHook(({ status }) => useOverviewHistory(status, true), { + initialProps: { status: new Status({ cpu: 90, netIO: { up: 8, down: 0 } }) }, + }); + await waitFor(() => expect(result.current.series.netUp).toEqual([7, 8])); + get.mockClear(); + rerender({ + status: new Status({ resourceScope: 'container', cpu: 10, netIO: { up: 9, down: 0 } }), + }); + expect(result.current.series.netUp).toEqual([7, 8, 9]); + expect(result.current.series.cpu).toEqual([10]); + await waitFor(() => expect(get).toHaveBeenCalledTimes(2)); + expect(get.mock.calls.map((args) => args[0]).sort()).toEqual([ + '/panel/api/server/history/cpu/2', + '/panel/api/server/history/swap/2', + ]); + await waitFor(() => expect(result.current.series.cpu).toEqual([5, 10])); + expect(result.current.series.netUp).toEqual([7, 8, 9]); +}); + +it('retains legacy padding for unrelated seed gaps while skipping missing container CPU', async () => { + get.mockImplementation(async (path: string) => ({ + success: true, + obj: path.includes('/netUp/') + ? [ + { t: 1, v: 7 }, + { t: 2, v: 8 }, + ] + : [{ t: 1, v: 5 }], + })); + const status = new Status({ + resourceScope: 'container', + unavailableMetrics: ['cpu', 'swap'], + netIO: { up: 9, down: 0 }, + }); + const { result } = renderHook(() => useOverviewHistory(status, true)); + await waitFor(() => expect(result.current.series.netUp).toEqual([7, 8, 9])); + expect(result.current.series.mem).toEqual([5, 0, 0]); + expect(result.current.series.cpu).toEqual([5]); + expect(result.current.series.swap).toEqual([5]); +}); + +it('waits for the first real status before seeding and never doubles that window', async () => { + get.mockImplementation(async (path: string) => ({ + success: true, + obj: path.includes('/cpu/') + ? [{ t: 1, v: 10 }] + : path.includes('/swap/') + ? [{ t: 2, v: 6 }] + : [ + { t: 1, v: 5 }, + { t: 2, v: 6 }, + ], + })); + const { result, rerender } = renderHook( + ({ status, hasData }) => useOverviewHistory(status, hasData), + { + initialProps: { status: new Status(), hasData: false }, + }, + ); + await act(async () => { + await Promise.resolve(); + }); + expect(get).not.toHaveBeenCalled(); + rerender({ + status: new Status({ resourceScope: 'container', cpu: 20, swap: { current: 0, total: 0 } }), + hasData: true, + }); + await waitFor(() => expect(result.current.series.cpu).toEqual([10, 20])); + expect(result.current.series.swap).toEqual([6, 0]); + expect(get).toHaveBeenCalledTimes(8); + rerender({ status: new Status({ resourceScope: 'container', cpu: 30 }), hasData: true }); + expect(result.current.series.cpu).toEqual([10, 20, 30]); + expect(get).toHaveBeenCalledTimes(8); +}); + +it('does not prepend the same seed after a temporary status error', async () => { + get.mockResolvedValue({ success: true, obj: [{ t: 1, v: 10 }] }); + const status = new Status({ resourceScope: 'container', cpu: 20 }); + const { result, rerender } = renderHook(({ hasData }) => useOverviewHistory(status, hasData), { + initialProps: { hasData: true }, + }); + await waitFor(() => expect(result.current.series.cpu).toEqual([10, 20])); + rerender({ hasData: false }); + rerender({ hasData: true }); + await act(async () => { + await Promise.resolve(); + }); + expect(get).toHaveBeenCalledTimes(8); + expect(result.current.series.cpu).toEqual([10, 20]); +}); + +it('does not reseed unrelated metrics after an empty initial history', async () => { + get.mockResolvedValue({ success: true, obj: [] }); + const { rerender } = renderHook(({ status }) => useOverviewHistory(status, true), { + initialProps: { status: new Status({ cpu: 50 }) }, + }); + await act(async () => { + await Promise.resolve(); + }); + get.mockClear(); + rerender({ status: new Status({ resourceScope: 'container', cpu: 20 }) }); + await waitFor(() => expect(get).toHaveBeenCalledTimes(2)); +}); + +it('can return to a previously seeded scope while a different seed is in flight', async () => { + get.mockResolvedValue({ success: true, obj: [{ t: 1, v: 10 }] }); + const { result, rerender } = renderHook(({ status }) => useOverviewHistory(status, true), { + initialProps: { status: new Status({ cpu: 50 }) }, + }); + await waitFor(() => expect(result.current.series.cpu).toEqual([10, 50])); + let release!: (value: { success: boolean; obj: { t: number; v: number }[] }) => void; + const pending = new Promise<{ success: boolean; obj: { t: number; v: number }[] }>((resolve) => { + release = resolve; + }); + get.mockReturnValue(pending); + rerender({ status: new Status({ resourceScope: 'container', cpu: 20 }) }); + get.mockResolvedValue({ success: true, obj: [{ t: 1, v: 10 }] }); + rerender({ status: new Status({ cpu: 60 }) }); + await waitFor(() => expect(result.current.series.cpu).toEqual([10, 60])); + await act(async () => release({ success: true, obj: [{ t: 1, v: 99 }] })); + expect(result.current.series.cpu).toEqual([10, 60]); +}); diff --git a/frontend/src/test/container-metrics.test.ts b/frontend/src/test/container-metrics.test.ts new file mode 100644 index 000000000..feca08c0a --- /dev/null +++ b/frontend/src/test/container-metrics.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; + +import { Status } from '@/models/status'; +import { StatusSchema } from '@/schemas/status'; + +describe('container metric scope', () => { + it('preserves backend scope and availability through validation into the UI model', () => { + const raw = StatusSchema.parse({ + resourceScope: 'container', + unavailableMetrics: ['cpu'], + cpu: 0, + cpuCores: 2, + logicalPro: 2, + mem: { current: 134217728, total: 2147483648 }, + swap: { current: 19357696, total: 536870912 }, + uptime: 52624, + loads: [1.25, 2.5, 3.75], + }); + const status = new Status(raw); + expect(status.resourceScope).toBe('container'); + expect(status.logicalPro).toBe(2); + expect(status.metricAvailable('cpu')).toBe(false); + expect(status.metricAvailable('swap')).toBe(true); + expect(status.swap.current).toBe(19357696); + expect(status.swap.total).toBe(536870912); + expect(status.swap.percent).toBeCloseTo(3.605688, 1); + expect(status.loads).toEqual([1.25, 2.5, 3.75]); + expect(status.metricAvailable('load')).toBe(true); + }); + + it('keeps legacy responses available and distinguishes disabled from missing swap', () => { + const legacy = new Status({ swap: { current: 0, total: 0 } }); + expect(legacy.resourceScope).toBe(''); + expect(legacy.metricAvailable('swap')).toBe(true); + expect(legacy.swap.percent).toBe(0); + const missing = new Status({ unavailableMetrics: ['swap'] }); + expect(missing.metricAvailable('swap')).toBe(false); + }); +}); diff --git a/frontend/src/test/node-metrics-availability.test.tsx b/frontend/src/test/node-metrics-availability.test.tsx new file mode 100644 index 000000000..7a1c4c667 --- /dev/null +++ b/frontend/src/test/node-metrics-availability.test.tsx @@ -0,0 +1,73 @@ +import { fireEvent, screen, within } from '@testing-library/react'; +import { describe, expect, it } from 'vitest'; +import NodeList from '@/pages/nodes/NodeList'; +import { NodeRecordSchema } from '@/schemas/node'; +import { renderWithProviders } from './test-utils'; +const noop = () => {}; + +describe('node metric availability', () => { + it.each([false, true])( + 'does not present unavailable CPU as measured zero (mobile=%s)', + (isMobile) => { + const node = NodeRecordSchema.parse({ + id: 1, + name: 'edge', + status: 'online', + enable: true, + cpuPct: 0, + memPct: 44.4, + uptimeSecs: 0, + unavailableMetrics: ['cpu', 'uptime'], + }); + renderWithProviders( + , + ); + if (isMobile) fireEvent.click(screen.getByRole('button', { name: 'More Information' })); + expect(screen.queryByText('0.0%')).toBeNull(); + expect(screen.getAllByText('—').length).toBeGreaterThanOrEqual(2); + }, + ); + it('keeps a measured zero CPU and an unprobed uptime apart', () => { + const node = NodeRecordSchema.parse({ + id: 1, + name: 'edge', + status: 'online', + enable: true, + cpuPct: 0, + memPct: 44.4, + uptimeSecs: 0, + }); + renderWithProviders( + , + ); + expect(screen.getByText('0.0%')).not.toBeNull(); + const headers = screen.getAllByRole('columnheader').map((h) => h.textContent); + const cells = within(screen.getAllByRole('row')[1]).getAllByRole('cell'); + expect(cells[headers.indexOf('Uptime')].textContent).toBe('-'); + }); +}); diff --git a/internal/database/model/model.go b/internal/database/model/model.go index ccb1b8afd..b46d29f36 100644 --- a/internal/database/model/model.go +++ b/internal/database/model/model.go @@ -864,9 +864,11 @@ type Node struct { CpuPct float64 `json:"cpuPct" example:"23.5"` MemPct float64 `json:"memPct" example:"45.1"` UptimeSecs uint64 `json:"uptimeSecs" example:"86400"` - NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"` - NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"` - LastError string `json:"lastError"` + // Nil on legacy rows and responses: zero remains a valid measured value. + UnavailableMetrics []string `json:"unavailableMetrics,omitempty" gorm:"serializer:json;column:unavailable_metrics"` + NetUp uint64 `json:"netUp" gorm:"column:net_up" example:"1048576"` + NetDown uint64 `json:"netDown" gorm:"column:net_down" example:"2097152"` + LastError string `json:"lastError"` // XrayState and XrayError are captured from the remote node's /panel/api/server/status // during heartbeats. They let the central panel distinguish "panel API reachable" diff --git a/internal/util/sys/lxc_metrics.go b/internal/util/sys/lxc_metrics.go new file mode 100644 index 000000000..4560a3297 --- /dev/null +++ b/internal/util/sys/lxc_metrics.go @@ -0,0 +1,10 @@ +package sys + +type LXCStats struct { + Scope string + CPU float64 + CPUs int + SwapUsed, SwapTotal uint64 + Uptime uint64 + Unavailable []string +} diff --git a/internal/util/sys/lxc_metrics_linux.go b/internal/util/sys/lxc_metrics_linux.go new file mode 100644 index 000000000..d4e185553 --- /dev/null +++ b/internal/util/sys/lxc_metrics_linux.go @@ -0,0 +1,274 @@ +//go:build linux + +package sys + +import ( + "errors" + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type LXCReader struct { + ProcRoot, CgroupRoot string + mu sync.Mutex + lastCPU uint64 + lastAt time.Time + lastCapacity float64 + sampled bool +} + +func (r *LXCReader) roots() (string, string) { + proc := r.ProcRoot + if proc == "" { + proc = HostProc() + } + cg := r.CgroupRoot + if cg == "" { + cg = "/sys/fs/cgroup" + } + return proc, cg +} + +type lxcMounts struct { + sources map[string]bool + cgroupRoot bool +} + +func (r *LXCReader) detect() (lxcMounts, error) { + proc, cg := r.roots() + data, err := os.ReadFile(filepath.Join(proc, "self/mountinfo")) + if err != nil { + return lxcMounts{}, err + } + mounts := lxcMounts{sources: map[string]bool{}} + valid := false + for _, line := range strings.Split(string(data), "\n") { + parts := strings.SplitN(line, " - ", 2) + if len(parts) != 2 { + continue + } + mount, fs := strings.Fields(parts[0]), strings.Fields(parts[1]) + if len(mount) <= 4 || len(fs) == 0 { + continue + } + valid = true + if fs[0] == "cgroup2" && mount[3] == "/" && mount[4] == cg { + mounts.cgroupRoot = true + } + if fs[0] == "fuse.lxcfs" { + switch mount[4] { + case "/proc/meminfo", "/proc/cpuinfo", "/proc/uptime": + mounts.sources[mount[4]] = true + } + } + } + if !valid { + return lxcMounts{}, errors.New("invalid mountinfo") + } + return mounts, nil +} + +func (r *LXCReader) UseScopedHistory() bool { + mounts, err := r.detect() + return len(mounts.sources) > 0 || err != nil +} + +func (r *LXCReader) resetCPU() { + r.mu.Lock() + r.sampled = false + r.mu.Unlock() +} + +// LXCFS supplies container swap and uptime; sysinfo bypasses those mounts. +func (r *LXCReader) Read(now time.Time) (*LXCStats, error) { + mounts, err := r.detect() + if len(mounts.sources) == 0 && err == nil { + r.resetCPU() + return nil, nil + } + out := &LXCStats{Scope: "container"} + if err != nil { + r.resetCPU() + out.Scope = "unknown" + out.Unavailable = []string{"cpu", "swap", "uptime"} + return out, fmt.Errorf("cannot determine metric scope: %w", err) + } + proc, cg := r.roots() + if mounts.sources["/proc/meminfo"] { + mem, err := readLXCBytes(filepath.Join(proc, "meminfo")) + total, tok := mem["SwapTotal"] + free, fok := mem["SwapFree"] + if err == nil && tok && fok && free <= total { + out.SwapTotal = total + out.SwapUsed = total - free + } else { + out.Unavailable = append(out.Unavailable, "swap") + } + } else { + out.Unavailable = append(out.Unavailable, "swap") + } + if mounts.sources["/proc/uptime"] { + data, err := os.ReadFile(filepath.Join(proc, "uptime")) + fields := strings.Fields(string(data)) + if err == nil && len(fields) > 0 { + value, e := strconv.ParseFloat(fields[0], 64) + if e == nil && !math.IsNaN(value) && !math.IsInf(value, 0) && value >= 0 && value < float64(math.MaxUint64) { + out.Uptime = uint64(value) + } else { + err = errors.New("invalid uptime") + } + } else { + err = errors.New("missing uptime") + } + if err != nil { + out.Unavailable = append(out.Unavailable, "uptime") + } + } else { + out.Unavailable = append(out.Unavailable, "uptime") + } + if !mounts.sources["/proc/cpuinfo"] || !mounts.cgroupRoot || !containerInitAtRoot(proc) { + r.resetCPU() + out.Unavailable = append(out.Unavailable, "cpu") + return out, nil + } + capacity, cpus, err := r.cpuCapacity(proc, cg) + out.CPUs = cpus + usage, usageErr := readUsage(filepath.Join(cg, "cpu.stat")) + r.mu.Lock() + defer r.mu.Unlock() + if err != nil || usageErr != nil { + r.sampled = false + out.Unavailable = append(out.Unavailable, "cpu") + } else { + elapsed := now.Sub(r.lastAt).Seconds() + if r.sampled && elapsed > 0 && usage >= r.lastCPU && capacity == r.lastCapacity { + out.CPU = math.Min(100, float64(usage-r.lastCPU)/1e6/elapsed/capacity*100) + } else { + out.Unavailable = append(out.Unavailable, "cpu") + } + r.lastCPU, r.lastAt, r.lastCapacity, r.sampled = usage, now, capacity, true + } + return out, nil +} + +// PID 1 must belong to this namespace root, not a host-visible LXC subtree. +func containerInitAtRoot(proc string) bool { + data, err := os.ReadFile(filepath.Join(proc, "1/cgroup")) + if err != nil { + return false + } + switch strings.TrimSpace(string(data)) { + case "0::/", "0::/init.scope": + return true + default: + return false + } +} + +func readLXCBytes(path string) (map[string]uint64, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + wanted := map[string]bool{"SwapTotal": true, "SwapFree": true} + values := map[string]uint64{} + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + key := strings.TrimSuffix(fields[0], ":") + if !wanted[key] { + continue + } + if len(fields) != 3 || fields[2] != "kB" { + return nil, errors.New("invalid meminfo unit") + } + if _, exists := values[key]; exists { + return nil, errors.New("duplicate meminfo field") + } + value, e := strconv.ParseUint(fields[1], 10, 64) + if e != nil || value > math.MaxUint64/1024 { + return nil, errors.New("invalid meminfo value") + } + values[key] = value * 1024 + } + return values, nil +} + +func (r *LXCReader) cpuCapacity(proc, cg string) (float64, int, error) { + data, err := os.ReadFile(filepath.Join(proc, "cpuinfo")) + if err != nil { + return 0, 0, err + } + ids := map[uint64]bool{} + for _, line := range strings.Split(string(data), "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok || strings.TrimSpace(key) != "processor" { + continue + } + id, e := strconv.ParseUint(strings.TrimSpace(value), 10, 64) + if e != nil || ids[id] { + return 0, 0, errors.New("invalid CPU list") + } + ids[id] = true + } + cpus := len(ids) + if cpus == 0 { + return 0, 0, errors.New("empty CPU list") + } + capacity := float64(cpus) + data, err = os.ReadFile(filepath.Join(cg, "cpu.max")) + if err != nil { + return 0, cpus, err + } + fields := strings.Fields(string(data)) + if len(fields) != 2 { + return 0, cpus, errors.New("invalid CPU quota") + } + period, e := strconv.ParseUint(fields[1], 10, 64) + if e != nil || period == 0 { + return 0, cpus, errors.New("invalid CPU period") + } + if fields[0] != "max" { + quota, e := strconv.ParseUint(fields[0], 10, 64) + if e != nil || quota == 0 { + return 0, cpus, errors.New("invalid CPU quota") + } + capacity = math.Min(capacity, float64(quota)/float64(period)) + } + return capacity, cpus, nil +} + +func readUsage(path string) (uint64, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + var usage uint64 + found := false + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != "usage_usec" { + continue + } + if len(fields) != 2 || found { + return 0, errors.New("invalid cgroup CPU usage") + } + usage, err = strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, err + } + found = true + } + if !found { + return 0, errors.New("missing cgroup CPU usage") + } + return usage, nil +} diff --git a/internal/util/sys/lxc_metrics_linux_test.go b/internal/util/sys/lxc_metrics_linux_test.go new file mode 100644 index 000000000..1c526b98b --- /dev/null +++ b/internal/util/sys/lxc_metrics_linux_test.go @@ -0,0 +1,274 @@ +//go:build linux + +package sys + +import ( + "math" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const lxcMountFixture = "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n" + +func lxcFixture(t *testing.T) *LXCReader { + t.Helper() + root := t.TempDir() + r := &LXCReader{ProcRoot: filepath.Join(root, "proc"), CgroupRoot: filepath.Join(root, "cgroup")} + fixtures := map[string]string{ + filepath.Join(r.ProcRoot, "self/mountinfo"): lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n", + filepath.Join(r.ProcRoot, "1/cgroup"): "0::/init.scope\n", + filepath.Join(r.ProcRoot, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n", + filepath.Join(r.ProcRoot, "cpuinfo"): "processor : 0\ncpu cores : 16\n\nprocessor : 1\ncpu cores : 16\n", + filepath.Join(r.ProcRoot, "uptime"): "52624.93 52624.93\n", + filepath.Join(r.CgroupRoot, "cpu.stat"): "usage_usec 222772076\nuser_usec 101515374\n", + filepath.Join(r.CgroupRoot, "cpu.max"): "max 100000\n", + } + for path, content := range fixtures { + fixtureWrite(t, path, content) + } + return r +} + +func fixtureWrite(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestLXCResourcesIgnoreHostTopologyAndInheritedMax(t *testing.T) { + r := lxcFixture(t) + got, err := r.Read(time.Unix(100, 0)) + if err != nil || got == nil { + t.Fatalf("sample=%+v err=%v", got, err) + } + if got.CPUs != 2 || got.SwapTotal != 512<<20 || got.SwapUsed != 19357696 || got.Uptime != 52624 { + t.Fatalf("wrong container resources: %+v", got) + } + if !slices.Equal(got.Unavailable, []string{"cpu"}) { + t.Fatalf("first sample=%+v", got) + } +} + +func TestLXCCPUUsesWholeContainerDeltaAndCapacity(t *testing.T) { + for _, tc := range []struct { + name, quota string + delta uint64 + want float64 + }{ + {"one CPU of two", "max 100000\n", 1000000, 50}, + {"idle is a valid zero", "max 100000\n", 0, 0}, + {"half CPU quota", "50000 100000\n", 125000, 25}, + {"quota above visible CPUs", "800000 100000\n", 1000000, 50}, + {"saturated interval bounded", "max 100000\n", 2100000, 100}, + } { + t.Run(tc.name, func(t *testing.T) { + r := lxcFixture(t) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), tc.quota) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1000000\n") + r.Read(time.Unix(100, 0)) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(1000000+tc.delta, 10)+"\n") + got, err := r.Read(time.Unix(101, 0)) + if err != nil || got.CPU != tc.want || slices.Contains(got.Unavailable, "cpu") { + t.Fatalf("sample=%+v err=%v want=%v", got, err, tc.want) + } + }) + } +} + +func TestLXCCounterClockAndCapacityChangesResetBaseline(t *testing.T) { + for _, scenario := range []string{"counter reset", "zero interval", "backward interval", "capacity change", "missing counter", "scope change"} { + t.Run(scenario, func(t *testing.T) { + r := lxcFixture(t) + r.Read(time.Unix(100, 0)) + at := time.Unix(101, 0) + switch scenario { + case "counter reset": + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 1\n") + case "zero interval": + at = time.Unix(100, 0) + case "backward interval": + at = time.Unix(99, 0) + case "capacity change": + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.max"), "100000 100000\n") + case "missing counter": + os.Remove(filepath.Join(r.CgroupRoot, "cpu.stat")) + r.Read(at) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 222772076\n") + case "scope change": + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n") + r.Read(at) + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), lxcMountFixture+"4 0 0:28 / "+r.CgroupRoot+" rw - cgroup2 cgroup rw\n") + } + got, err := r.Read(at) + if err != nil || !slices.Contains(got.Unavailable, "cpu") { + t.Fatalf("invalid interval=%+v err=%v", got, err) + } + usage, _ := readUsage(filepath.Join(r.CgroupRoot, "cpu.stat")) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec "+strconv.FormatUint(usage+500000, 10)+"\n") + got, err = r.Read(at.Add(time.Second)) + want := 25.0 + if scenario == "capacity change" { + want = 50 + } + if err != nil || got.CPU != want || slices.Contains(got.Unavailable, "cpu") { + t.Fatalf("recovery=%+v err=%v", got, err) + } + }) + } +} + +func TestLXCInvalidMetricsNeverUseHostFallback(t *testing.T) { + for _, tc := range []struct{ path, content, metric string }{ + {"meminfo", "SwapTotal: 1 kB\nSwapFree: 2 kB\n", "swap"}, + {"meminfo", "SwapTotal: 18446744073709551615 kB\nSwapFree: 0 kB\n", "swap"}, + {"meminfo", "SwapTotal: 1 MB\nSwapFree: 0 kB\n", "swap"}, + {"meminfo", "SwapTotal: 1 kB\nSwapTotal: 1 kB\nSwapFree: 0 kB\n", "swap"}, + {"meminfo", "SwapTotal: 1 kB\n", "swap"}, + {"uptime", "NaN 0\n", "uptime"}, + {"uptime", "+Inf 0\n", "uptime"}, + {"uptime", "-1 0\n", "uptime"}, + {"uptime", "18446744073709551616 0\n", "uptime"}, + {"uptime", "", "uptime"}, + {"cpuinfo", "cpu cores : 16\n", "cpu"}, + {"cpuinfo", "processor : 0\nprocessor : 0\n", "cpu"}, + } { + t.Run(tc.metric+"/"+tc.content, func(t *testing.T) { + r := lxcFixture(t) + fixtureWrite(t, filepath.Join(r.ProcRoot, tc.path), tc.content) + got, err := r.Read(time.Unix(100, 0)) + if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) { + t.Fatalf("invalid sample=%+v err=%v", got, err) + } + for _, v := range got.Unavailable { + if v != "cpu" && v != "swap" && v != "uptime" { + t.Fatalf("unrelated metric changed: %s", v) + } + } + }) + } +} + +func TestLXCCPURejectsMalformedCgroupData(t *testing.T) { + for _, tc := range []struct{ path, value string }{ + {"cpu.max", "max 0"}, + {"cpu.max", "0 100000"}, + {"cpu.max", "NaN 100000"}, + {"cpu.max", "max"}, + {"cpu.stat", "user_usec 123"}, + {"cpu.stat", "usage_usec -1"}, + {"cpu.stat", "usage_usec 1\nusage_usec 2"}, + } { + t.Run(tc.path+"/"+tc.value, func(t *testing.T) { + r := lxcFixture(t) + r.Read(time.Unix(100, 0)) + fixtureWrite(t, filepath.Join(r.CgroupRoot, tc.path), tc.value) + got, err := r.Read(time.Unix(101, 0)) + if err != nil || !slices.Contains(got.Unavailable, "cpu") || math.IsNaN(got.CPU) || math.IsInf(got.CPU, 0) { + t.Fatalf("bad CPU=%+v err=%v", got, err) + } + }) + } +} + +func TestLXCZeroSwapIsValidAndRAMIsNotParsed(t *testing.T) { + r := lxcFixture(t) + fixtureWrite(t, filepath.Join(r.ProcRoot, "meminfo"), "MemTotal: not-a-number kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n") + got, err := r.Read(time.Unix(100, 0)) + if err != nil || got.SwapTotal != 0 || got.SwapUsed != 0 || slices.Contains(got.Unavailable, "swap") { + t.Fatalf("disabled swap=%+v err=%v", got, err) + } +} + +func TestLXCEachProcSourceRequiresItsOwnMount(t *testing.T) { + for _, tc := range []struct{ path, metric string }{{"meminfo", "swap"}, {"cpuinfo", "cpu"}, {"uptime", "uptime"}} { + t.Run(tc.path, func(t *testing.T) { + r := lxcFixture(t) + var lines []string + for _, line := range strings.Split(lxcMountFixture, "\n") { + if !strings.Contains(line, "/proc/"+tc.path) { + lines = append(lines, line) + } + } + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), strings.Join(lines, "\n")) + got, err := r.Read(time.Unix(100, 0)) + if err != nil || got == nil || !slices.Contains(got.Unavailable, tc.metric) { + t.Fatalf("unverified mount=%+v err=%v", got, err) + } + }) + } +} + +func TestNativeMetricsKeepOriginalPath(t *testing.T) { + r := lxcFixture(t) + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "1 0 0:1 / / rw - ext4 /dev/root rw\n") + os.RemoveAll(r.CgroupRoot) + got, err := r.Read(time.Unix(100, 0)) + if err != nil || got != nil || r.UseScopedHistory() { + t.Fatalf("native=%+v err=%v", got, err) + } +} + +func TestUnreadableScopeNeverSelectsNativeHostMetrics(t *testing.T) { + for _, bad := range []string{"missing", "malformed"} { + t.Run(bad, func(t *testing.T) { + r := lxcFixture(t) + if bad == "missing" { + os.Remove(filepath.Join(r.ProcRoot, "self/mountinfo")) + } else { + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), "invalid") + } + got, err := r.Read(time.Unix(100, 0)) + if err == nil || got == nil || got.Scope != "unknown" || !slices.Equal(got.Unavailable, []string{"cpu", "swap", "uptime"}) || !r.UseScopedHistory() { + t.Fatalf("unknown=%+v err=%v", got, err) + } + }) + } +} + +func TestLXCCPURejectsUnverifiedCgroupRoot(t *testing.T) { + for _, scenario := range []string{"missing mount", "subtree mount", "legacy controller", "host namespace", "missing init membership"} { + t.Run(scenario, func(t *testing.T) { + r := lxcFixture(t) + r.Read(time.Unix(100, 0)) + mounts := lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n" + switch scenario { + case "missing mount": + mounts = lxcMountFixture + case "subtree mount": + mounts = lxcMountFixture + "4 0 0:28 /system.slice " + r.CgroupRoot + " rw - cgroup2 cgroup rw\n" + case "legacy controller": + mounts = lxcMountFixture + "4 0 0:28 / " + r.CgroupRoot + " rw - cgroup cgroup rw\n" + case "host namespace": + fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/lxc/example/init.scope\n") + case "missing init membership": + os.Remove(filepath.Join(r.ProcRoot, "1/cgroup")) + } + fixtureWrite(t, filepath.Join(r.ProcRoot, "self/mountinfo"), mounts) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n") + got, err := r.Read(time.Unix(101, 0)) + if err != nil || got == nil || !slices.Contains(got.Unavailable, "cpu") || got.CPU != 0 || got.SwapTotal != 512<<20 || got.Uptime != 52624 { + t.Fatalf("unverified cgroup leaked CPU or changed proc metrics: %+v err=%v", got, err) + } + }) + } +} + +func TestLXCCPUAcceptsContainerInitAtNamespaceRoot(t *testing.T) { + r := lxcFixture(t) + fixtureWrite(t, filepath.Join(r.ProcRoot, "1/cgroup"), "0::/\n") + r.Read(time.Unix(100, 0)) + fixtureWrite(t, filepath.Join(r.CgroupRoot, "cpu.stat"), "usage_usec 223772076\n") + got, err := r.Read(time.Unix(101, 0)) + if err != nil || got.CPU != 50 || slices.Contains(got.Unavailable, "cpu") { + t.Fatalf("root init sample=%+v err=%v", got, err) + } +} diff --git a/internal/util/sys/lxc_metrics_other.go b/internal/util/sys/lxc_metrics_other.go new file mode 100644 index 000000000..3acedb66d --- /dev/null +++ b/internal/util/sys/lxc_metrics_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package sys + +import "time" + +type LXCReader struct{} + +func (r *LXCReader) Read(now time.Time) (*LXCStats, error) { return nil, nil } +func (r *LXCReader) UseScopedHistory() bool { return false } diff --git a/internal/web/service/discord/gateway.go b/internal/web/service/discord/gateway.go index 98b540681..6199ab15f 100644 --- a/internal/web/service/discord/gateway.go +++ b/internal/web/service/discord/gateway.go @@ -487,9 +487,6 @@ func (g *GatewayClient) sendStatus(ctx context.Context) { hostname = "3x-ui" } - days := status.Uptime / 86400 - hours := (status.Uptime % 86400) / 3600 - var onlines []string if process := service.XrayProcess(); process != nil && process.IsRunning() { onlines = process.GetOnlineClients() @@ -515,7 +512,7 @@ func (g *GatewayClient) sendStatus(ctx context.Context) { Fields: []EmbedField{ {Name: tr("discord.fields.panelVersion"), Value: config.GetPanelVersion(), Inline: true}, {Name: tr("discord.fields.xrayCore"), Value: fmt.Sprintf("%s (%s)", status.Xray.Version, status.Xray.State), Inline: true}, - {Name: tr("pages.index.uptime"), Value: tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours)), Inline: true}, + {Name: tr("pages.index.uptime"), Value: formatStatusUptime(status, tr), Inline: true}, {Name: tr("discord.fields.systemLoad"), Value: fmt.Sprintf("%.2f, %.2f, %.2f", load1, load2, load3), Inline: true}, {Name: tr("pages.index.memory"), Value: fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total))), Inline: true}, {Name: tr("pages.index.historyTitleOnline"), Value: strconv.Itoa(len(onlines)), Inline: true}, diff --git a/internal/web/service/discord/report.go b/internal/web/service/discord/report.go index 012a1da9f..6d6013f65 100644 --- a/internal/web/service/discord/report.go +++ b/internal/web/service/discord/report.go @@ -102,9 +102,7 @@ func (s *DiscordService) BuildReport(ctx context.Context, server ServerProvider, } tr := translator(s.settingService) - days := status.Uptime / 86400 - hours := (status.Uptime % 86400) / 3600 - uptimeStr := tr("discord.values.uptime", "Days=="+fmt.Sprint(days), "Hours=="+fmt.Sprint(hours)) + uptimeStr := formatStatusUptime(status, tr) ramStr := fmt.Sprintf("%s / %s", common.FormatTraffic(int64(status.Mem.Current)), common.FormatTraffic(int64(status.Mem.Total))) trafficStr := tr("discord.values.traffic", @@ -239,3 +237,11 @@ func getInterfaceIPs() (ipv4, ipv6 string) { } return strings.Join(v4s, ", "), strings.Join(v6s, ", ") } + +// formatStatusUptime is shared by scheduled reports and the status command. +func formatStatusUptime(status *service.Status, tr func(string, ...string) string) string { + if !status.MetricAvailable("uptime") { + return "—" + } + return tr("discord.values.uptime", "Days=="+fmt.Sprint(status.Uptime/86400), "Hours=="+fmt.Sprint((status.Uptime%86400)/3600)) +} diff --git a/internal/web/service/discord/report_test.go b/internal/web/service/discord/report_test.go index ce75d140d..94270f78f 100644 --- a/internal/web/service/discord/report_test.go +++ b/internal/web/service/discord/report_test.go @@ -229,3 +229,61 @@ func TestSendReport_DeliversEmbedWhenBackupUploadIsRejected(t *testing.T) { t.Fatal("report embed was never delivered: it rode on the rejected backup upload") } } + +func TestBuildReportShowsUnavailableUptime(t *testing.T) { + for _, unavailable := range []bool{true, false} { + t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) { + settings := setupTestDB(t) + _ = settings.SetDiscordLang("en-US") + _ = settings.SetDiscordBotBackup(false) + status := &service.Status{Loads: []float64{0, 0, 0}} + if unavailable { + status.UnavailableMetrics = []string{"uptime"} + } + server := &mockServerProvider{status: status} + svc := NewDiscordService(settings) + report, _, err := svc.BuildReport(context.Background(), server, nil) + if err != nil { + t.Fatal(err) + } + want := "0d 0h" + if unavailable { + want = "—" + } + check := func(fields []EmbedField) { + t.Helper() + for _, f := range fields { + if f.Name == "Uptime" { + if f.Value != want { + t.Fatalf("uptime=%q want %q", f.Value, want) + } + return + } + } + t.Fatal("uptime field missing") + } + check(report.Embeds[0].Fields) + _ = settings.SetDiscordBotToken("synthetic-test") + _ = settings.SetDiscordChannelId("synthetic-channel") + sent := make(chan MessagePayload, 1) + rest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload MessagePayload + _ = json.NewDecoder(r.Body).Decode(&payload) + sent <- payload + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"synthetic-message"}`)) + })) + defer rest.Close() + svc.SetBaseURL(rest.URL) + svc.SetHTTPClient(rest.Client()) + g := NewGatewayClient(svc, settings, server, nil, nil) + g.sendStatus(context.Background()) + select { + case msg := <-sent: + check(msg.Embeds[0].Fields) + default: + t.Fatal("no local status payload") + } + }) + } +} diff --git a/internal/web/service/node.go b/internal/web/service/node.go index 21cd49a94..0b6442097 100644 --- a/internal/web/service/node.go +++ b/internal/web/service/node.go @@ -32,15 +32,16 @@ import ( ) type HeartbeatPatch struct { - Status string - LastHeartbeat int64 - LatencyMs int - XrayVersion string - PanelVersion string - Guid string - CpuPct float64 - MemPct float64 - UptimeSecs uint64 + Status string + LastHeartbeat int64 + LatencyMs int + XrayVersion string + PanelVersion string + Guid string + CpuPct float64 + MemPct float64 + UptimeSecs uint64 + UnavailableMetrics []string // NetUp/NetDown are the node's current interface throughput (bytes/sec), // summed over non-virtual interfaces, read from its status response. NetUp uint64 @@ -978,20 +979,27 @@ func (s *NodeService) UpdatePanels(ids []int, dev bool) ([]NodeUpdateResult, err func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error { db := database.GetDB() + // Map updates bypass GORM field serializers. Encode the availability list + // explicitly so it survives list requests and panel restarts. + unavailable, err := json.Marshal(p.UnavailableMetrics) + if err != nil { + return err + } updates := map[string]any{ - "status": p.Status, - "last_heartbeat": p.LastHeartbeat, - "latency_ms": p.LatencyMs, - "xray_version": p.XrayVersion, - "panel_version": p.PanelVersion, - "cpu_pct": p.CpuPct, - "mem_pct": p.MemPct, - "uptime_secs": p.UptimeSecs, - "net_up": p.NetUp, - "net_down": p.NetDown, - "last_error": p.LastError, - "xray_state": p.XrayState, - "xray_error": p.XrayError, + "unavailable_metrics": string(unavailable), + "status": p.Status, + "last_heartbeat": p.LastHeartbeat, + "latency_ms": p.LatencyMs, + "xray_version": p.XrayVersion, + "panel_version": p.PanelVersion, + "cpu_pct": p.CpuPct, + "mem_pct": p.MemPct, + "uptime_secs": p.UptimeSecs, + "net_up": p.NetUp, + "net_down": p.NetDown, + "last_error": p.LastError, + "xray_state": p.XrayState, + "xray_error": p.XrayError, } // Only learn the GUID; never clear a known one if an old-build node (or a // failed probe) reports none, so the stable identity survives blips. @@ -1004,7 +1012,9 @@ func (s *NodeService) UpdateHeartbeat(id int, p HeartbeatPatch) error { } if p.Status == "online" { now := time.Unix(p.LastHeartbeat, 0) - nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct) + if !slices.Contains(p.UnavailableMetrics, "cpu") { + nodeMetrics.append(nodeMetricKey(id, "cpu"), now, p.CpuPct) + } nodeMetrics.append(nodeMetricKey(id, "mem"), now, p.MemPct) nodeMetrics.append(nodeMetricKey(id, "netUp"), now, float64(p.NetUp)) nodeMetrics.append(nodeMetricKey(id, "netDown"), now, float64(p.NetDown)) @@ -1335,8 +1345,9 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string) Success bool `json:"success"` Msg string `json:"msg"` Obj *struct { - CpuPct float64 `json:"cpu"` - Mem struct { + CpuPct float64 `json:"cpu"` + UnavailableMetrics []string `json:"unavailableMetrics"` + Mem struct { Current uint64 `json:"current"` Total uint64 `json:"total"` } `json:"mem"` @@ -1370,6 +1381,7 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string) } o := envelope.Obj patch.CpuPct = o.CpuPct + patch.UnavailableMetrics = o.UnavailableMetrics if o.Mem.Total > 0 { patch.MemPct = float64(o.Mem.Current) * 100.0 / float64(o.Mem.Total) } @@ -1385,14 +1397,15 @@ func (s *NodeService) probe(ctx context.Context, n *model.Node, proxyURL string) } type ProbeResultUI struct { - Status string `json:"status" example:"online"` - LatencyMs int `json:"latencyMs" example:"42"` - XrayVersion string `json:"xrayVersion" example:"25.10.31"` - PanelVersion string `json:"panelVersion" example:"v3.x.x"` - CpuPct float64 `json:"cpuPct" example:"12.5"` - MemPct float64 `json:"memPct" example:"45.2"` - UptimeSecs uint64 `json:"uptimeSecs" example:"86400"` - Error string `json:"error"` + Status string `json:"status" example:"online"` + LatencyMs int `json:"latencyMs" example:"42"` + XrayVersion string `json:"xrayVersion" example:"25.10.31"` + PanelVersion string `json:"panelVersion" example:"v3.x.x"` + CpuPct float64 `json:"cpuPct" example:"12.5"` + MemPct float64 `json:"memPct" example:"45.2"` + UptimeSecs uint64 `json:"uptimeSecs" example:"86400"` + UnavailableMetrics []string `json:"unavailableMetrics,omitempty"` + Error string `json:"error"` // XrayState/XrayError are populated on successful probes even when the node's // Xray core is not healthy. The UI uses them for a distinct "panel ok, xray failed" indicator. XrayState string `json:"xrayState"` @@ -1401,15 +1414,16 @@ type ProbeResultUI struct { func (p HeartbeatPatch) ToUI(ok bool) ProbeResultUI { r := ProbeResultUI{ - LatencyMs: p.LatencyMs, - XrayVersion: p.XrayVersion, - PanelVersion: p.PanelVersion, - CpuPct: p.CpuPct, - MemPct: p.MemPct, - UptimeSecs: p.UptimeSecs, - Error: FriendlyProbeError(p.LastError), - XrayState: p.XrayState, - XrayError: p.XrayError, + LatencyMs: p.LatencyMs, + XrayVersion: p.XrayVersion, + PanelVersion: p.PanelVersion, + CpuPct: p.CpuPct, + MemPct: p.MemPct, + UptimeSecs: p.UptimeSecs, + UnavailableMetrics: p.UnavailableMetrics, + Error: FriendlyProbeError(p.LastError), + XrayState: p.XrayState, + XrayError: p.XrayError, } if ok { r.Status = "online" diff --git a/internal/web/service/node_contract.go b/internal/web/service/node_contract.go index b21e615be..782808c9b 100644 --- a/internal/web/service/node_contract.go +++ b/internal/web/service/node_contract.go @@ -34,6 +34,7 @@ type NodeView struct { CpuPct float64 `json:"cpuPct" example:"12.5"` MemPct float64 `json:"memPct" example:"45.2"` UptimeSecs uint64 `json:"uptimeSecs" example:"86400"` + UnavailableMetrics []string `json:"unavailableMetrics,omitempty"` NetUp uint64 `json:"netUp" example:"2097152"` NetDown uint64 `json:"netDown" example:"1048576"` LastError string `json:"lastError" example:""` @@ -82,6 +83,7 @@ func toNodeView(n *model.Node) *NodeView { CpuPct: n.CpuPct, MemPct: n.MemPct, UptimeSecs: n.UptimeSecs, + UnavailableMetrics: n.UnavailableMetrics, NetUp: n.NetUp, NetDown: n.NetDown, LastError: n.LastError, diff --git a/internal/web/service/node_netmetrics_test.go b/internal/web/service/node_netmetrics_test.go index 60b4ef695..b2b172c86 100644 --- a/internal/web/service/node_netmetrics_test.go +++ b/internal/web/service/node_netmetrics_test.go @@ -2,15 +2,20 @@ package service import ( "context" + "encoding/json" "net/http" "net/http/httptest" "net/url" + "path/filepath" + "reflect" "slices" "strconv" + "sync/atomic" "testing" "time" "github.com/mhsanaei/3x-ui/v3/internal/database" + "github.com/mhsanaei/3x-ui/v3/internal/database/dbtest" "github.com/mhsanaei/3x-ui/v3/internal/database/model" ) @@ -70,3 +75,131 @@ func TestNodeMetricKeysIncludesNet(t *testing.T) { } } } + +func availabilityNodeFixture(t *testing.T, body string) (*NodeService, *model.Node, func(string)) { + t.Helper() + dbtest.InitDB(t, filepath.Join(t.TempDir(), "node.db")) + var response atomic.Value + response.Store(body) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(response.Load().(string))) + })) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(u.Port()) + if err != nil { + t.Fatal(err) + } + n := &model.Node{Name: "sample", Scheme: "http", Address: u.Hostname(), Port: port, BasePath: "/", Enable: true, AllowPrivateAddress: true, TlsVerifyMode: "skip"} + if err := database.GetDB().Create(n).Error; err != nil { + t.Fatal(err) + } + old := nodeMetrics + nodeMetrics = newMetricHistory() + t.Cleanup(func() { nodeMetrics = old }) + return &NodeService{}, n, func(s string) { response.Store(s) } +} + +func unavailableMetricsOf(t *testing.T, value any) []string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + var out struct { + Unavailable []string `json:"unavailableMetrics"` + } + if err := json.Unmarshal(data, &out); err != nil { + t.Fatal(err) + } + return out.Unavailable +} + +const ( + unavailableStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"unavailableMetrics":["cpu","uptime"],"mem":{"current":1,"total":2},"xray":{"state":"running"}}}` + zeroStatusBody = `{"success":true,"obj":{"cpu":0,"uptime":0,"mem":{"current":1,"total":2},"xray":{"state":"running"}}}` +) + +func TestProbeKeepsMetricAvailability(t *testing.T) { + svc, n, _ := availabilityNodeFixture(t, unavailableStatusBody) + patch, err := svc.Probe(context.Background(), n) + if err != nil { + t.Fatal(err) + } + if got := unavailableMetricsOf(t, patch.ToUI(true)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) { + t.Fatalf("probe availability=%v, want cpu+uptime unavailable", got) + } +} + +func TestUpdateHeartbeatKeepsUnavailableCPUOutOfHistory(t *testing.T) { + svc, n, setBody := availabilityNodeFixture(t, unavailableStatusBody) + patch, err := svc.Probe(context.Background(), n) + if err != nil { + t.Fatal(err) + } + patch.Status = "online" + if err := svc.UpdateHeartbeat(n.Id, patch); err != nil { + t.Fatal(err) + } + if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 0 { + t.Fatalf("unavailable CPU became history: %v", points) + } + var stored model.Node + if err := database.GetDB().First(&stored, n.Id).Error; err != nil { + t.Fatal(err) + } + if got := unavailableMetricsOf(t, toNodeView(&stored)); !reflect.DeepEqual(got, []string{"cpu", "uptime"}) { + t.Fatalf("stored availability=%v", got) + } + if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "mem"), 2, 60); len(points) != 1 || points[0]["v"] != float64(50) { + t.Fatalf("memory history changed: %v", points) + } + setBody(zeroStatusBody) + patch, err = svc.Probe(context.Background(), n) + if err != nil { + t.Fatal(err) + } + patch.Status = "online" + if err := svc.UpdateHeartbeat(n.Id, patch); err != nil { + t.Fatal(err) + } + if err := database.GetDB().First(&stored, n.Id).Error; err != nil { + t.Fatal(err) + } + if got := unavailableMetricsOf(t, toNodeView(&stored)); len(got) != 0 { + t.Fatalf("valid zero did not recover: %v", got) + } + if points := nodeMetrics.aggregate(nodeMetricKey(n.Id, "cpu"), 2, 60); len(points) != 1 || points[0]["v"] != float64(0) { + t.Fatalf("valid zero lost: %v", points) + } +} + +func TestNodeAvailabilityColumnKeepsLegacyRows(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + dbtest.InitDB(t, path) + n := &model.Node{Name: "legacy", CpuPct: 0, UptimeSecs: 0} + if err := database.GetDB().Create(n).Error; err != nil { + t.Fatal(err) + } + // Recreate the pre-change schema, then let the normal startup migration run. + if err := database.GetDB().Migrator().DropColumn(&model.Node{}, "UnavailableMetrics"); err != nil { + t.Fatal(err) + } + if err := database.CloseDB(); err != nil { + t.Fatal(err) + } + if err := database.InitDB(path); err != nil { + t.Fatal(err) + } + var stored model.Node + if err := database.GetDB().First(&stored, n.Id).Error; err != nil { + t.Fatal(err) + } + if stored.Name != "legacy" || stored.CpuPct != 0 || stored.UptimeSecs != 0 || len(stored.UnavailableMetrics) != 0 { + t.Fatalf("legacy row changed: id=%d availability=%v", stored.Id, stored.UnavailableMetrics) + } +} diff --git a/internal/web/service/server.go b/internal/web/service/server.go index caf106d50..eba80cefc 100644 --- a/internal/web/service/server.go +++ b/internal/web/service/server.go @@ -66,6 +66,9 @@ const ( // Status represents comprehensive system and application status information. // It includes CPU, memory, disk, network statistics, and Xray process status. type Status struct { + ResourceScope string `json:"resourceScope,omitempty"` + UnavailableMetrics []string `json:"unavailableMetrics,omitempty"` + T time.Time `json:"-"` Cpu float64 `json:"cpu"` CpuCores int `json:"cpuCores"` @@ -143,6 +146,8 @@ type Release struct { // ServerService provides business logic for server monitoring and management. // It handles system status collection, IP detection, and application statistics. type ServerService struct { + lxcMetrics sys.LXCReader + xrayService XrayService inboundService InboundService settingService SettingService @@ -355,7 +360,7 @@ func (s *ServerService) GetDefaultLogOutboundTags() (freedoms, blackholes []stri // Kept for back-compat with the original /panel/api/server/cpuHistory/:bucket route; // the response key is "cpu" (not "v") so legacy consumers parse unchanged. func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) []map[string]any { - out := systemMetrics.aggregate("cpu", bucketSeconds, maxPoints) + out := systemMetrics.aggregate(s.systemMetricKey("cpu"), bucketSeconds, maxPoints) for _, p := range out { p["cpu"] = p["v"] delete(p, "v") @@ -367,7 +372,7 @@ func (s *ServerService) AggregateCpuHistory(bucketSeconds int, maxPoints int) [] // known system metric (see SystemMetricKeys). Output points have keys // {"t": unixSec, "v": value}; the caller decides how to format the value. func (s *ServerService) AggregateSystemMetric(metric string, bucketSeconds int, maxPoints int) []map[string]any { - return systemMetrics.aggregate(metric, bucketSeconds, maxPoints) + return systemMetrics.aggregate(s.systemMetricKey(metric), bucketSeconds, maxPoints) } type LogEntry struct { @@ -526,53 +531,61 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status { T: now, } - // CPU stats - util, err := s.sampleCPUUtilization() - if err != nil { - logger.Warning("get cpu percent failed:", err) + scoped, scopeErr := s.lxcMetrics.Read(now) + if scopeErr != nil { + logger.Warning("get scoped metrics failed:", scopeErr) + } + if scoped != nil { + applyLXCStats(status, scoped) } else { - status.Cpu = util - } - - status.CpuCores, err = cpu.Counts(false) - if err != nil { - logger.Warning("get cpu cores count failed:", err) - } - - status.LogicalPro = runtime.NumCPU() - - if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute { - s.lastCpuInfoAttempt = time.Now() - done := make(chan struct{}) - go func() { - defer close(done) - cpuInfos, err := cpu.Info() - if err != nil { - logger.Warning("get cpu info failed:", err) - return - } - if len(cpuInfos) > 0 { - s.cachedCpuSpeedMhz = cpuInfos[0].Mhz - status.CpuSpeedMhz = s.cachedCpuSpeedMhz - } else { - logger.Warning("could not find cpu info") - } - }() - select { - case <-done: - case <-time.After(1500 * time.Millisecond): - logger.Warning("cpu info query timed out; will retry later") + // CPU stats + util, err := s.sampleCPUUtilization() + if err != nil { + logger.Warning("get cpu percent failed:", err) + } else { + status.Cpu = util } - } else if s.cachedCpuSpeedMhz != 0 { - status.CpuSpeedMhz = s.cachedCpuSpeedMhz - } - // Uptime - upTime, err := host.Uptime() - if err != nil { - logger.Warning("get uptime failed:", err) - } else { - status.Uptime = upTime + status.CpuCores, err = cpu.Counts(false) + if err != nil { + logger.Warning("get cpu cores count failed:", err) + } + + status.LogicalPro = runtime.NumCPU() + + if status.CpuSpeedMhz = s.cachedCpuSpeedMhz; s.cachedCpuSpeedMhz == 0 && time.Since(s.lastCpuInfoAttempt) > 5*time.Minute { + s.lastCpuInfoAttempt = time.Now() + done := make(chan struct{}) + go func() { + defer close(done) + cpuInfos, err := cpu.Info() + if err != nil { + logger.Warning("get cpu info failed:", err) + return + } + if len(cpuInfos) > 0 { + s.cachedCpuSpeedMhz = cpuInfos[0].Mhz + status.CpuSpeedMhz = s.cachedCpuSpeedMhz + } else { + logger.Warning("could not find cpu info") + } + }() + select { + case <-done: + case <-time.After(1500 * time.Millisecond): + logger.Warning("cpu info query timed out; will retry later") + } + } else if s.cachedCpuSpeedMhz != 0 { + status.CpuSpeedMhz = s.cachedCpuSpeedMhz + } + + // Uptime + upTime, err := host.Uptime() + if err != nil { + logger.Warning("get uptime failed:", err) + } else { + status.Uptime = upTime + } } // Memory stats @@ -584,12 +597,14 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status { status.Mem.Total = memInfo.Total } - swapInfo, err := mem.SwapMemory() - if err != nil { - logger.Warning("get swap memory failed:", err) - } else { - status.Swap.Current = swapInfo.Used - status.Swap.Total = swapInfo.Total + if scoped == nil { + swapInfo, err := mem.SwapMemory() + if err != nil { + logger.Warning("get swap memory failed:", err) + } else { + status.Swap.Current = swapInfo.Used + status.Swap.Total = swapInfo.Total + } } // Disk stats @@ -736,7 +751,7 @@ func (s *ServerService) GetStatus(lastStatus *Status) *Status { // AppendCpuSample is preserved for callers that only have the CPU number. // New callers should prefer AppendStatusSample which writes the full set. func (s *ServerService) AppendCpuSample(t time.Time, v float64) { - systemMetrics.append("cpu", t, v) + systemMetrics.append(s.systemMetricKey("cpu"), t, v) } // AppendStatusSample writes one tick of every metric we keep — CPU, memory @@ -747,14 +762,14 @@ func (s *ServerService) AppendStatusSample(t time.Time, status *Status) { if status == nil { return } - systemMetrics.append("cpu", t, status.Cpu) + appendScopedMetric(status, "cpu", t, status.Cpu) if status.Mem.Total > 0 { systemMetrics.append("mem", t, float64(status.Mem.Current)*100.0/float64(status.Mem.Total)) } if status.Swap.Total > 0 { - systemMetrics.append("swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total)) + appendScopedMetric(status, "swap", t, float64(status.Swap.Current)*100.0/float64(status.Swap.Total)) } else { - systemMetrics.append("swap", t, 0) + appendScopedMetric(status, "swap", t, 0) } systemMetrics.append("netUp", t, float64(status.NetIO.Up)) systemMetrics.append("netDown", t, float64(status.NetIO.Down)) diff --git a/internal/web/service/server_scope.go b/internal/web/service/server_scope.go new file mode 100644 index 000000000..28453150c --- /dev/null +++ b/internal/web/service/server_scope.go @@ -0,0 +1,58 @@ +package service + +import ( + "slices" + "time" + + "github.com/mhsanaei/3x-ui/v3/internal/util/sys" +) + +const containerHistoryPrefix = "lxc-v1/" + +func scopedMetric(metric string) bool { + return metric == "cpu" || metric == "swap" +} + +func (s *ServerService) systemMetricKey(metric string) string { + if !scopedMetric(metric) { + return metric + } + if status := s.LastStatus(); status != nil { + if status.ResourceScope != "" { + return containerHistoryPrefix + metric + } + return metric + } + if s.lxcMetrics.UseScopedHistory() { + return containerHistoryPrefix + metric + } + return metric +} + +// Scope metadata applies only to CPU, swap and uptime. +func applyLXCStats(status *Status, sample *sys.LXCStats) { + status.ResourceScope = sample.Scope + status.UnavailableMetrics = append([]string(nil), sample.Unavailable...) + status.Cpu = sample.CPU + status.CpuCores = sample.CPUs + status.LogicalPro = sample.CPUs + status.Swap.Current = sample.SwapUsed + status.Swap.Total = sample.SwapTotal + status.Uptime = sample.Uptime +} + +// MetricAvailable distinguishes an unavailable metric from a measured zero. +// Legacy status responses omit metadata and retain their previous semantics. +func (s *Status) MetricAvailable(metric string) bool { + return s != nil && !slices.Contains(s.UnavailableMetrics, metric) +} + +func appendScopedMetric(status *Status, metric string, t time.Time, value float64) { + if !status.MetricAvailable(metric) { + return + } + if status.ResourceScope != "" { + metric = containerHistoryPrefix + metric + } + systemMetrics.append(metric, t, value) +} diff --git a/internal/web/service/server_scope_test.go b/internal/web/service/server_scope_test.go new file mode 100644 index 000000000..db3cd34a0 --- /dev/null +++ b/internal/web/service/server_scope_test.go @@ -0,0 +1,145 @@ +//go:build linux + +package service + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "slices" + "testing" + "time" + + "github.com/mhsanaei/3x-ui/v3/internal/database/dbtest" + "github.com/mhsanaei/3x-ui/v3/internal/util/sys" +) + +func TestContainerStatusOnlyOverridesCPUSwapAndUptime(t *testing.T) { + root := t.TempDir() + t.Setenv("XUI_DB_FOLDER", root) + dbtest.InitDB(t, filepath.Join(root, "test.db")) + sys.SelfRSS() + proc, cg := filepath.Join(root, "proc"), filepath.Join(root, "cg") + fixtures := map[string]string{ + filepath.Join(proc, "self/mountinfo"): "1 0 0:92 /proc/meminfo /proc/meminfo rw - fuse.lxcfs lxcfs rw\n2 0 0:92 /proc/cpuinfo /proc/cpuinfo rw - fuse.lxcfs lxcfs rw\n3 0 0:92 /proc/uptime /proc/uptime rw - fuse.lxcfs lxcfs rw\n4 0 0:28 / " + cg + " rw - cgroup2 cgroup rw\n", + filepath.Join(proc, "1/cgroup"): "0::/init.scope\n", + filepath.Join(proc, "meminfo"): "MemTotal: 2097152 kB\nMemAvailable: 1966080 kB\nSwapTotal: 524288 kB\nSwapFree: 505384 kB\n", + filepath.Join(proc, "cpuinfo"): "processor : 0\ncpu cores : 16\nprocessor : 1\ncpu cores : 16\n", + filepath.Join(proc, "uptime"): "52624.93 52624.93\n", + filepath.Join(proc, "loadavg"): "1.25 2.50 3.75 1/5 1\n", + filepath.Join(proc, "diskstats"): "8 0 vda 1 0 2 0 1 0 4 0 0 0 0\n", + filepath.Join(proc, "stat"): "cpu 100 0 50 1000 0 0 0 0\n", + filepath.Join(proc, "net/dev"): "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n", + filepath.Join(cg, "cpu.stat"): "usage_usec 222772076\n", + filepath.Join(cg, "cpu.max"): "max 100000\n", + filepath.Join(cg, "io.stat"): "259:0 rbytes=999999999 wbytes=999999999\n", + } + for path, content := range fixtures { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + t.Setenv("HOST_PROC", proc) + t.Setenv("HOST_SYS", filepath.Join(root, "sys")) + svc := &ServerService{cachedIPv4: "192.0.2.1", noIPv6: true, cachedCpuSpeedMhz: 5000, lxcMetrics: sys.LXCReader{ProcRoot: proc, CgroupRoot: cg}} + previous := &Status{T: time.Now().Add(-time.Second)} + got := svc.GetStatus(previous) + if got.ResourceScope != "container" || got.CpuCores != 2 || got.LogicalPro != 2 || got.CpuSpeedMhz != 0 || got.Swap.Total != 512<<20 || got.Swap.Current != 19357696 || got.Uptime != 52624 { + t.Fatalf("incorrect corrected resources: %+v", got) + } + if got.Mem.Total != 2<<30 || got.Mem.Current != 128<<20 || !reflect.DeepEqual(got.Loads, []float64{1.25, 2.5, 3.75}) || got.DiskTraffic.Read != 1024 || got.DiskTraffic.Write != 2048 || got.DiskIO.Read == 0 || got.DiskIO.Write == 0 { + t.Fatalf("unrelated metric path changed: %+v", got) + } + if !slices.Equal(got.UnavailableMetrics, []string{"cpu"}) { + t.Fatalf("unexpected unavailable metrics: %v", got.UnavailableMetrics) + } + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + var wire map[string]json.RawMessage + if err = json.Unmarshal(raw, &wire); err != nil { + t.Fatal(err) + } + for _, key := range []string{"cpu", "cpuCores", "logicalPro", "swap", "uptime", "mem", "loads", "diskIO", "diskTraffic", "resourceScope", "unavailableMetrics"} { + if _, ok := wire[key]; !ok { + t.Fatalf("API field lost: %s", key) + } + } + if err = os.WriteFile(filepath.Join(proc, "self/mountinfo"), []byte("1 0 0:1 / / rw - ext4 /dev/root rw\n"), 0o600); err != nil { + t.Fatal(err) + } + native := svc.GetStatus(previous) + if native.ResourceScope != "" || len(native.UnavailableMetrics) != 0 || native.CpuSpeedMhz != 5000 || native.Mem != got.Mem || !reflect.DeepEqual(native.Loads, got.Loads) || native.DiskTraffic != got.DiskTraffic { + t.Fatalf("native regression: %+v", native) + } +} + +func TestScopedResourcesLeaveUnrelatedStatusUntouched(t *testing.T) { + status := &Status{Loads: []float64{1, 2, 3}, CpuSpeedMhz: 0} + status.Mem.Current, status.Mem.Total = 123, 456 + status.DiskTraffic.Read, status.DiskTraffic.Write = 99, 101 + status.DiskIO.Read, status.DiskIO.Write = 11, 12 + before := *status + applyLXCStats(status, &sys.LXCStats{Scope: "container", CPUs: 2, SwapUsed: 8, SwapTotal: 512, Uptime: 15}) + if status.Mem != before.Mem || status.DiskTraffic != before.DiskTraffic || status.DiskIO != before.DiskIO || !reflect.DeepEqual(status.Loads, before.Loads) { + t.Fatalf("unrelated status overwritten: %+v", status) + } +} + +func TestContainerHistorySeparatesOnlyCPUAndSwap(t *testing.T) { + old := systemMetrics + systemMetrics = newMetricHistory() + defer func() { systemMetrics = old }() + now := time.Now() + systemMetrics.append("cpu", now, 90) + systemMetrics.append("swap", now, 62) + for key, value := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} { + systemMetrics.append(key, now, value) + } + status := &Status{T: now, ResourceScope: "container", Cpu: 10, Loads: []float64{1.25, 2.5, 3.75}, UnavailableMetrics: []string{"swap"}} + status.Mem.Total, status.Mem.Current = 100, 6 + status.DiskIO.Read, status.NetIO.Up = 1024, 7 + svc := &ServerService{lastStatus: status} + svc.AppendStatusSample(now, status) + cpu := svc.AggregateCpuHistory(2, 60) + if len(cpu) != 1 || cpu[0]["cpu"] != float64(10) { + t.Fatalf("CPU history=%v", cpu) + } + native := &ServerService{lastStatus: &Status{}} + if host := native.AggregateCpuHistory(2, 60); len(host) != 1 || host[0]["cpu"] != float64(90) { + t.Fatalf("legacy CPU changed: %v", host) + } + if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 0 { + t.Fatalf("missing swap became zero: %v", points) + } + for key, want := range map[string]float64{"mem": 6, "load1": 1.25, "diskRead": 1024, "netUp": 7} { + if svc.systemMetricKey(key) != key { + t.Fatalf("unrelated history renamed: %s", key) + } + points := svc.AggregateSystemMetric(key, 2, 60) + if len(points) != 1 || points[0]["v"] != want { + t.Fatalf("unrelated %s history=%v", key, points) + } + } + status.UnavailableMetrics = nil + svc.AppendStatusSample(now, status) + if points := svc.AggregateSystemMetric("swap", 2, 60); len(points) != 1 || points[0]["v"] != float64(0) { + t.Fatalf("disabled swap lost: %v", points) + } + snapshot := systemMetrics.snapshot() + for _, key := range []string{"cpu", "swap", containerHistoryPrefix + "cpu", containerHistoryPrefix + "swap"} { + if _, ok := snapshot[key]; !ok { + t.Fatalf("missing preserved series: %s", key) + } + } + for key := range snapshot { + if len(key) > len(containerHistoryPrefix) && key[:len(containerHistoryPrefix)] == containerHistoryPrefix && key != containerHistoryPrefix+"cpu" && key != containerHistoryPrefix+"swap" { + t.Fatalf("unexpected scoped metric: %s", key) + } + } +} diff --git a/internal/web/service/tgbot/tgbot_report.go b/internal/web/service/tgbot/tgbot_report.go index acfe377de..c1da45bbc 100644 --- a/internal/web/service/tgbot/tgbot_report.go +++ b/internal/web/service/tgbot/tgbot_report.go @@ -187,7 +187,12 @@ func (t *Tgbot) prepareServerUsageInfo() string { info += t.I18nBot("tgbot.messages.ipv6", "IPv6=="+ipv6) } - info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+strconv.FormatUint(t.lastStatus.Uptime/86400, 10), "Unit=="+t.I18nBot("tgbot.days")) + uptime, uptimeUnit := "—", "" + if t.lastStatus.MetricAvailable("uptime") { + uptime = strconv.FormatUint(t.lastStatus.Uptime/86400, 10) + uptimeUnit = t.I18nBot("tgbot.days") + } + info += t.I18nBot("tgbot.messages.serverUpTime", "UpTime=="+uptime, "Unit=="+uptimeUnit) info += t.I18nBot("tgbot.messages.serverLoad", "Load1=="+strconv.FormatFloat(t.lastStatus.Loads[0], 'f', 2, 64), "Load2=="+strconv.FormatFloat(t.lastStatus.Loads[1], 'f', 2, 64), "Load3=="+strconv.FormatFloat(t.lastStatus.Loads[2], 'f', 2, 64)) info += t.I18nBot("tgbot.messages.serverMemory", "Current=="+common.FormatTraffic(int64(t.lastStatus.Mem.Current)), "Total=="+common.FormatTraffic(int64(t.lastStatus.Mem.Total))) info += t.I18nBot("tgbot.messages.onlinesCount", "Count=="+fmt.Sprint(len(onlines))) diff --git a/internal/web/service/tgbot/tgbot_report_batching_test.go b/internal/web/service/tgbot/tgbot_report_batching_test.go index d3af5f7e0..3adcf9c6a 100644 --- a/internal/web/service/tgbot/tgbot_report_batching_test.go +++ b/internal/web/service/tgbot/tgbot_report_batching_test.go @@ -14,11 +14,14 @@ import ( "time" "github.com/mymmrac/telego" + "github.com/nicksnyder/go-i18n/v2/i18n" + "golang.org/x/text/language" "gorm.io/gorm" "github.com/mhsanaei/3x-ui/v3/internal/database" "github.com/mhsanaei/3x-ui/v3/internal/database/dbtest" "github.com/mhsanaei/3x-ui/v3/internal/database/model" + "github.com/mhsanaei/3x-ui/v3/internal/web/locale" "github.com/mhsanaei/3x-ui/v3/internal/web/service" "github.com/mhsanaei/3x-ui/v3/internal/xray" ) @@ -284,3 +287,44 @@ func TestNotifyExhaustedLooksUpEachIDOnce(t *testing.T) { }) } } + +func TestServerUsageInfoShowsUnavailableUptime(t *testing.T) { + bundle := i18n.NewBundle(language.English) + _ = bundle.AddMessages(language.English, &i18n.Message{ID: "tgbot.messages.serverUpTime", Other: "Uptime: {{.UpTime}} {{.Unit}}\n"}, &i18n.Message{ID: "tgbot.days", Other: "days"}) + oldLocale := locale.LocalizerBot + locale.LocalizerBot = i18n.NewLocalizer(bundle, "en") + t.Cleanup(func() { locale.LocalizerBot = oldLocale }) + statusCache.mutex.Lock() + oldStatus, oldTime := statusCache.data, statusCache.timestamp + statusCache.mutex.Unlock() + serverStatsCache.mutex.Lock() + oldStats, oldStatsTime := serverStatsCache.data, serverStatsCache.timestamp + serverStatsCache.mutex.Unlock() + t.Cleanup(func() { + statusCache.mutex.Lock() + statusCache.data, statusCache.timestamp = oldStatus, oldTime + statusCache.mutex.Unlock() + serverStatsCache.mutex.Lock() + serverStatsCache.data, serverStatsCache.timestamp = oldStats, oldStatsTime + serverStatsCache.mutex.Unlock() + }) + for _, unavailable := range []bool{true, false} { + t.Run(map[bool]string{true: "unavailable", false: "valid zero"}[unavailable], func(t *testing.T) { + bot := &Tgbot{} + status := &service.Status{Loads: []float64{0, 0, 0}} + if unavailable { + status.UnavailableMetrics = []string{"uptime"} + } + bot.setCachedStatus(status) + bot.setCachedServerStats("") + info := bot.prepareServerUsageInfo() + want := "Uptime: 0 days" + if unavailable { + want = "Uptime: —" + } + if !strings.Contains(info, want) { + t.Fatalf("report did not contain %q: %s", want, info) + } + }) + } +}