mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-22 15:12:23 +03:00
feat(server): native systemd sd_notify watchdog (Type=notify) (#10662)
Merged — validated together with a batch of related maxmad64bis PRs in one combined worktree (typecheck:core clean, complexity/cognitive-complexity/file-size/changelog gates green, focused tests passing). Thanks for the contribution!
This commit is contained in:
@@ -229,6 +229,15 @@ PORT=20128
|
||||
# unaffected by this dev-only flag).
|
||||
OMNIROUTE_USE_TURBOPACK=1
|
||||
|
||||
# Disable systemd sd_notify (Type=notify / WatchdogSec=) even when running
|
||||
# under a systemd unit with NOTIFY_SOCKET set.
|
||||
# Used by: scripts/dev/systemd-notify.mjs. Set to 1 to disable.
|
||||
# OMNIROUTE_DISABLE_SD_NOTIFY=1
|
||||
|
||||
# Injected by systemd when running under a service unit (sd_notify protocol).
|
||||
# Read by scripts/dev/systemd-notify.mjs — never set this yourself.
|
||||
# NOTIFY_SOCKET=/run/systemd/notify
|
||||
|
||||
# Skip the SQLite integrity health check on startup (faster boot on large DBs).
|
||||
# Used by: src/lib/db/core.ts, src/lib/db/healthCheck.ts. Set to 1 to skip.
|
||||
# OMNIROUTE_SKIP_DB_HEALTHCHECK=1
|
||||
|
||||
@@ -121,7 +121,16 @@ function writeLinuxSystemdUnit(cliPath) {
|
||||
"Wants=network-online.target",
|
||||
"",
|
||||
"[Service]",
|
||||
"Type=simple",
|
||||
// Type=notify + WatchdogSec: the server sends READY=1 once listening and
|
||||
// WATCHDOG=1 every 60s; if its event loop ever blocks (frozen process),
|
||||
// the pings stop and systemd kills+restarts the service. NotifyAccess=all
|
||||
// because the pings come from the server child, not the serve supervisor.
|
||||
// Foreground serve only: `--daemon` escapes the cgroup and would break
|
||||
// the notify handshake.
|
||||
"Type=notify",
|
||||
"NotifyAccess=all",
|
||||
"WatchdogSec=180",
|
||||
"TimeoutStartSec=300",
|
||||
`ExecStart=${buildServeExecLine(cliPath, { tray: false })}`,
|
||||
"Restart=on-failure",
|
||||
"RestartSec=5",
|
||||
|
||||
1
changelog.d/features/10662-systemd-notify.md
Normal file
1
changelog.d/features/10662-systemd-notify.md
Normal file
@@ -0,0 +1 @@
|
||||
- feat(server): emit systemd sd_notify READY/WATCHDOG/STOPPING (generated unit becomes Type=notify with WatchdogSec=180) so a frozen server process is killed and restarted by systemd instead of lingering undetected
|
||||
@@ -153,6 +153,8 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari
|
||||
| `NODE_ENV` | `production` | Next.js core | Controls logging verbosity, caching, error detail exposure, and Next.js optimizations. |
|
||||
| `OMNIROUTE_USE_TURBOPACK` | `1` (Turbopack — code default) | `package.json` / Next.js 16 | Turbopack is the default bundler for `npm run dev` and `npm run build` (2-3× faster builds, benchmarked). Set to `0` to fall back to webpack on Windows, when running into native binding / bundler-compat incompatibilities, **or on RAM-constrained machines** — Turbopack production builds on this Next.js version line (16.2.x) are known upstream to peak far higher in memory than webpack on large module graphs (Next 16.3's Turbopack memory-eviction fix is not yet stable); webpack fallback peaks much lower. See #6409. |
|
||||
| `OMNIROUTE_SKIP_DB_HEALTHCHECK` | _(unset)_ | `src/lib/db/core.ts` / `src/lib/db/healthCheck.ts` | Set to `1` to skip the SQLite integrity health check on startup. Useful for faster boot on large databases. |
|
||||
| `NOTIFY_SOCKET` | _(unset)_ | systemd (sd_notify protocol) | Set by systemd when the process runs under a service unit with sd_notify integration; OmniRoute reads it (see `OMNIROUTE_DISABLE_SD_NOTIFY`) to send READY/WATCHDOG notifications. Never set by the user. |
|
||||
| `OMNIROUTE_DISABLE_SD_NOTIFY` | _(unset)_ | `scripts/dev/systemd-notify.mjs` | Set to `1` to disable systemd sd_notify (Type=notify / WatchdogSec=) even when running under a systemd unit. The notifier is a no-op outside systemd regardless. |
|
||||
| `CREDENTIAL_HEALTH_CHECK_INTERVAL` | `300000` | `open-sse/config/constants.ts` / `src/lib/credentialHealth/scheduler.ts` | Interval (ms) for the background credential health check scheduler. Minimum: 10000 (10s). |
|
||||
| `CREDENTIAL_HEALTH_CACHE_TTL` | `300000` | `open-sse/config/constants.ts` / `src/lib/credentialHealth/cache.ts` | TTL (ms) for cached credential health status. |
|
||||
| `OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK` | `false` | `src/lib/credentialHealth/scheduler.ts` | Set to `1` or `true` to disable background periodic testing of provider connections. Search providers (SEARCH_VALIDATOR_CONFIGS in `src/lib/providers/validation/searchProviders.ts`, e.g. `tavily-search`) are always excluded from the sweep — their "validation" is a real billed upstream query, so they are never health-checked on a timer (#9970). |
|
||||
|
||||
@@ -183,6 +183,11 @@ const EXTRA_MODULE_ENTRIES = [
|
||||
src: ["scripts", "dev", "main-server-timeouts.mjs"],
|
||||
dest: ["main-server-timeouts.mjs"],
|
||||
},
|
||||
{
|
||||
label: "systemd sd_notify helper (server-ws.mjs dependency)",
|
||||
src: ["scripts", "dev", "systemd-notify.mjs"],
|
||||
dest: ["systemd-notify.mjs"],
|
||||
},
|
||||
{
|
||||
label: "HTTP method guard (server-ws.mjs dependency)",
|
||||
src: ["scripts", "dev", "http-method-guard.cjs"],
|
||||
|
||||
@@ -49,6 +49,9 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [
|
||||
"package.json",
|
||||
"peer-stamp.mjs",
|
||||
"main-server-timeouts.mjs",
|
||||
// server-ws.mjs import (sd_notify helper) — enforced by the closure test
|
||||
// tests/unit/pack-artifact-server-ws-closure.test.ts.
|
||||
"systemd-notify.mjs",
|
||||
"responses-ws-proxy.mjs",
|
||||
"bin/chatgpt-web-codex-mcp.mjs",
|
||||
"scripts/dev/sync-env.mjs",
|
||||
@@ -184,6 +187,8 @@ export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [
|
||||
"dist/responses-ws-proxy.mjs",
|
||||
"dist/peer-stamp.mjs",
|
||||
"dist/main-server-timeouts.mjs",
|
||||
// server-ws.mjs import (sd_notify helper) — enforced by the closure test.
|
||||
"dist/systemd-notify.mjs",
|
||||
"dist/http-method-guard.cjs",
|
||||
// #5452: regression guard — make check:pack-artifact fail loudly if the TLS
|
||||
// opt-in sidecar (imported by dist/server-ws.mjs) ever vanishes from the tarball.
|
||||
|
||||
@@ -61,6 +61,10 @@ const IGNORE_FROM_CODE = new Set([
|
||||
"APPDATA",
|
||||
"LOCALAPPDATA",
|
||||
"XDG_CONFIG_HOME",
|
||||
// systemd-injected notify socket path (sd_notify protocol, see
|
||||
// scripts/dev/systemd-notify.mjs) — set by systemd only when running under
|
||||
// a unit, never user config.
|
||||
"NOTIFY_SOCKET",
|
||||
// XDG Base Directory cache root — read (never defined by OmniRoute) so the
|
||||
// Android/Termux serve path can honor an operator-set cache location (#8519).
|
||||
"XDG_CACHE_HOME",
|
||||
|
||||
@@ -15,6 +15,7 @@ import { ensureNativeSqlite } from "./ensure-native-sqlite.mjs";
|
||||
import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopackCacheHeal.mjs";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
|
||||
const { maybeHandleDisallowedMethod } = methodGuard;
|
||||
const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard;
|
||||
@@ -60,6 +61,13 @@ for (const [key, value] of Object.entries(mergedEnv)) {
|
||||
}
|
||||
}
|
||||
|
||||
// systemd sd_notify (Type=notify / WatchdogSec=): this process owns the
|
||||
// watchdog pings — if its event loop blocks (freeze), the pings stop and
|
||||
// systemd kills the service. No-op outside systemd (no NOTIFY_SOCKET).
|
||||
// Created AFTER .env is merged so the OMNIROUTE_DISABLE_SD_NOTIFY opt-out
|
||||
// documented in .env is honored on this path too.
|
||||
const systemdNotifier = createSystemdNotifier();
|
||||
|
||||
// The mergedEnv copy above pulls NODE_ENV straight from `.env` — and the shipped
|
||||
// `.env.example` default is `NODE_ENV=production`. Next's programmatic `next()`
|
||||
// entry (unlike the `next` CLI) trusts that value verbatim, so `npm run dev`
|
||||
@@ -184,6 +192,7 @@ async function start() {
|
||||
});
|
||||
|
||||
const shutdown = async (signal) => {
|
||||
systemdNotifier.stopping();
|
||||
try {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
await nextApp.close();
|
||||
@@ -202,6 +211,8 @@ async function start() {
|
||||
console.log(
|
||||
`[Next] ${mode} server listening on http://${hostname}:${dashboardPort} (${bundler})`
|
||||
);
|
||||
systemdNotifier.ready();
|
||||
systemdNotifier.startWatchdog();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,20 @@ import methodGuard from "./http-method-guard.cjs";
|
||||
import headResponseGuard from "./head-response-guard.cjs";
|
||||
import { resolveTlsOptions, createServerListener } from "./tls-options.mjs";
|
||||
import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
|
||||
import { createSystemdNotifier } from "./systemd-notify.mjs";
|
||||
|
||||
// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one
|
||||
// whose event loop can freeze (cold /v1/models rebuild), so it must own the
|
||||
// watchdog pings — a blocked loop stops the pings and systemd kills the
|
||||
// service. No-op outside systemd (no NOTIFY_SOCKET).
|
||||
const systemdNotifier = createSystemdNotifier();
|
||||
let systemdReadySent = false;
|
||||
// NOTE: if an operator sets NEXT_MANUAL_SIG_HANDLE=1, Next never registers its
|
||||
// own signal cleanup and these once() handlers would suppress Node's default
|
||||
// signal exit (process lingers until systemd's stop-timeout SIGKILL). Nothing
|
||||
// in this repo sets that var; acceptable, documented behavior.
|
||||
process.once("SIGINT", () => systemdNotifier.stopping());
|
||||
process.once("SIGTERM", () => systemdNotifier.stopping());
|
||||
|
||||
const originalCreateServer = http.createServer.bind(http);
|
||||
const proxiesByPort = new Map();
|
||||
@@ -209,6 +223,15 @@ http.createServer = function createServerWithResponsesWs(...args) {
|
||||
return originalAddListener(eventName, listener);
|
||||
};
|
||||
|
||||
// sd_notify READY once the main listener is actually accepting, then arm
|
||||
// the watchdog keep-alive interval (unref'd — never keeps the process up).
|
||||
server.once("listening", () => {
|
||||
if (systemdReadySent) return;
|
||||
systemdReadySent = true;
|
||||
systemdNotifier.ready();
|
||||
systemdNotifier.startWatchdog();
|
||||
});
|
||||
|
||||
return server;
|
||||
};
|
||||
|
||||
|
||||
98
scripts/dev/systemd-notify.mjs
Normal file
98
scripts/dev/systemd-notify.mjs
Normal file
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Minimal systemd sd_notify integration (sd_notify(3) protocol).
|
||||
*
|
||||
* Node's stable API has no AF_UNIX datagram socket support (node:dgram is
|
||||
* udp4/udp6 only), so notifications are sent by spawning the `systemd-notify`
|
||||
* binary — present on every systemd host, no extra dependency.
|
||||
*
|
||||
* Everything is guarded: without a NOTIFY_SOCKET (plain terminal, Docker,
|
||||
* Electron, Windows) the notifier is a no-op and costs nothing. Set
|
||||
* OMNIROUTE_DISABLE_SD_NOTIFY=1 to force-disable even under systemd.
|
||||
*
|
||||
* A watchdog keep-alive interval lives in the main event loop of the process
|
||||
* that runs it: if that loop is ever blocked (frozen server, cf. the cold
|
||||
* /v1/models rebuild freeze), the pings stop and systemd kills the service
|
||||
* after WatchdogSec=.
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
export const SD_NOTIFY_BINARY = "systemd-notify";
|
||||
export const SD_NOTIFY_SOCKET_ENV = "NOTIFY_SOCKET";
|
||||
export const SD_NOTIFY_DISABLE_ENV = "OMNIROUTE_DISABLE_SD_NOTIFY";
|
||||
// Ping every 60s — satisfies any systemd WatchdogSec= >= 120s (systemd
|
||||
// requires keep-alive pings at most every WatchdogSec/2).
|
||||
export const SD_NOTIFY_WATCHDOG_INTERVAL_MS = 60_000;
|
||||
|
||||
export function isSystemdNotifyEnabled(env = process.env) {
|
||||
return Boolean(env[SD_NOTIFY_SOCKET_ENV]) && env[SD_NOTIFY_DISABLE_ENV] !== "1";
|
||||
}
|
||||
|
||||
export function buildNotifyMessage(kind) {
|
||||
switch (kind) {
|
||||
case "ready":
|
||||
return "READY=1";
|
||||
case "watchdog":
|
||||
return "WATCHDOG=1";
|
||||
case "stopping":
|
||||
return "STOPPING=1";
|
||||
default:
|
||||
throw new Error(`[omniroute][sd_notify] unknown message kind: ${kind}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function createSystemdNotifier({
|
||||
env = process.env,
|
||||
binary = SD_NOTIFY_BINARY,
|
||||
watchdogIntervalMs = SD_NOTIFY_WATCHDOG_INTERVAL_MS,
|
||||
spawnFn = spawn,
|
||||
onWarn = (message) => console.warn(message),
|
||||
} = {}) {
|
||||
const enabled = isSystemdNotifyEnabled(env);
|
||||
let disabled = false;
|
||||
let watchdogTimer = null;
|
||||
|
||||
const send = (kind) => {
|
||||
if (!enabled || disabled) return;
|
||||
const child = spawnFn(binary, [buildNotifyMessage(kind)], { env, stdio: "ignore" });
|
||||
// Never let a hung systemd-notify keep the process alive.
|
||||
child.unref?.();
|
||||
child.on("error", (err) => {
|
||||
// A failed send means systemd never sees the keep-alive: the service
|
||||
// would be killed as unhealthy anyway, so disabling loudly (one
|
||||
// warning) is safer than spamming errors forever.
|
||||
disabled = true;
|
||||
if (watchdogTimer) {
|
||||
clearInterval(watchdogTimer);
|
||||
watchdogTimer = null;
|
||||
}
|
||||
onWarn(
|
||||
`[omniroute][sd_notify] failed to send '${kind}' (${err?.code ?? err?.message ?? err}); sd_notify disabled for this process`
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
enabled,
|
||||
ready() {
|
||||
send("ready");
|
||||
},
|
||||
watchdog() {
|
||||
send("watchdog");
|
||||
},
|
||||
stopping() {
|
||||
send("stopping");
|
||||
},
|
||||
startWatchdog() {
|
||||
if (!enabled || disabled || watchdogTimer) return;
|
||||
watchdogTimer = setInterval(() => send("watchdog"), watchdogIntervalMs);
|
||||
watchdogTimer.unref?.();
|
||||
},
|
||||
dispose() {
|
||||
if (watchdogTimer) {
|
||||
clearInterval(watchdogTimer);
|
||||
watchdogTimer = null;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -101,3 +101,41 @@ test("Linux enable path prefers graphical desktop autostart over systemd", () =>
|
||||
assert.ok(systemdBranch > -1, "expected a systemd fallback branch");
|
||||
assert.ok(graphicalBranch < systemdBranch, "graphical autostart should be preferred");
|
||||
});
|
||||
|
||||
test("systemd branch writes Type=notify sd_notify directives (headless, stubs succeed)", async () => {
|
||||
if (process.platform !== "linux") return;
|
||||
const stubBin = join(tmpDir, "stub-bin");
|
||||
const unitPath = join(tmpDir, ".config", "systemd", "user", "omniroute.service");
|
||||
const envKeys = ["DISPLAY", "WAYLAND_DISPLAY", "XDG_CURRENT_DESKTOP"] as const;
|
||||
const savedEnv: Record<string, string | undefined> = {};
|
||||
for (const key of envKeys) savedEnv[key] = process.env[key];
|
||||
|
||||
try {
|
||||
for (const key of envKeys) delete process.env[key];
|
||||
for (const name of ["systemctl", "loginctl"]) {
|
||||
writeFileSync(join(stubBin, name), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
}
|
||||
// autostart.mjs reads process.env / runs the stubs at call time, so a
|
||||
// cached module is fine.
|
||||
const { enable } = await import("../../../bin/cli/tray/autostart.mjs");
|
||||
|
||||
const ok = enable();
|
||||
assert.equal(ok, true, "enable() should succeed through the systemd branch");
|
||||
assert.ok(existsSync(unitPath), "systemd unit should be written");
|
||||
|
||||
const unit = readFileSync(unitPath, "utf8");
|
||||
assert.match(unit, /Type=notify/);
|
||||
assert.match(unit, /NotifyAccess=all/);
|
||||
assert.match(unit, /WatchdogSec=180/);
|
||||
assert.match(unit, /TimeoutStartSec=300/);
|
||||
assert.match(unit, /Restart=on-failure/);
|
||||
} finally {
|
||||
for (const key of envKeys) {
|
||||
if (savedEnv[key] === undefined) delete process.env[key];
|
||||
else process.env[key] = savedEnv[key];
|
||||
}
|
||||
for (const name of ["systemctl", "loginctl"]) {
|
||||
writeFileSync(join(stubBin, name), "#!/bin/sh\nexit 1\n", { mode: 0o755 });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -223,6 +223,7 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball",
|
||||
"dist/responses-ws-proxy.mjs",
|
||||
"dist/server-ws.mjs",
|
||||
"dist/src/lib/usage/callLogArtifactWorker.js",
|
||||
"dist/systemd-notify.mjs",
|
||||
"dist/tls-options.mjs",
|
||||
"dist/webdav-handler.mjs",
|
||||
"scripts/build/colocateOptionals.mjs",
|
||||
|
||||
258
tests/unit/systemd-notify.test.mjs
Normal file
258
tests/unit/systemd-notify.test.mjs
Normal file
@@ -0,0 +1,258 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
|
||||
import {
|
||||
buildNotifyMessage,
|
||||
createSystemdNotifier,
|
||||
isSystemdNotifyEnabled,
|
||||
SD_NOTIFY_WATCHDOG_INTERVAL_MS,
|
||||
} from "../../scripts/dev/systemd-notify.mjs";
|
||||
|
||||
function fakeSpawn(recorder) {
|
||||
return (binary, args, options) => {
|
||||
const child = new EventEmitter();
|
||||
recorder.push({ binary, args, options, child });
|
||||
return child;
|
||||
};
|
||||
}
|
||||
|
||||
function systemdNotifyAvailable() {
|
||||
try {
|
||||
return spawnSync("systemd-notify", ["--version"], { stdio: "ignore" }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function python3Available() {
|
||||
try {
|
||||
return spawnSync("python3", ["--version"], { stdio: "ignore" }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Waits for the listener to emit `expected` lines (in order), then resolves
|
||||
// with everything it saw. Fails loudly on timeout or premature exit.
|
||||
// BARRIER=1 datagrams (sd_notify synchronization emitted by the systemd-notify
|
||||
// CLI after every message) are noise for this contract and are skipped.
|
||||
function waitForLines(child, expected, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let buf = "";
|
||||
const seen = [];
|
||||
const timer = setTimeout(
|
||||
() => reject(new Error(`listener timed out after ${timeoutMs}ms; got: ${seen.join(", ")}`)),
|
||||
timeoutMs
|
||||
);
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
buf += chunk;
|
||||
let idx;
|
||||
while ((idx = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, idx).trim();
|
||||
buf = buf.slice(idx + 1);
|
||||
if (!line || line === "BARRIER=1") continue;
|
||||
seen.push(line);
|
||||
if (seen.length === expected.length) {
|
||||
clearTimeout(timer);
|
||||
resolve([...seen]);
|
||||
}
|
||||
}
|
||||
});
|
||||
child.on("exit", () => {
|
||||
if (seen.length < expected.length) {
|
||||
clearTimeout(timer);
|
||||
reject(new Error(`listener exited early; got: ${seen.join(", ")}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test("isSystemdNotifyEnabled: false without NOTIFY_SOCKET", () => {
|
||||
assert.equal(isSystemdNotifyEnabled({}), false);
|
||||
assert.equal(isSystemdNotifyEnabled({ OMNIROUTE_DISABLE_SD_NOTIFY: "0" }), false);
|
||||
});
|
||||
|
||||
test("isSystemdNotifyEnabled: true when NOTIFY_SOCKET is set", () => {
|
||||
assert.equal(isSystemdNotifyEnabled({ NOTIFY_SOCKET: "/run/systemd/notify" }), true);
|
||||
});
|
||||
|
||||
test("isSystemdNotifyEnabled: opt-out OMNIROUTE_DISABLE_SD_NOTIFY=1 wins", () => {
|
||||
assert.equal(
|
||||
isSystemdNotifyEnabled({
|
||||
NOTIFY_SOCKET: "/run/systemd/notify",
|
||||
OMNIROUTE_DISABLE_SD_NOTIFY: "1",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("notifier honors the opt-out (no spawn, no timer)", async () => {
|
||||
const calls = [];
|
||||
const notifier = createSystemdNotifier({
|
||||
env: { NOTIFY_SOCKET: "/run/systemd/notify", OMNIROUTE_DISABLE_SD_NOTIFY: "1" },
|
||||
spawnFn: fakeSpawn(calls),
|
||||
});
|
||||
assert.equal(notifier.enabled, false);
|
||||
notifier.ready();
|
||||
notifier.startWatchdog();
|
||||
await new Promise((resolve) => setTimeout(resolve, 15));
|
||||
assert.deepEqual(calls, []);
|
||||
notifier.dispose();
|
||||
});
|
||||
|
||||
// Contract with the generated systemd unit (bin/cli/tray/autostart.mjs):
|
||||
// systemd requires keep-alive pings at most every WatchdogSec/2, and the
|
||||
// generated unit pins WatchdogSec=180. If either side drifts, this fails
|
||||
// loudly instead of silently breaking the watchdog.
|
||||
test("watchdog cadence satisfies the generated unit's WatchdogSec", () => {
|
||||
const generatedWatchdogSec = 180;
|
||||
assert.ok(
|
||||
2 * SD_NOTIFY_WATCHDOG_INTERVAL_MS <= generatedWatchdogSec * 1000,
|
||||
`ping interval ${SD_NOTIFY_WATCHDOG_INTERVAL_MS}ms must be <= WatchdogSec/2 (${generatedWatchdogSec / 2}s)`
|
||||
);
|
||||
});
|
||||
|
||||
test("buildNotifyMessage: known kinds", () => {
|
||||
assert.equal(buildNotifyMessage("ready"), "READY=1");
|
||||
assert.equal(buildNotifyMessage("watchdog"), "WATCHDOG=1");
|
||||
assert.equal(buildNotifyMessage("stopping"), "STOPPING=1");
|
||||
});
|
||||
|
||||
test("buildNotifyMessage: unknown kind throws", () => {
|
||||
assert.throws(() => buildNotifyMessage("bogus"), /unknown message kind/);
|
||||
});
|
||||
|
||||
test("disabled notifier is a complete no-op (no spawn, no timer)", async () => {
|
||||
const calls = [];
|
||||
const notifier = createSystemdNotifier({ env: {}, spawnFn: fakeSpawn(calls) });
|
||||
assert.equal(notifier.enabled, false);
|
||||
notifier.ready();
|
||||
notifier.watchdog();
|
||||
notifier.stopping();
|
||||
notifier.startWatchdog();
|
||||
await new Promise((resolve) => setTimeout(resolve, 15));
|
||||
assert.deepEqual(calls, []);
|
||||
notifier.dispose();
|
||||
});
|
||||
|
||||
test("enabled notifier sends the right message per kind", () => {
|
||||
const calls = [];
|
||||
const env = { NOTIFY_SOCKET: "/run/systemd/notify" };
|
||||
const notifier = createSystemdNotifier({ env, spawnFn: fakeSpawn(calls) });
|
||||
assert.equal(notifier.enabled, true);
|
||||
notifier.ready();
|
||||
notifier.watchdog();
|
||||
notifier.stopping();
|
||||
assert.deepEqual(
|
||||
calls.map((c) => c.args),
|
||||
[["READY=1"], ["WATCHDOG=1"], ["STOPPING=1"]]
|
||||
);
|
||||
calls.forEach((c) => {
|
||||
assert.equal(c.binary, "systemd-notify");
|
||||
assert.equal(c.options.env, env);
|
||||
assert.deepEqual(c.options.stdio, "ignore");
|
||||
});
|
||||
});
|
||||
|
||||
test("spawn failure disables the notifier once and stops the watchdog", async () => {
|
||||
const calls = [];
|
||||
const warnings = [];
|
||||
const env = { NOTIFY_SOCKET: "/run/systemd/notify" };
|
||||
const notifier = createSystemdNotifier({
|
||||
env,
|
||||
spawnFn: fakeSpawn(calls),
|
||||
onWarn: (m) => warnings.push(m),
|
||||
});
|
||||
notifier.ready();
|
||||
calls[0].child.emit("error", { code: "ENOENT" });
|
||||
assert.equal(notifier.enabled, true); // env guard unchanged
|
||||
notifier.watchdog();
|
||||
notifier.stopping();
|
||||
notifier.startWatchdog();
|
||||
assert.equal(calls.length, 1, "no further spawn after disable");
|
||||
assert.equal(warnings.length, 1, "warning emitted exactly once");
|
||||
assert.match(warnings[0], /ENOENT/);
|
||||
notifier.dispose();
|
||||
});
|
||||
|
||||
test("watchdog interval pings repeatedly and dispose() stops it", async () => {
|
||||
const calls = [];
|
||||
const env = { NOTIFY_SOCKET: "/run/systemd/notify" };
|
||||
// Interval is intentionally tiny (5ms) with a generous wait window (150ms — 30x the
|
||||
// interval) so this stays deterministic under CI/CPU-contended runs: only 2 pings are
|
||||
// required, well within reach even at >10x scheduler jitter.
|
||||
const notifier = createSystemdNotifier({
|
||||
env,
|
||||
watchdogIntervalMs: 5,
|
||||
spawnFn: fakeSpawn(calls),
|
||||
});
|
||||
notifier.ready();
|
||||
notifier.startWatchdog();
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
notifier.dispose();
|
||||
const watchdogCalls = calls.filter((c) => c.args[0] === "WATCHDOG=1").length;
|
||||
assert.ok(watchdogCalls >= 2, `expected >= 2 watchdog pings, got ${watchdogCalls}`);
|
||||
const before = calls.length;
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
assert.equal(calls.length, before, "no ping after dispose");
|
||||
});
|
||||
|
||||
// End-to-end handshake against the real `systemd-notify` binary: a fake
|
||||
// NOTIFY_SOCKET (AF_UNIX datagram listener on a temp path, hosted by a tiny
|
||||
// python3 helper — node:dgram only supports udp4/udp6) receives the exact
|
||||
// datagrams a real systemd would see. Skipped where the binary or python3 is
|
||||
// absent (Windows runners, minimal containers) — the spawn-level contract
|
||||
// above still applies there.
|
||||
const PY_DGRAM_LISTENER = `
|
||||
import socket, sys, os
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
p = sys.argv[1]
|
||||
try:
|
||||
os.unlink(p)
|
||||
except OSError:
|
||||
pass
|
||||
s.bind(p)
|
||||
print("LISTENING", flush=True)
|
||||
while True:
|
||||
try:
|
||||
print(s.recv(4096).decode(), flush=True)
|
||||
except OSError:
|
||||
break
|
||||
`;
|
||||
|
||||
test(
|
||||
"real handshake: systemd-notify delivers READY=1/WATCHDOG=1/STOPPING=1 over a fake NOTIFY_SOCKET",
|
||||
{
|
||||
skip:
|
||||
(!systemdNotifyAvailable() || !python3Available()) &&
|
||||
"systemd-notify and/or python3 unavailable",
|
||||
},
|
||||
async () => {
|
||||
const sockPath = path.join(os.tmpdir(), `omniroute-sdnotify-${process.pid}.sock`);
|
||||
const listener = spawn("python3", ["-c", PY_DGRAM_LISTENER, sockPath], {
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
try {
|
||||
await waitForLines(listener, ["LISTENING"], 5000);
|
||||
const notifier = createSystemdNotifier({
|
||||
env: { NOTIFY_SOCKET: sockPath, PATH: process.env.PATH },
|
||||
});
|
||||
assert.equal(notifier.enabled, true);
|
||||
notifier.ready();
|
||||
notifier.watchdog();
|
||||
notifier.stopping();
|
||||
const received = await waitForLines(listener, ["READY=1", "WATCHDOG=1", "STOPPING=1"], 10000);
|
||||
assert.deepEqual(received, ["READY=1", "WATCHDOG=1", "STOPPING=1"]);
|
||||
notifier.dispose();
|
||||
} finally {
|
||||
listener.kill();
|
||||
fs.rmSync(sockPath, { force: true });
|
||||
}
|
||||
}
|
||||
);
|
||||
Reference in New Issue
Block a user