diff --git a/.env.example b/.env.example index d193fa35a0..d5d61a1f99 100644 --- a/.env.example +++ b/.env.example @@ -229,6 +229,15 @@ PORT=20128 # unaffected by this dev-only flag). OMNIROUTE_USE_TURBOPACK=1 +# Disable systemd sd_notify (Type=notify / WatchdogSec=) even when running +# under a systemd unit with NOTIFY_SOCKET set. +# Used by: scripts/dev/systemd-notify.mjs. Set to 1 to disable. +# OMNIROUTE_DISABLE_SD_NOTIFY=1 + +# Injected by systemd when running under a service unit (sd_notify protocol). +# Read by scripts/dev/systemd-notify.mjs — never set this yourself. +# NOTIFY_SOCKET=/run/systemd/notify + # Skip the SQLite integrity health check on startup (faster boot on large DBs). # Used by: src/lib/db/core.ts, src/lib/db/healthCheck.ts. Set to 1 to skip. # OMNIROUTE_SKIP_DB_HEALTHCHECK=1 diff --git a/bin/cli/tray/autostart.mjs b/bin/cli/tray/autostart.mjs index 6c1ba21aee..a61a5c739a 100644 --- a/bin/cli/tray/autostart.mjs +++ b/bin/cli/tray/autostart.mjs @@ -121,7 +121,16 @@ function writeLinuxSystemdUnit(cliPath) { "Wants=network-online.target", "", "[Service]", - "Type=simple", + // Type=notify + WatchdogSec: the server sends READY=1 once listening and + // WATCHDOG=1 every 60s; if its event loop ever blocks (frozen process), + // the pings stop and systemd kills+restarts the service. NotifyAccess=all + // because the pings come from the server child, not the serve supervisor. + // Foreground serve only: `--daemon` escapes the cgroup and would break + // the notify handshake. + "Type=notify", + "NotifyAccess=all", + "WatchdogSec=180", + "TimeoutStartSec=300", `ExecStart=${buildServeExecLine(cliPath, { tray: false })}`, "Restart=on-failure", "RestartSec=5", diff --git a/changelog.d/features/10662-systemd-notify.md b/changelog.d/features/10662-systemd-notify.md new file mode 100644 index 0000000000..5a02e7df25 --- /dev/null +++ b/changelog.d/features/10662-systemd-notify.md @@ -0,0 +1 @@ +- feat(server): emit systemd sd_notify READY/WATCHDOG/STOPPING (generated unit becomes Type=notify with WatchdogSec=180) so a frozen server process is killed and restarted by systemd instead of lingering undetected diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 981cdc473b..a6823b086b 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -153,6 +153,8 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `NODE_ENV` | `production` | Next.js core | Controls logging verbosity, caching, error detail exposure, and Next.js optimizations. | | `OMNIROUTE_USE_TURBOPACK` | `1` (Turbopack — code default) | `package.json` / Next.js 16 | Turbopack is the default bundler for `npm run dev` and `npm run build` (2-3× faster builds, benchmarked). Set to `0` to fall back to webpack on Windows, when running into native binding / bundler-compat incompatibilities, **or on RAM-constrained machines** — Turbopack production builds on this Next.js version line (16.2.x) are known upstream to peak far higher in memory than webpack on large module graphs (Next 16.3's Turbopack memory-eviction fix is not yet stable); webpack fallback peaks much lower. See #6409. | | `OMNIROUTE_SKIP_DB_HEALTHCHECK` | _(unset)_ | `src/lib/db/core.ts` / `src/lib/db/healthCheck.ts` | Set to `1` to skip the SQLite integrity health check on startup. Useful for faster boot on large databases. | +| `NOTIFY_SOCKET` | _(unset)_ | systemd (sd_notify protocol) | Set by systemd when the process runs under a service unit with sd_notify integration; OmniRoute reads it (see `OMNIROUTE_DISABLE_SD_NOTIFY`) to send READY/WATCHDOG notifications. Never set by the user. | +| `OMNIROUTE_DISABLE_SD_NOTIFY` | _(unset)_ | `scripts/dev/systemd-notify.mjs` | Set to `1` to disable systemd sd_notify (Type=notify / WatchdogSec=) even when running under a systemd unit. The notifier is a no-op outside systemd regardless. | | `CREDENTIAL_HEALTH_CHECK_INTERVAL` | `300000` | `open-sse/config/constants.ts` / `src/lib/credentialHealth/scheduler.ts` | Interval (ms) for the background credential health check scheduler. Minimum: 10000 (10s). | | `CREDENTIAL_HEALTH_CACHE_TTL` | `300000` | `open-sse/config/constants.ts` / `src/lib/credentialHealth/cache.ts` | TTL (ms) for cached credential health status. | | `OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK` | `false` | `src/lib/credentialHealth/scheduler.ts` | Set to `1` or `true` to disable background periodic testing of provider connections. Search providers (SEARCH_VALIDATOR_CONFIGS in `src/lib/providers/validation/searchProviders.ts`, e.g. `tavily-search`) are always excluded from the sweep — their "validation" is a real billed upstream query, so they are never health-checked on a timer (#9970). | diff --git a/scripts/build/assembleStandalone.mjs b/scripts/build/assembleStandalone.mjs index ccdd61a654..b4c8d12c1f 100644 --- a/scripts/build/assembleStandalone.mjs +++ b/scripts/build/assembleStandalone.mjs @@ -183,6 +183,11 @@ const EXTRA_MODULE_ENTRIES = [ src: ["scripts", "dev", "main-server-timeouts.mjs"], dest: ["main-server-timeouts.mjs"], }, + { + label: "systemd sd_notify helper (server-ws.mjs dependency)", + src: ["scripts", "dev", "systemd-notify.mjs"], + dest: ["systemd-notify.mjs"], + }, { label: "HTTP method guard (server-ws.mjs dependency)", src: ["scripts", "dev", "http-method-guard.cjs"], diff --git a/scripts/build/pack-artifact-policy.ts b/scripts/build/pack-artifact-policy.ts index e1076d70fe..f5edcf994c 100644 --- a/scripts/build/pack-artifact-policy.ts +++ b/scripts/build/pack-artifact-policy.ts @@ -49,6 +49,9 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [ "package.json", "peer-stamp.mjs", "main-server-timeouts.mjs", + // server-ws.mjs import (sd_notify helper) — enforced by the closure test + // tests/unit/pack-artifact-server-ws-closure.test.ts. + "systemd-notify.mjs", "responses-ws-proxy.mjs", "bin/chatgpt-web-codex-mcp.mjs", "scripts/dev/sync-env.mjs", @@ -184,6 +187,8 @@ export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [ "dist/responses-ws-proxy.mjs", "dist/peer-stamp.mjs", "dist/main-server-timeouts.mjs", + // server-ws.mjs import (sd_notify helper) — enforced by the closure test. + "dist/systemd-notify.mjs", "dist/http-method-guard.cjs", // #5452: regression guard — make check:pack-artifact fail loudly if the TLS // opt-in sidecar (imported by dist/server-ws.mjs) ever vanishes from the tarball. diff --git a/scripts/check/check-env-doc-sync.mjs b/scripts/check/check-env-doc-sync.mjs index 427813063e..1fba62bfd5 100644 --- a/scripts/check/check-env-doc-sync.mjs +++ b/scripts/check/check-env-doc-sync.mjs @@ -61,6 +61,10 @@ const IGNORE_FROM_CODE = new Set([ "APPDATA", "LOCALAPPDATA", "XDG_CONFIG_HOME", + // systemd-injected notify socket path (sd_notify protocol, see + // scripts/dev/systemd-notify.mjs) — set by systemd only when running under + // a unit, never user config. + "NOTIFY_SOCKET", // XDG Base Directory cache root — read (never defined by OmniRoute) so the // Android/Termux serve path can honor an operator-set cache location (#8519). "XDG_CACHE_HOME", diff --git a/scripts/dev/run-next.mjs b/scripts/dev/run-next.mjs index 54c33e56df..325398c286 100644 --- a/scripts/dev/run-next.mjs +++ b/scripts/dev/run-next.mjs @@ -15,6 +15,7 @@ import { ensureNativeSqlite } from "./ensure-native-sqlite.mjs"; import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopackCacheHeal.mjs"; import { randomUUID } from "node:crypto"; import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs"; +import { createSystemdNotifier } from "./systemd-notify.mjs"; const { maybeHandleDisallowedMethod } = methodGuard; const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard; @@ -60,6 +61,13 @@ for (const [key, value] of Object.entries(mergedEnv)) { } } +// systemd sd_notify (Type=notify / WatchdogSec=): this process owns the +// watchdog pings — if its event loop blocks (freeze), the pings stop and +// systemd kills the service. No-op outside systemd (no NOTIFY_SOCKET). +// Created AFTER .env is merged so the OMNIROUTE_DISABLE_SD_NOTIFY opt-out +// documented in .env is honored on this path too. +const systemdNotifier = createSystemdNotifier(); + // The mergedEnv copy above pulls NODE_ENV straight from `.env` — and the shipped // `.env.example` default is `NODE_ENV=production`. Next's programmatic `next()` // entry (unlike the `next` CLI) trusts that value verbatim, so `npm run dev` @@ -184,6 +192,7 @@ async function start() { }); const shutdown = async (signal) => { + systemdNotifier.stopping(); try { await new Promise((resolve) => server.close(resolve)); await nextApp.close(); @@ -202,6 +211,8 @@ async function start() { console.log( `[Next] ${mode} server listening on http://${hostname}:${dashboardPort} (${bundler})` ); + systemdNotifier.ready(); + systemdNotifier.startWatchdog(); }); } diff --git a/scripts/dev/standalone-server-ws.mjs b/scripts/dev/standalone-server-ws.mjs index ee5f0a1bec..65fb3ab65a 100644 --- a/scripts/dev/standalone-server-ws.mjs +++ b/scripts/dev/standalone-server-ws.mjs @@ -8,6 +8,20 @@ import methodGuard from "./http-method-guard.cjs"; import headResponseGuard from "./head-response-guard.cjs"; import { resolveTlsOptions, createServerListener } from "./tls-options.mjs"; import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs"; +import { createSystemdNotifier } from "./systemd-notify.mjs"; + +// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one +// whose event loop can freeze (cold /v1/models rebuild), so it must own the +// watchdog pings — a blocked loop stops the pings and systemd kills the +// service. No-op outside systemd (no NOTIFY_SOCKET). +const systemdNotifier = createSystemdNotifier(); +let systemdReadySent = false; +// NOTE: if an operator sets NEXT_MANUAL_SIG_HANDLE=1, Next never registers its +// own signal cleanup and these once() handlers would suppress Node's default +// signal exit (process lingers until systemd's stop-timeout SIGKILL). Nothing +// in this repo sets that var; acceptable, documented behavior. +process.once("SIGINT", () => systemdNotifier.stopping()); +process.once("SIGTERM", () => systemdNotifier.stopping()); const originalCreateServer = http.createServer.bind(http); const proxiesByPort = new Map(); @@ -209,6 +223,15 @@ http.createServer = function createServerWithResponsesWs(...args) { return originalAddListener(eventName, listener); }; + // sd_notify READY once the main listener is actually accepting, then arm + // the watchdog keep-alive interval (unref'd — never keeps the process up). + server.once("listening", () => { + if (systemdReadySent) return; + systemdReadySent = true; + systemdNotifier.ready(); + systemdNotifier.startWatchdog(); + }); + return server; }; diff --git a/scripts/dev/systemd-notify.mjs b/scripts/dev/systemd-notify.mjs new file mode 100644 index 0000000000..778deb79d7 --- /dev/null +++ b/scripts/dev/systemd-notify.mjs @@ -0,0 +1,98 @@ +/** + * Minimal systemd sd_notify integration (sd_notify(3) protocol). + * + * Node's stable API has no AF_UNIX datagram socket support (node:dgram is + * udp4/udp6 only), so notifications are sent by spawning the `systemd-notify` + * binary — present on every systemd host, no extra dependency. + * + * Everything is guarded: without a NOTIFY_SOCKET (plain terminal, Docker, + * Electron, Windows) the notifier is a no-op and costs nothing. Set + * OMNIROUTE_DISABLE_SD_NOTIFY=1 to force-disable even under systemd. + * + * A watchdog keep-alive interval lives in the main event loop of the process + * that runs it: if that loop is ever blocked (frozen server, cf. the cold + * /v1/models rebuild freeze), the pings stop and systemd kills the service + * after WatchdogSec=. + */ + +import { spawn } from "node:child_process"; + +export const SD_NOTIFY_BINARY = "systemd-notify"; +export const SD_NOTIFY_SOCKET_ENV = "NOTIFY_SOCKET"; +export const SD_NOTIFY_DISABLE_ENV = "OMNIROUTE_DISABLE_SD_NOTIFY"; +// Ping every 60s — satisfies any systemd WatchdogSec= >= 120s (systemd +// requires keep-alive pings at most every WatchdogSec/2). +export const SD_NOTIFY_WATCHDOG_INTERVAL_MS = 60_000; + +export function isSystemdNotifyEnabled(env = process.env) { + return Boolean(env[SD_NOTIFY_SOCKET_ENV]) && env[SD_NOTIFY_DISABLE_ENV] !== "1"; +} + +export function buildNotifyMessage(kind) { + switch (kind) { + case "ready": + return "READY=1"; + case "watchdog": + return "WATCHDOG=1"; + case "stopping": + return "STOPPING=1"; + default: + throw new Error(`[omniroute][sd_notify] unknown message kind: ${kind}`); + } +} + +export function createSystemdNotifier({ + env = process.env, + binary = SD_NOTIFY_BINARY, + watchdogIntervalMs = SD_NOTIFY_WATCHDOG_INTERVAL_MS, + spawnFn = spawn, + onWarn = (message) => console.warn(message), +} = {}) { + const enabled = isSystemdNotifyEnabled(env); + let disabled = false; + let watchdogTimer = null; + + const send = (kind) => { + if (!enabled || disabled) return; + const child = spawnFn(binary, [buildNotifyMessage(kind)], { env, stdio: "ignore" }); + // Never let a hung systemd-notify keep the process alive. + child.unref?.(); + child.on("error", (err) => { + // A failed send means systemd never sees the keep-alive: the service + // would be killed as unhealthy anyway, so disabling loudly (one + // warning) is safer than spamming errors forever. + disabled = true; + if (watchdogTimer) { + clearInterval(watchdogTimer); + watchdogTimer = null; + } + onWarn( + `[omniroute][sd_notify] failed to send '${kind}' (${err?.code ?? err?.message ?? err}); sd_notify disabled for this process` + ); + }); + }; + + return { + enabled, + ready() { + send("ready"); + }, + watchdog() { + send("watchdog"); + }, + stopping() { + send("stopping"); + }, + startWatchdog() { + if (!enabled || disabled || watchdogTimer) return; + watchdogTimer = setInterval(() => send("watchdog"), watchdogIntervalMs); + watchdogTimer.unref?.(); + }, + dispose() { + if (watchdogTimer) { + clearInterval(watchdogTimer); + watchdogTimer = null; + } + }, + }; +} diff --git a/tests/unit/cli/autostart-linux.test.ts b/tests/unit/cli/autostart-linux.test.ts index 4e337652f9..26bb63de20 100644 --- a/tests/unit/cli/autostart-linux.test.ts +++ b/tests/unit/cli/autostart-linux.test.ts @@ -101,3 +101,41 @@ test("Linux enable path prefers graphical desktop autostart over systemd", () => assert.ok(systemdBranch > -1, "expected a systemd fallback branch"); assert.ok(graphicalBranch < systemdBranch, "graphical autostart should be preferred"); }); + +test("systemd branch writes Type=notify sd_notify directives (headless, stubs succeed)", async () => { + if (process.platform !== "linux") return; + const stubBin = join(tmpDir, "stub-bin"); + const unitPath = join(tmpDir, ".config", "systemd", "user", "omniroute.service"); + const envKeys = ["DISPLAY", "WAYLAND_DISPLAY", "XDG_CURRENT_DESKTOP"] as const; + const savedEnv: Record = {}; + for (const key of envKeys) savedEnv[key] = process.env[key]; + + try { + for (const key of envKeys) delete process.env[key]; + for (const name of ["systemctl", "loginctl"]) { + writeFileSync(join(stubBin, name), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + } + // autostart.mjs reads process.env / runs the stubs at call time, so a + // cached module is fine. + const { enable } = await import("../../../bin/cli/tray/autostart.mjs"); + + const ok = enable(); + assert.equal(ok, true, "enable() should succeed through the systemd branch"); + assert.ok(existsSync(unitPath), "systemd unit should be written"); + + const unit = readFileSync(unitPath, "utf8"); + assert.match(unit, /Type=notify/); + assert.match(unit, /NotifyAccess=all/); + assert.match(unit, /WatchdogSec=180/); + assert.match(unit, /TimeoutStartSec=300/); + assert.match(unit, /Restart=on-failure/); + } finally { + for (const key of envKeys) { + if (savedEnv[key] === undefined) delete process.env[key]; + else process.env[key] = savedEnv[key]; + } + for (const name of ["systemctl", "loginctl"]) { + writeFileSync(join(stubBin, name), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + } + } +}); diff --git a/tests/unit/pack-artifact-policy.test.ts b/tests/unit/pack-artifact-policy.test.ts index 071c72cdbb..857a834ad3 100644 --- a/tests/unit/pack-artifact-policy.test.ts +++ b/tests/unit/pack-artifact-policy.test.ts @@ -223,6 +223,7 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball", "dist/responses-ws-proxy.mjs", "dist/server-ws.mjs", "dist/src/lib/usage/callLogArtifactWorker.js", + "dist/systemd-notify.mjs", "dist/tls-options.mjs", "dist/webdav-handler.mjs", "scripts/build/colocateOptionals.mjs", diff --git a/tests/unit/systemd-notify.test.mjs b/tests/unit/systemd-notify.test.mjs new file mode 100644 index 0000000000..cb9e441f8f --- /dev/null +++ b/tests/unit/systemd-notify.test.mjs @@ -0,0 +1,258 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawn, spawnSync } from "node:child_process"; + +import { + buildNotifyMessage, + createSystemdNotifier, + isSystemdNotifyEnabled, + SD_NOTIFY_WATCHDOG_INTERVAL_MS, +} from "../../scripts/dev/systemd-notify.mjs"; + +function fakeSpawn(recorder) { + return (binary, args, options) => { + const child = new EventEmitter(); + recorder.push({ binary, args, options, child }); + return child; + }; +} + +function systemdNotifyAvailable() { + try { + return spawnSync("systemd-notify", ["--version"], { stdio: "ignore" }).status === 0; + } catch { + return false; + } +} + +function python3Available() { + try { + return spawnSync("python3", ["--version"], { stdio: "ignore" }).status === 0; + } catch { + return false; + } +} + +// Waits for the listener to emit `expected` lines (in order), then resolves +// with everything it saw. Fails loudly on timeout or premature exit. +// BARRIER=1 datagrams (sd_notify synchronization emitted by the systemd-notify +// CLI after every message) are noise for this contract and are skipped. +function waitForLines(child, expected, timeoutMs) { + return new Promise((resolve, reject) => { + let buf = ""; + const seen = []; + const timer = setTimeout( + () => reject(new Error(`listener timed out after ${timeoutMs}ms; got: ${seen.join(", ")}`)), + timeoutMs + ); + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + buf += chunk; + let idx; + while ((idx = buf.indexOf("\n")) >= 0) { + const line = buf.slice(0, idx).trim(); + buf = buf.slice(idx + 1); + if (!line || line === "BARRIER=1") continue; + seen.push(line); + if (seen.length === expected.length) { + clearTimeout(timer); + resolve([...seen]); + } + } + }); + child.on("exit", () => { + if (seen.length < expected.length) { + clearTimeout(timer); + reject(new Error(`listener exited early; got: ${seen.join(", ")}`)); + } + }); + }); +} + +test("isSystemdNotifyEnabled: false without NOTIFY_SOCKET", () => { + assert.equal(isSystemdNotifyEnabled({}), false); + assert.equal(isSystemdNotifyEnabled({ OMNIROUTE_DISABLE_SD_NOTIFY: "0" }), false); +}); + +test("isSystemdNotifyEnabled: true when NOTIFY_SOCKET is set", () => { + assert.equal(isSystemdNotifyEnabled({ NOTIFY_SOCKET: "/run/systemd/notify" }), true); +}); + +test("isSystemdNotifyEnabled: opt-out OMNIROUTE_DISABLE_SD_NOTIFY=1 wins", () => { + assert.equal( + isSystemdNotifyEnabled({ + NOTIFY_SOCKET: "/run/systemd/notify", + OMNIROUTE_DISABLE_SD_NOTIFY: "1", + }), + false + ); +}); + +test("notifier honors the opt-out (no spawn, no timer)", async () => { + const calls = []; + const notifier = createSystemdNotifier({ + env: { NOTIFY_SOCKET: "/run/systemd/notify", OMNIROUTE_DISABLE_SD_NOTIFY: "1" }, + spawnFn: fakeSpawn(calls), + }); + assert.equal(notifier.enabled, false); + notifier.ready(); + notifier.startWatchdog(); + await new Promise((resolve) => setTimeout(resolve, 15)); + assert.deepEqual(calls, []); + notifier.dispose(); +}); + +// Contract with the generated systemd unit (bin/cli/tray/autostart.mjs): +// systemd requires keep-alive pings at most every WatchdogSec/2, and the +// generated unit pins WatchdogSec=180. If either side drifts, this fails +// loudly instead of silently breaking the watchdog. +test("watchdog cadence satisfies the generated unit's WatchdogSec", () => { + const generatedWatchdogSec = 180; + assert.ok( + 2 * SD_NOTIFY_WATCHDOG_INTERVAL_MS <= generatedWatchdogSec * 1000, + `ping interval ${SD_NOTIFY_WATCHDOG_INTERVAL_MS}ms must be <= WatchdogSec/2 (${generatedWatchdogSec / 2}s)` + ); +}); + +test("buildNotifyMessage: known kinds", () => { + assert.equal(buildNotifyMessage("ready"), "READY=1"); + assert.equal(buildNotifyMessage("watchdog"), "WATCHDOG=1"); + assert.equal(buildNotifyMessage("stopping"), "STOPPING=1"); +}); + +test("buildNotifyMessage: unknown kind throws", () => { + assert.throws(() => buildNotifyMessage("bogus"), /unknown message kind/); +}); + +test("disabled notifier is a complete no-op (no spawn, no timer)", async () => { + const calls = []; + const notifier = createSystemdNotifier({ env: {}, spawnFn: fakeSpawn(calls) }); + assert.equal(notifier.enabled, false); + notifier.ready(); + notifier.watchdog(); + notifier.stopping(); + notifier.startWatchdog(); + await new Promise((resolve) => setTimeout(resolve, 15)); + assert.deepEqual(calls, []); + notifier.dispose(); +}); + +test("enabled notifier sends the right message per kind", () => { + const calls = []; + const env = { NOTIFY_SOCKET: "/run/systemd/notify" }; + const notifier = createSystemdNotifier({ env, spawnFn: fakeSpawn(calls) }); + assert.equal(notifier.enabled, true); + notifier.ready(); + notifier.watchdog(); + notifier.stopping(); + assert.deepEqual( + calls.map((c) => c.args), + [["READY=1"], ["WATCHDOG=1"], ["STOPPING=1"]] + ); + calls.forEach((c) => { + assert.equal(c.binary, "systemd-notify"); + assert.equal(c.options.env, env); + assert.deepEqual(c.options.stdio, "ignore"); + }); +}); + +test("spawn failure disables the notifier once and stops the watchdog", async () => { + const calls = []; + const warnings = []; + const env = { NOTIFY_SOCKET: "/run/systemd/notify" }; + const notifier = createSystemdNotifier({ + env, + spawnFn: fakeSpawn(calls), + onWarn: (m) => warnings.push(m), + }); + notifier.ready(); + calls[0].child.emit("error", { code: "ENOENT" }); + assert.equal(notifier.enabled, true); // env guard unchanged + notifier.watchdog(); + notifier.stopping(); + notifier.startWatchdog(); + assert.equal(calls.length, 1, "no further spawn after disable"); + assert.equal(warnings.length, 1, "warning emitted exactly once"); + assert.match(warnings[0], /ENOENT/); + notifier.dispose(); +}); + +test("watchdog interval pings repeatedly and dispose() stops it", async () => { + const calls = []; + const env = { NOTIFY_SOCKET: "/run/systemd/notify" }; + // Interval is intentionally tiny (5ms) with a generous wait window (150ms — 30x the + // interval) so this stays deterministic under CI/CPU-contended runs: only 2 pings are + // required, well within reach even at >10x scheduler jitter. + const notifier = createSystemdNotifier({ + env, + watchdogIntervalMs: 5, + spawnFn: fakeSpawn(calls), + }); + notifier.ready(); + notifier.startWatchdog(); + await new Promise((resolve) => setTimeout(resolve, 150)); + notifier.dispose(); + const watchdogCalls = calls.filter((c) => c.args[0] === "WATCHDOG=1").length; + assert.ok(watchdogCalls >= 2, `expected >= 2 watchdog pings, got ${watchdogCalls}`); + const before = calls.length; + await new Promise((resolve) => setTimeout(resolve, 30)); + assert.equal(calls.length, before, "no ping after dispose"); +}); + +// End-to-end handshake against the real `systemd-notify` binary: a fake +// NOTIFY_SOCKET (AF_UNIX datagram listener on a temp path, hosted by a tiny +// python3 helper — node:dgram only supports udp4/udp6) receives the exact +// datagrams a real systemd would see. Skipped where the binary or python3 is +// absent (Windows runners, minimal containers) — the spawn-level contract +// above still applies there. +const PY_DGRAM_LISTENER = ` +import socket, sys, os +s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) +p = sys.argv[1] +try: + os.unlink(p) +except OSError: + pass +s.bind(p) +print("LISTENING", flush=True) +while True: + try: + print(s.recv(4096).decode(), flush=True) + except OSError: + break +`; + +test( + "real handshake: systemd-notify delivers READY=1/WATCHDOG=1/STOPPING=1 over a fake NOTIFY_SOCKET", + { + skip: + (!systemdNotifyAvailable() || !python3Available()) && + "systemd-notify and/or python3 unavailable", + }, + async () => { + const sockPath = path.join(os.tmpdir(), `omniroute-sdnotify-${process.pid}.sock`); + const listener = spawn("python3", ["-c", PY_DGRAM_LISTENER, sockPath], { + stdio: ["ignore", "pipe", "ignore"], + }); + try { + await waitForLines(listener, ["LISTENING"], 5000); + const notifier = createSystemdNotifier({ + env: { NOTIFY_SOCKET: sockPath, PATH: process.env.PATH }, + }); + assert.equal(notifier.enabled, true); + notifier.ready(); + notifier.watchdog(); + notifier.stopping(); + const received = await waitForLines(listener, ["READY=1", "WATCHDOG=1", "STOPPING=1"], 10000); + assert.deepEqual(received, ["READY=1", "WATCHDOG=1", "STOPPING=1"]); + notifier.dispose(); + } finally { + listener.kill(); + fs.rmSync(sockPath, { force: true }); + } + } +);