feat(server): native systemd sd_notify watchdog (Type=notify) (#10662)

Merged — validated together with a batch of related maxmad64bis PRs in one combined worktree (typecheck:core clean, complexity/cognitive-complexity/file-size/changelog gates green, focused tests passing). Thanks for the contribution!

scripts/build/assembleStandalone.mjs

@@ -183,6 +183,11 @@ const EXTRA_MODULE_ENTRIES = [
     src: ["scripts", "dev", "main-server-timeouts.mjs"],
     dest: ["main-server-timeouts.mjs"],
   },
+  {
+    label: "systemd sd_notify helper (server-ws.mjs dependency)",
+    src: ["scripts", "dev", "systemd-notify.mjs"],
+    dest: ["systemd-notify.mjs"],
+  },
   {
     label: "HTTP method guard (server-ws.mjs dependency)",
     src: ["scripts", "dev", "http-method-guard.cjs"],

scripts/build/pack-artifact-policy.ts

@@ -49,6 +49,9 @@ export const APP_STAGING_ALLOWED_EXACT_PATHS: string[] = [
   "package.json",
   "peer-stamp.mjs",
   "main-server-timeouts.mjs",
+  // server-ws.mjs import (sd_notify helper) — enforced by the closure test
+  // tests/unit/pack-artifact-server-ws-closure.test.ts.
+  "systemd-notify.mjs",
   "responses-ws-proxy.mjs",
   "bin/chatgpt-web-codex-mcp.mjs",
   "scripts/dev/sync-env.mjs",
@@ -184,6 +187,8 @@ export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [
   "dist/responses-ws-proxy.mjs",
   "dist/peer-stamp.mjs",
   "dist/main-server-timeouts.mjs",
+  // server-ws.mjs import (sd_notify helper) — enforced by the closure test.
+  "dist/systemd-notify.mjs",
   "dist/http-method-guard.cjs",
   // #5452: regression guard — make check:pack-artifact fail loudly if the TLS
   // opt-in sidecar (imported by dist/server-ws.mjs) ever vanishes from the tarball.

scripts/check/check-env-doc-sync.mjs

@@ -61,6 +61,10 @@ const IGNORE_FROM_CODE = new Set([
   "APPDATA",
   "LOCALAPPDATA",
   "XDG_CONFIG_HOME",
+  // systemd-injected notify socket path (sd_notify protocol, see
+  // scripts/dev/systemd-notify.mjs) — set by systemd only when running under
+  // a unit, never user config.
+  "NOTIFY_SOCKET",
   // XDG Base Directory cache root — read (never defined by OmniRoute) so the
   // Android/Termux serve path can honor an operator-set cache location (#8519).
   "XDG_CACHE_HOME",

scripts/dev/run-next.mjs

@@ -15,6 +15,7 @@ import { ensureNativeSqlite } from "./ensure-native-sqlite.mjs";
 import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopackCacheHeal.mjs";
 import { randomUUID } from "node:crypto";
 import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
+import { createSystemdNotifier } from "./systemd-notify.mjs";
 
 const { maybeHandleDisallowedMethod } = methodGuard;
 const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard;
@@ -60,6 +61,13 @@ for (const [key, value] of Object.entries(mergedEnv)) {
   }
 }
 
+// systemd sd_notify (Type=notify / WatchdogSec=): this process owns the
+// watchdog pings — if its event loop blocks (freeze), the pings stop and
+// systemd kills the service. No-op outside systemd (no NOTIFY_SOCKET).
+// Created AFTER .env is merged so the OMNIROUTE_DISABLE_SD_NOTIFY opt-out
+// documented in .env is honored on this path too.
+const systemdNotifier = createSystemdNotifier();
+
 // The mergedEnv copy above pulls NODE_ENV straight from `.env` — and the shipped
 // `.env.example` default is `NODE_ENV=production`. Next's programmatic `next()`
 // entry (unlike the `next` CLI) trusts that value verbatim, so `npm run dev`
@@ -184,6 +192,7 @@ async function start() {
   });
 
   const shutdown = async (signal) => {
+    systemdNotifier.stopping();
     try {
       await new Promise((resolve) => server.close(resolve));
       await nextApp.close();
@@ -202,6 +211,8 @@ async function start() {
     console.log(
       `[Next] ${mode} server listening on http://${hostname}:${dashboardPort} (${bundler})`
     );
+    systemdNotifier.ready();
+    systemdNotifier.startWatchdog();
   });
 }
 

scripts/dev/standalone-server-ws.mjs

@@ -8,6 +8,20 @@ import methodGuard from "./http-method-guard.cjs";
 import headResponseGuard from "./head-response-guard.cjs";
 import { resolveTlsOptions, createServerListener } from "./tls-options.mjs";
 import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs";
+import { createSystemdNotifier } from "./systemd-notify.mjs";
+
+// systemd sd_notify (Type=notify / WatchdogSec=): this process is the one
+// whose event loop can freeze (cold /v1/models rebuild), so it must own the
+// watchdog pings — a blocked loop stops the pings and systemd kills the
+// service. No-op outside systemd (no NOTIFY_SOCKET).
+const systemdNotifier = createSystemdNotifier();
+let systemdReadySent = false;
+// NOTE: if an operator sets NEXT_MANUAL_SIG_HANDLE=1, Next never registers its
+// own signal cleanup and these once() handlers would suppress Node's default
+// signal exit (process lingers until systemd's stop-timeout SIGKILL). Nothing
+// in this repo sets that var; acceptable, documented behavior.
+process.once("SIGINT", () => systemdNotifier.stopping());
+process.once("SIGTERM", () => systemdNotifier.stopping());
 
 const originalCreateServer = http.createServer.bind(http);
 const proxiesByPort = new Map();
@@ -209,6 +223,15 @@ http.createServer = function createServerWithResponsesWs(...args) {
     return originalAddListener(eventName, listener);
   };
 
+  // sd_notify READY once the main listener is actually accepting, then arm
+  // the watchdog keep-alive interval (unref'd — never keeps the process up).
+  server.once("listening", () => {
+    if (systemdReadySent) return;
+    systemdReadySent = true;
+    systemdNotifier.ready();
+    systemdNotifier.startWatchdog();
+  });
+
   return server;
 };
 

scripts/dev/systemd-notify.mjs

@@ -0,0 +1,98 @@
+/**
+ * Minimal systemd sd_notify integration (sd_notify(3) protocol).
+ *
+ * Node's stable API has no AF_UNIX datagram socket support (node:dgram is
+ * udp4/udp6 only), so notifications are sent by spawning the `systemd-notify`
+ * binary — present on every systemd host, no extra dependency.
+ *
+ * Everything is guarded: without a NOTIFY_SOCKET (plain terminal, Docker,
+ * Electron, Windows) the notifier is a no-op and costs nothing. Set
+ * OMNIROUTE_DISABLE_SD_NOTIFY=1 to force-disable even under systemd.
+ *
+ * A watchdog keep-alive interval lives in the main event loop of the process
+ * that runs it: if that loop is ever blocked (frozen server, cf. the cold
+ * /v1/models rebuild freeze), the pings stop and systemd kills the service
+ * after WatchdogSec=.
+ */
+
+import { spawn } from "node:child_process";
+
+export const SD_NOTIFY_BINARY = "systemd-notify";
+export const SD_NOTIFY_SOCKET_ENV = "NOTIFY_SOCKET";
+export const SD_NOTIFY_DISABLE_ENV = "OMNIROUTE_DISABLE_SD_NOTIFY";
+// Ping every 60s — satisfies any systemd WatchdogSec= >= 120s (systemd
+// requires keep-alive pings at most every WatchdogSec/2).
+export const SD_NOTIFY_WATCHDOG_INTERVAL_MS = 60_000;
+
+export function isSystemdNotifyEnabled(env = process.env) {
+  return Boolean(env[SD_NOTIFY_SOCKET_ENV]) && env[SD_NOTIFY_DISABLE_ENV] !== "1";
+}
+
+export function buildNotifyMessage(kind) {
+  switch (kind) {
+    case "ready":
+      return "READY=1";
+    case "watchdog":
+      return "WATCHDOG=1";
+    case "stopping":
+      return "STOPPING=1";
+    default:
+      throw new Error(`[omniroute][sd_notify] unknown message kind: ${kind}`);
+  }
+}
+
+export function createSystemdNotifier({
+  env = process.env,
+  binary = SD_NOTIFY_BINARY,
+  watchdogIntervalMs = SD_NOTIFY_WATCHDOG_INTERVAL_MS,
+  spawnFn = spawn,
+  onWarn = (message) => console.warn(message),
+} = {}) {
+  const enabled = isSystemdNotifyEnabled(env);
+  let disabled = false;
+  let watchdogTimer = null;
+
+  const send = (kind) => {
+    if (!enabled || disabled) return;
+    const child = spawnFn(binary, [buildNotifyMessage(kind)], { env, stdio: "ignore" });
+    // Never let a hung systemd-notify keep the process alive.
+    child.unref?.();
+    child.on("error", (err) => {
+      // A failed send means systemd never sees the keep-alive: the service
+      // would be killed as unhealthy anyway, so disabling loudly (one
+      // warning) is safer than spamming errors forever.
+      disabled = true;
+      if (watchdogTimer) {
+        clearInterval(watchdogTimer);
+        watchdogTimer = null;
+      }
+      onWarn(
+        `[omniroute][sd_notify] failed to send '${kind}' (${err?.code ?? err?.message ?? err}); sd_notify disabled for this process`
+      );
+    });
+  };
+
+  return {
+    enabled,
+    ready() {
+      send("ready");
+    },
+    watchdog() {
+      send("watchdog");
+    },
+    stopping() {
+      send("stopping");
+    },
+    startWatchdog() {
+      if (!enabled || disabled || watchdogTimer) return;
+      watchdogTimer = setInterval(() => send("watchdog"), watchdogIntervalMs);
+      watchdogTimer.unref?.();
+    },
+    dispose() {
+      if (watchdogTimer) {
+        clearInterval(watchdogTimer);
+        watchdogTimer = null;
+      }
+    },
+  };
+}