diff --git a/scripts/dev/httpClientAbortGuard.mjs b/scripts/dev/httpClientAbortGuard.mjs new file mode 100644 index 0000000000..9fdabf7a61 --- /dev/null +++ b/scripts/dev/httpClientAbortGuard.mjs @@ -0,0 +1,17 @@ +"use strict"; + +/** + * Re-export of the shared client-abort guard so the Node-only dev server + * (`run-next.mjs`) keeps importing from its original relative path. The real + * implementation lives in `src/shared/utils/httpClientAbortGuard.mjs` (importable + * from both `.mjs` and the TypeScript servers under `src/`, tsconfig allowJs). + * + * @module + */ + +export { + isClientAbortError, + shouldSwallowUncaught, + attachRequestStreamGuards, + installProcessCrashGuard, +} from "../../src/shared/utils/httpClientAbortGuard.mjs"; diff --git a/scripts/dev/run-next.mjs b/scripts/dev/run-next.mjs index 8282a88d0c..b22150978f 100644 --- a/scripts/dev/run-next.mjs +++ b/scripts/dev/run-next.mjs @@ -16,6 +16,10 @@ import { isTurbopackCacheCorruption, purgeAllTurbopackCaches } from "./turbopack import { randomUUID } from "node:crypto"; import { getMainServerTimeoutConfig } from "./main-server-timeouts.mjs"; import { createSystemdNotifier } from "./systemd-notify.mjs"; +import { + attachRequestStreamGuards, + installProcessCrashGuard, +} from "./httpClientAbortGuard.mjs"; const { maybeHandleDisallowedMethod } = methodGuard; const { wrapRequestListenerWithHeadResponseGuard } = headResponseGuard; @@ -162,6 +166,13 @@ async function prepareWithHeal() { } async function start() { + // Safety net: a client aborting a connection (browser navigation, HMR reconnect, + // Back/Forward cache) can emit `Error: aborted`/`ECONNRESET` on the request + // stream. Without this the single missed listener becomes an uncaughtException + // that takes the whole server down — surfacing as a wall of ERR_CONNECTION_REFUSED + // after login. Benign aborts are swallowed; genuine errors still crash loudly. + installProcessCrashGuard(); + await prepareWithHeal(); const requestHandler = nextApp.getRequestHandler(); @@ -176,6 +187,10 @@ async function start() { const server = http.createServer( wrapRequestListenerWithHeadResponseGuard((req, res) => { + // Absorb client-abort errors (browser closes the socket during + // navigation/HMR/bfcache) on the request/response streams so they never + // surface as an uncaughtException that kills the whole server (#fix-dev-server-aborted). + attachRequestStreamGuards(req, res); if (maybeHandleDisallowedMethod(req, res)) return; // Stamp the real TCP peer IP before Next sees the request, so the authz // middleware can decide LOCAL_ONLY locality without trusting the Host header. diff --git a/src/lib/apiBridgeServer.ts b/src/lib/apiBridgeServer.ts index 4109dfd6ac..6f2f2e6cdc 100644 --- a/src/lib/apiBridgeServer.ts +++ b/src/lib/apiBridgeServer.ts @@ -3,6 +3,10 @@ import type { IncomingMessage, ServerResponse } from "http"; import net from "net"; import { getRuntimePorts } from "@/lib/runtime/ports"; import { getApiBridgeTimeoutConfig } from "@/shared/utils/runtimeTimeouts"; +import { + attachRequestStreamGuards, + installProcessCrashGuard, +} from "@/shared/utils/httpClientAbortGuard"; const API_BRIDGE_TIMEOUTS = getApiBridgeTimeoutConfig(process.env, (message) => { console.warn(`[API Bridge] ${message}`); @@ -169,6 +173,11 @@ declare global { } export function initApiBridgeServer(): void { + // Safety net: a client aborting a connection can emit `Error: aborted`/ + // ECONNRESET on the request stream; without this the single missed listener + // becomes an uncaughtException that kills the server. Benign aborts are + // swallowed; genuine errors still crash loudly (#fix-dev-server-aborted). + installProcessCrashGuard(); if (globalThis.__omnirouteApiBridgeStarted) return; const { apiPort, dashboardPort } = getRuntimePorts(); @@ -177,6 +186,10 @@ export function initApiBridgeServer(): void { const host = process.env.API_HOST || "127.0.0.1"; const server = http.createServer((req, res) => { + // Absorb client-abort errors (browser closes the socket during navigation/ + // HMR/bfcache) on the request/response streams so they never surface as an + // uncaughtException that kills the server (#fix-dev-server-aborted). + attachRequestStreamGuards(req, res); const rawUrl = req.url || "/"; const pathname = rawUrl.split("?")[0] || "/"; diff --git a/src/lib/services/embedWsProxy.ts b/src/lib/services/embedWsProxy.ts index b2b1e10370..89bdb3d1a5 100644 --- a/src/lib/services/embedWsProxy.ts +++ b/src/lib/services/embedWsProxy.ts @@ -27,6 +27,10 @@ import type { IncomingMessage } from "node:http"; import { getSupervisor } from "./registry"; import { getOrCreateApiKey } from "./apiKey"; +import { + attachRequestStreamGuards, + installProcessCrashGuard, +} from "@/shared/utils/httpClientAbortGuard"; const DEFAULT_HOST = "127.0.0.1"; const DEFAULT_PORT = 20131; @@ -242,12 +246,21 @@ export function resolveEmbedWsHost(): string { * Idempotent — safe to call multiple times. */ export function initEmbedWsProxy(): void { + // Safety net: a client aborting a connection can emit `Error: aborted`/ + // ECONNRESET on the request stream; without this the single missed listener + // becomes an uncaughtException that kills the server. Benign aborts are + // swallowed; genuine errors still crash loudly (#fix-dev-server-aborted). + installProcessCrashGuard(); if (globalThis.__omnirouteEmbedWsStarted) return; const host = resolveEmbedWsHost(); const port = parseInt(process.env.EMBED_WS_PROXY_PORT ?? String(DEFAULT_PORT), 10); const server = http.createServer((_req, res) => { + // Absorb client-abort errors (browser closes the socket during navigation/ + // HMR/bfcache) on the request/response streams so they never surface as an + // uncaughtException that kills the server (#fix-dev-server-aborted). + attachRequestStreamGuards(_req, res); res.writeHead(426, "Upgrade Required", { "content-type": "application/json" }); res.end(JSON.stringify({ error: "upgrade_required", message: "Use WebSocket." })); }); diff --git a/src/server/ws/liveServer.ts b/src/server/ws/liveServer.ts index 4c2287fa20..254e403fb7 100644 --- a/src/server/ws/liveServer.ts +++ b/src/server/ws/liveServer.ts @@ -33,6 +33,11 @@ import type { DashboardEventName, DashboardEventMap, DashboardChannel } from "@/ import { CHANNEL_EVENTS, getChannelForEvent } from "@/lib/events/types"; import { isAutomatedTestProcess, isBuildProcess } from "@/shared/utils/testProcess"; +import { + attachRequestStreamGuards, + installProcessCrashGuard, +} from "@/shared/utils/httpClientAbortGuard"; + import { buildAllowedOrigins, buildAllowedHosts, @@ -460,6 +465,11 @@ export async function startLiveDashboardServer( port = DEFAULT_PORT, host = DEFAULT_HOST ): Promise { + // Safety net: a client aborting a connection can emit `Error: aborted`/ + // ECONNRESET on the request stream; without this the single missed listener + // becomes an uncaughtException that kills the server. Benign aborts are + // swallowed; genuine errors still crash loudly (#fix-dev-server-aborted). + installProcessCrashGuard(); if (!process.env.JWT_SECRET) { console.warn( " \x1b[33m⚠ Warning: JWT_SECRET is not set in the environment.\x1b[0m\n" + @@ -469,6 +479,10 @@ export async function startLiveDashboardServer( } const server = createServer((req, res) => { + // Absorb client-abort errors (browser closes the socket during navigation/ + // HMR/bfcache) on the request/response streams so they never surface as an + // uncaughtException that kills the server (#fix-dev-server-aborted). + attachRequestStreamGuards(req, res); handleInternalEventRequest(req, res); }); const wss = new WebSocketServer({ server }); diff --git a/src/shared/utils/httpClientAbortGuard.mjs b/src/shared/utils/httpClientAbortGuard.mjs new file mode 100644 index 0000000000..ff2fb43ced --- /dev/null +++ b/src/shared/utils/httpClientAbortGuard.mjs @@ -0,0 +1,141 @@ +"use strict"; + +/** + * HTTP client-abort crash guard (#fix-dev-server-aborted). + * + * Node's http.Server turns an 'error' event on an IncomingMessage/ServerResponse + * into an uncaughtException (and therefore a process exit) WHENEVER the emitter + * has no listener. The single most common such error is a *client* abort: the + * browser closes the TCP socket (navigation, Back/Forward cache, HMR reconnect, + * cancelling a fetch) while the server is still streaming the response. Node + * emits `Error: aborted` / `ERR_STREAM_PREMATURE_CLOSE` / `ECONNRESET` on the + * request stream, and absent a handler it kills the whole server process. + * + * That surfaced as "login succeeds, then the dashboard hangs with a wall of + * `net::ERR_CONNECTION_REFUSED`": after auth the SPA opens many polling + * connections + a live WebSocket; stray client-side socket closes during + * navigation/HMR were taking the dev server down. + * + * Two layers: + * 1. `attachRequestStreamGuards(req, res)` — per-request listeners that absorb + * client-abort errors so they never bubble to the process level. Call it + * inside every `http.createServer((req, res) => …)` request listener. + * 2. `installProcessCrashGuard()` — a last-resort safety net on + * `process.on('uncaughtException' | 'unhandledRejection')` that swallows + * the same benign client-abort errors but otherwise preserves the existing + * crash semantics (so genuine bugs still surface). Idempotent. + * + * Kept as a `.mjs` module (no build step) so it is importable both from the + * Node-only dev server (`scripts/dev/run-next.mjs`) and from the TypeScript + * servers under `src/` (tsconfig `allowJs: true`). + * + * @module + */ + +/** + * @param {unknown} err + * @returns {boolean} true when `err` represents a client closing the + * connection rather than a server-side fault. + */ +export function isClientAbortError(err) { + if (!err || typeof err !== "object") return false; + const e = /** @type {NodeJS.ErrnoException} */ (err); + // Node emits `Error: aborted` (no code) from http.Server#abortIncoming. + if (e.message === "aborted" || e.message === "Aborted") return true; + switch (e.code) { + case "ERR_STREAM_PREMATURE_CLOSE": + case "ECONNRESET": + case "EPIPE": + case "ECONNABORTED": + case "ETIMEDOUT": + case "ENOTCONN": + case "ECANCELED": + return true; + default: + return false; + } +} + +/** + * Decide whether a process-level uncaughtException/unhandledRejection should be + * swallowed (benign client-abort) or allowed to surface (genuine bug). + * + * Pure + exported so it can be unit-tested without poking process listeners. + * + * @param {unknown} err + * @param {string | undefined} origin Node's uncaughtException origin (e.g. + * "uncaughtException" / "unhandledRejection"); absent/empty for rejections. + * @returns {boolean} true => swallow (log only), false => re-throw / let crash. + */ +export function shouldSwallowUncaught(err, origin) { + if (!isClientAbortError(err)) return false; + // Only swallow when the origin matches what the guard installed for. If some + // other subsystem raised it (e.g. a deliberate `throw` in a domain), keep the + // existing crash semantics. + return !origin || origin === "uncaughtException" || origin === "unhandledRejection"; +} + +/** + * Attach `error` listeners to a request/response pair that swallow client-abort + * errors. Idempotent per (req, res) pair via a Symbol flag. + * + * @param {import("node:http").IncomingMessage} req + * @param {import("node:http").ServerResponse} res + */ +export function attachRequestStreamGuards(req, res) { + const flag = Symbol.for("omniroute.requestAbortGuard"); + if (req[flag] || res[flag]) return; + req[flag] = true; + res[flag] = true; + + req.on("error", (err) => { + if (!isClientAbortError(err)) { + // Re-emit a genuine request error through the normal channel so it is + // still observable in logs, but never as an uncaughtException. + console.error("[server] request stream error:", err); + } + }); + + res.on("error", (err) => { + if (!isClientAbortError(err)) { + console.error("[server] response stream error:", err); + } + }); +} + +let crashGuardInstalled = false; + +/** + * Install process-level safety nets. Idempotent. Benign client-abort errors are + * logged once and swallowed; everything else is re-thrown on a fresh stack so + * the process keeps its current crash semantics (genuine bugs still crash/hang + * loudly, and a supervisor or test harness sees them). + * + * @param {(level: "warn" | "error", ...args: unknown[]) => void} [log] + */ +export function installProcessCrashGuard(log) { + if (crashGuardInstalled) return; + crashGuardInstalled = true; + + const logger = log ?? console; + + process.on("uncaughtException", (err, origin) => { + if (shouldSwallowUncaught(err, origin)) { + logger("warn", "[server] swallowed client-abort uncaughtException:", err?.message ?? err); + return; + } + throw err; + }); + + process.on("unhandledRejection", (reason) => { + if (shouldSwallowUncaught(reason, "unhandledRejection")) { + logger( + "warn", + "[server] swallowed client-abort unhandledRejection:", + reason?.message ?? reason + ); + return; + } + throw reason; + }); +} diff --git a/tests/unit/httpClientAbortGuard.test.mjs b/tests/unit/httpClientAbortGuard.test.mjs new file mode 100644 index 0000000000..a7044207b0 --- /dev/null +++ b/tests/unit/httpClientAbortGuard.test.mjs @@ -0,0 +1,119 @@ +"use strict"; + +import assert from "node:assert"; +import { test } from "node:test"; +import { EventEmitter } from "node:events"; +import { + isClientAbortError, + shouldSwallowUncaught, + attachRequestStreamGuards, + installProcessCrashGuard, +} from "../../scripts/dev/httpClientAbortGuard.mjs"; +import * as sharedGuard from "../../src/shared/utils/httpClientAbortGuard.mjs"; + +// The dev server imports from scripts/dev/httpClientAbortGuard.mjs, while the +// TypeScript servers (apiBridgeServer, liveServer, embedWsProxy) import from +// src/shared/utils/httpClientAbortGuard.mjs. The scripts/dev copy must be a pure +// re-export of the shared implementation — verify they are the SAME functions +// (single source of truth, no drift). +test("scripts/dev guard re-exports the shared src implementation (single source of truth)", () => { + assert.equal(isClientAbortError, sharedGuard.isClientAbortError); + assert.equal(shouldSwallowUncaught, sharedGuard.shouldSwallowUncaught); + assert.equal(attachRequestStreamGuards, sharedGuard.attachRequestStreamGuards); + assert.equal(installProcessCrashGuard, sharedGuard.installProcessCrashGuard); + // And the shared module exposes everything the TS servers rely on. + for (const name of [ + "isClientAbortError", + "shouldSwallowUncaught", + "attachRequestStreamGuards", + "installProcessCrashGuard", + ]) { + assert.equal(typeof sharedGuard[name], "function", `shared guard must export ${name}`); + } +}); + +// Minimal stand-ins for IncomingMessage / ServerResponse that expose the +// `error` event (Node's http streams are EventEmitters). +function makeReq() { + return new EventEmitter(); +} +function makeRes() { + const res = new EventEmitter(); + res.end = () => res; + res.write = () => true; + return res; +} + +test("isClientAbortError matches the exact production crash signature", () => { + // Reproduces the Node `abortIncoming` error seen in the app log: + // uncaughtException: aborted / Error: aborted (no code) + const aborted = Object.assign(new Error("aborted"), {}); + assert.equal(isClientAbortError(aborted), true, "plain 'aborted' must be absorbed"); + + for (const code of [ + "ECONNRESET", + "EPIPE", + "ERR_STREAM_PREMATURE_CLOSE", + "ECONNABORTED", + "ETIMEDOUT", + "ENOTCONN", + "ECANCELED", + ]) { + const err = Object.assign(new Error(code), { code }); + assert.equal(isClientAbortError(err), true, `${code} must be absorbed`); + } +}); + +test("isClientAbortError rejects genuine server errors", () => { + const real = Object.assign(new Error("boom"), { code: "ENOSPC" }); + assert.equal(isClientAbortError(real), false); + const noCode = new Error("something else entirely"); + assert.equal(isClientAbortError(noCode), false); +}); + +test("attachRequestStreamGuards swallows a client abort on req without throwing", () => { + const req = makeReq(); + const res = makeRes(); + attachRequestStreamGuards(req, res); + + // Must NOT throw / bubble as uncaughtException. + assert.doesNotThrow(() => { + req.emit("error", Object.assign(new Error("aborted"), {})); + res.emit("error", Object.assign(new Error("aborted"), {})); + }); +}); + +test("attachRequestStreamGuards is idempotent (no double listeners / no throw)", () => { + const req = makeReq(); + const res = makeRes(); + attachRequestStreamGuards(req, res); + assert.doesNotThrow(() => attachRequestStreamGuards(req, res)); + // A second abort must also be absorbed quietly. + assert.doesNotThrow(() => { + req.emit("error", Object.assign(new Error("ECONNRESET"), { code: "ECONNRESET" })); + }); +}); + +test("shouldSwallowUncaught absorbs the real 'aborted' uncaughtException signature", () => { + // The exact error Node raises from http.Server#abortIncoming in the log: + // uncaughtException: aborted / Error: aborted (no code) + const abortErr = new Error("aborted"); + assert.equal(shouldSwallowUncaught(abortErr, "uncaughtException"), true); + assert.equal(shouldSwallowUncaught(abortErr, undefined), true); + assert.equal( + shouldSwallowUncaught(Object.assign(new Error("ECONNRESET"), { code: "ECONNRESET" }), "uncaughtException"), + true + ); +}); + +test("shouldSwallowUncaught preserves crash semantics for genuine errors", () => { + const realErr = new Error("genuine failure"); + assert.equal(shouldSwallowUncaught(realErr, "uncaughtException"), false); + const realErr2 = Object.assign(new Error("disk full"), { code: "ENOSPC" }); + assert.equal(shouldSwallowUncaught(realErr2, "uncaughtException"), false); +}); + +test("installProcessCrashGuard does not throw on import and is idempotent", () => { + assert.doesNotThrow(() => installProcessCrashGuard(() => {})); + assert.doesNotThrow(() => installProcessCrashGuard(() => {})); +});