Files
OmniRoute/tests/unit/cli-stop-supervisor-respawn-9455.test.ts
Diego Rodrigues de Sa e Souza 3d4f3e4960 test(infra): retry recursive temp-dir removal instead of failing a shard on ENOTEMPTY (#11966) (#11968)
* test(infra): retry recursive temp-dir removal instead of failing a shard on ENOTEMPTY (#11966)

Two shards on release/v3.8.51 went red in one day with the same signature —
"ENOTEMPTY, Directory not empty: /tmp/omniroute-<test>-XXXXXX" — from
combo-same-provider-cascade (Unit Tests fast-path 4/4, on a PR that touches only
.github/) and auth-policy-embeddings-webfetch-7785 (the 20k-test TIA step). Both pass
alone and on re-run: the cleanup races something still writing into the directory
(SQLite WAL/-shm checkpoint, a worker, the backup) and under a loaded hosted runner
the window opens. 1154 test files do their own cleanup with
fs.rmSync(dir, { recursive: true, force: true }); 57 already asked for retries.

One-shot codemod (scripts/ad-hoc/codemod-rm-maxretries.mjs, kept for the record):
every rm / rmSync / rmdirSync option object with `recursive: true` and no
`maxRetries` gains `maxRetries: 5, retryDelay: 100` — Node itself then retries
ENOTEMPTY/EBUSY/EPERM for up to ~0.5 s before giving up. 2243 call sites in 1292
files under tests/, the shared tests/_setup/isolateDataDir.ts exit hook included.
Only the option object changes: no call site, assertion or import is touched.

Validation: prettier and ESLint (with the frozen suppressions) clean on all 1292
files; a random 20-file sample runs green (quota-redis-store hangs identically on
the untouched tree — it needs a Redis on localhost, an environment matter). The
four unit shards on this PR are the full run.

* fix(quality): let check-forgotten-sibling-tests read a 1,000-file diff

The gate shells out to `git diff` through execFileSync with Node's default 1 MB
maxBuffer; the 1,292-file codemod in this PR is the first diff large enough to
overflow it, and the gate died with `spawnSync git ENOBUFS` before comparing
anything. 64 MB is far above any real PR and costs nothing when unused.
2026-08-29 01:17:40 -03:00

230 lines
8.6 KiB
TypeScript

import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
// Repro for #9455: omniroute stop reports success but supervisor respawns child.
//
// Defect 1: runStopCommand() kills the child ("server") PID but never stops the
// supervisor, which immediately respawns the child. The fix must have stop.mjs
// read the "supervisor" PID file and SIGTERM the supervisor FIRST (its handler
// sets isShuttingDown=true, kills the child, exits cleanly — no respawn).
// Plus serve.mjs must persist the supervisor PID via writePidFile("supervisor", ...).
//
// Defect 2: killByPort() was a no-op on win32 (`if (process.platform === "win32") return;`)
// yet runStopCommand still printed "Server stopped." and returned 0. The fix must
// implement a win32 path using netstat -ano + process.kill.
const ORIGINAL_DATA_DIR = process.env.DATA_DIR;
const ORIGINAL_FETCH = globalThis.fetch;
const ORIGINAL_PLATFORM = process.platform;
type KillByPortDeps = {
platform?: string;
execFileAsync?: (cmd: string, args: string[]) => Promise<{ stdout: string; stderr: string }>;
processKill?: (pid: number, signal: string | number) => boolean;
isPidRunning?: (pid: number) => boolean;
sleep?: (ms: number) => Promise<void>;
};
type KillByPortFn = (port: number, deps?: KillByPortDeps) => Promise<boolean>;
function createTempDataDir() {
return fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-stop-sup-"));
}
function setupDataDir(dataDir: string) {
fs.mkdirSync(path.join(dataDir, "server"), { recursive: true });
fs.mkdirSync(path.join(dataDir, "supervisor"), { recursive: true });
}
function setServerPid(dataDir: string, p: number) {
fs.writeFileSync(path.join(dataDir, "server", ".pid"), String(p), "utf8");
}
function setSupervisorPid(dataDir: string, p: number) {
fs.writeFileSync(path.join(dataDir, "supervisor", ".pid"), String(p), "utf8");
}
async function withEnv(fn: (dataDir: string) => Promise<void>) {
const dataDir = createTempDataDir();
process.env.DATA_DIR = dataDir;
globalThis.fetch = (async () => {
throw new Error("server offline");
}) as typeof fetch;
const origLog = console.log;
const origErr = console.error;
console.log = () => {};
console.error = () => {};
try {
await fn(dataDir);
} finally {
console.log = origLog;
console.error = origErr;
globalThis.fetch = ORIGINAL_FETCH;
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
}
}
// Track process.kill calls so the test can assert which PIDs were signalled
// without touching real processes. PIDs >= 1000000 are treated as alive.
function trackKills() {
const kills: Array<{ pid: number; signal: string | number }> = [];
const origKill = process.kill.bind(process);
type KillFn = (pid: number, signal?: NodeJS.Signals | number) => boolean;
const stub: KillFn = (pid, signal = 0) => {
if (signal === 0) {
return pid >= 1000000 ? true : (origKill(pid, 0), true);
}
if (pid >= 1000000) {
kills.push({ pid, signal: signal as string | number });
return true;
}
try {
origKill(pid, signal as NodeJS.Signals);
kills.push({ pid, signal: signal as string | number });
return true;
} catch {
return false;
}
};
(process as unknown as { kill: KillFn }).kill = stub;
return {
kills,
restore() {
(process as unknown as { kill: KillFn }).kill = origKill as KillFn;
},
};
}
test("Defect 1: stop must SIGTERM the supervisor BEFORE the child so it does not respawn (#9455)", async () => {
await withEnv(async (dataDir) => {
setupDataDir(dataDir);
const SUPERVISOR_PID = 1000123;
const CHILD_PID = 1000456;
setSupervisorPid(dataDir, SUPERVISOR_PID);
setServerPid(dataDir, CHILD_PID);
const tracker = trackKills();
try {
const { runStopCommand } = await import("../../bin/cli/commands/stop.mjs");
await runStopCommand({});
const signalled = tracker.kills.map((k) => k.pid);
assert.ok(
signalled.includes(SUPERVISOR_PID),
`supervisor PID ${SUPERVISOR_PID} must be signalled; got ${JSON.stringify(signalled)}`
);
// Supervisor must be signalled before the child (cascade order).
const supIdx = signalled.indexOf(SUPERVISOR_PID);
const childIdx = signalled.indexOf(CHILD_PID);
if (childIdx !== -1) {
assert.ok(
supIdx < childIdx,
`supervisor must be killed before child (supIdx=${supIdx} childIdx=${childIdx})`
);
}
} finally {
tracker.restore();
}
});
});
test("Defect 1b: pid.mjs SERVICES array must include supervisor so killAllSubprocesses reaches it (#9455)", async () => {
const tmpDir = os.tmpdir() + "/omniroute-sup-pid-" + Date.now();
process.env.DATA_DIR = tmpDir;
try {
const { writePidFile, readPidFile } = await import("../../bin/cli/utils/pid.mjs");
const ok = writePidFile("supervisor", 555555);
assert.equal(ok, true, "writePidFile('supervisor', ...) must succeed");
assert.equal(readPidFile("supervisor"), 555555);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR;
else process.env.DATA_DIR = ORIGINAL_DATA_DIR;
}
const pidSrc = fs.readFileSync(path.join(process.cwd(), "bin/cli/utils/pid.mjs"), "utf8");
assert.ok(
/SERVICES\s*=\s*\[[^\]]*"supervisor"[^\]]*\]/.test(pidSrc),
'pid.mjs SERVICES array must include "supervisor"'
);
});
test("Defect 2: killByPort on win32 must actually kill the port listener via netstat -ano (#9455)", async () => {
const FAKE_WIN_PID = 1000789;
const kills: Array<{ pid: number; signal: string | number }> = [];
const deps = {
platform: "win32",
execFileAsync: async (cmd: string, args: string[]) => {
if (cmd.endsWith("netstat")) {
return {
stdout: ` TCP 0.0.0.0:20128 0.0.0.0:0 LISTENING ${FAKE_WIN_PID}\r\n`,
stderr: "",
};
}
return { stdout: "", stderr: "" };
},
processKill: (p: number, sig: string | number) => {
kills.push({ pid: p, signal: sig });
return true;
},
isPidRunning: (_p: number) => false, // pretend SIGTERM already killed it
sleep: async (_ms: number) => {},
};
const { killByPort } = await import("../../bin/cli/commands/stop.mjs");
const freed = await (killByPort as unknown as KillByPortFn)(20128, deps);
assert.equal(freed, true, "port must be reported free after killing the listener");
assert.ok(
kills.some((k) => k.pid === FAKE_WIN_PID),
`win32 killByPort must signal the netstat PID ${FAKE_WIN_PID}; got ${JSON.stringify(kills)}`
);
});
test("Defect 2b: killByPort on win32 with no listener returns true and signals nothing (#9455)", async () => {
const kills: Array<{ pid: number; signal: string | number }> = [];
const deps = {
platform: "win32",
execFileAsync: async (_cmd: string, _args: string[]) => ({ stdout: "", stderr: "" }),
processKill: (p: number, sig: string | number) => {
kills.push({ pid: p, signal: sig });
return true;
},
isPidRunning: (_p: number) => false,
sleep: async (_ms: number) => {},
};
const { killByPort } = await import("../../bin/cli/commands/stop.mjs");
const freed = await (killByPort as unknown as KillByPortFn)(20128, deps);
assert.equal(freed, true);
assert.equal(kills.length, 0, "no PIDs should be signalled when none are listening");
});
test("netstat parsing: only LISTENING lines matching the exact port are selected (#9455)", async () => {
const stdout = [
" TCP 0.0.0.0:20128 0.0.0.0:0 LISTENING 111",
" TCP 127.0.0.1:20128 0.0.0.0:0 LISTENING 222",
" TCP 0.0.0.0:120128 0.0.0.0:0 LISTENING 333", // different port (prefix)
" TCP 0.0.0.0:20128 0.0.0.0:0 TIME_WAIT 444", // not listening
].join("\r\n");
const kills: Array<{ pid: number; signal: string | number }> = [];
const deps = {
platform: "win32",
execFileAsync: async (_cmd: string, _args: string[]) => ({ stdout, stderr: "" }),
processKill: (p: number, sig: string | number) => {
kills.push({ pid: p, signal: sig });
return true;
},
isPidRunning: (_p: number) => false,
sleep: async (_ms: number) => {},
};
const { killByPort } = await import("../../bin/cli/commands/stop.mjs");
await (killByPort as unknown as KillByPortFn)(20128, deps);
const signalled = kills.map((k) => k.pid).sort();
assert.deepEqual(signalled, [111, 222], "only exact-port LISTENING PIDs must be killed");
void ORIGINAL_PLATFORM;
});