#!/usr/bin/env node /** * check:pack-boot — boot-smoke of the REAL npm tarball (#7065 class killer, WS1.2/T1). * * Three releases shipped a tarball that crashed on every boot (tls-options/3.8.41, * head-response-guard VPS #7040 + npm #7065) because no gate ever EXECUTED the * artifact: structure checks (check:pack-artifact) validate lists, not runtime. * This gate packs the tree, installs the tarball into a clean prefix, boots the * installed CLI and polls /api/monitoring/health until it proves the artifact * starts — regardless of WHICH packaging list drifted. * * Requires a built dist/ (run after `npm run build:cli`, e.g. in the CI * package-artifact job or `check:release-green --with-build`). Exit codes: * 0 = boots and reports the right version · 1 = boot failed · 2 = missing build. */ import { execFileSync, spawn } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const POLL_INTERVAL_MS = 2_000; const BOOT_DEADLINE_MS = 240_000; const SQLJS_STARTUP_MARKER = "Pre-initializing sql.js WASM"; export const REQUIRED_SQLJS_RUNTIME_FILES = Object.freeze([ "dist/node_modules/sql.js/package.json", "dist/node_modules/sql.js/dist/sql-wasm.js", "dist/node_modules/sql.js/dist/sql-wasm.wasm", ]); /** Parse `npm pack --json` output into the generated tarball filename. */ export function pickTarball(packJsonOutput) { const parsed = JSON.parse(packJsonOutput); const filename = Array.isArray(parsed) ? parsed[0]?.filename : undefined; if (!filename) throw new Error("npm pack --json returned no filename"); // npm >=9 may emit scoped names with "/" — normalize to the on-disk file name. return filename.replace(/\//g, "-"); } /** * Boot verdict: HTTP 200 + a JSON body reporting the version we just packed. * `status` is logged but NOT asserted — a clean install with zero providers may * legitimately report degraded states; the gate targets boot crashes, not health. */ export function evaluateBoot(httpStatus, body, expectedVersion) { const failures = []; if (httpStatus !== 200) failures.push(`health HTTP ${httpStatus} (expected 200)`); if (!body || typeof body !== "object") failures.push("health body is not JSON"); else if (body.version !== expectedVersion) failures.push(`version "${body.version}" (expected "${expectedVersion}")`); return { ok: failures.length === 0, failures }; } /** Deterministic-enough free-ish port in a range CI runners don't use. */ export function pickPort(seed = process.pid) { return 23000 + (seed % 4000); } export function findMissingSqlJsRuntimeFiles(packageRoot, exists = fs.existsSync) { return REQUIRED_SQLJS_RUNTIME_FILES.filter( (relativePath) => !exists(path.join(packageRoot, relativePath)) ); } export function evaluateSqlJsRoundTrip({ startupOutput, beforeValue, patchedValue, readBackValue, }) { const failures = []; if (!startupOutput.includes(SQLJS_STARTUP_MARKER)) { failures.push("server output did not confirm the forced sql.js startup path"); } if (patchedValue !== !beforeValue) { failures.push( `PATCH debugMode returned ${String(patchedValue)} (expected ${String(!beforeValue)})` ); } if (readBackValue !== !beforeValue) { failures.push( `GET debugMode returned ${String(readBackValue)} (expected ${String(!beforeValue)})` ); } return { ok: failures.length === 0, failures }; } /** * After a clean shutdown + restart with the same DATA_DIR, the value written in boot #1 * must be read back from disk in boot #2. sql.js is in-memory with debounced/flush writes, * so this proves the persisted file actually landed and the restart reads it. */ export function evaluateRestartPersistence({ expectedValue, restartValue }) { const failures = []; if (restartValue !== expectedValue) { failures.push( `restart GET debugMode returned ${String(restartValue)} (expected ${String(expectedValue)} after restart)` ); } return { ok: failures.length === 0, failures }; } async function readJsonResponse(url, options) { const response = await fetch(url, options); const body = await response.json().catch(() => null); return { response, body }; } async function verifySettingsRoundTrip(baseUrl, startupOutput) { const initial = await readJsonResponse(`${baseUrl}/api/settings`); if (initial.response.status !== 200 || !initial.body || typeof initial.body !== "object") { return { ok: false, failures: [`initial settings HTTP ${initial.response.status} or non-JSON body`], }; } const beforeValue = initial.body.debugMode === true; const expectedValue = !beforeValue; const patched = await readJsonResponse(`${baseUrl}/api/settings`, { method: "PATCH", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ debugMode: expectedValue }), }); if (patched.response.status !== 200 || !patched.body || typeof patched.body !== "object") { return { ok: false, failures: [`settings PATCH HTTP ${patched.response.status} or non-JSON body`], }; } const readBack = await readJsonResponse(`${baseUrl}/api/settings`); if (readBack.response.status !== 200 || !readBack.body || typeof readBack.body !== "object") { return { ok: false, failures: [`settings read-back HTTP ${readBack.response.status} or non-JSON body`], }; } return { ...evaluateSqlJsRoundTrip({ startupOutput, beforeValue, patchedValue: patched.body.debugMode, readBackValue: readBack.body.debugMode, }), // The exact value boot #2 must read back from disk to prove persistence. expectedValue, }; } function log(msg) { console.log(`[pack-boot] ${msg}`); } /** Node sets exitCode/signalCode synchronously when the process dies — authoritative. */ function hasExited(child) { return child.exitCode !== null || child.signalCode !== null; } /** * SIGTERM the process GROUP and wait for its REAL exit — the graceful-shutdown handler * (initGracefulShutdown) drains requests, checkpoints the DB via closeDbInstance(), then * calls process.exit(0). A fixed sleep + hard kill could SIGKILL mid-flush and silently * drop the very persistence this gate proves, so SIGKILL is a last resort after the grace * deadline, and a CONFIRMED exit is required before returning: if even SIGKILL fails to * reap, throw, so boot #2 cannot start against a port a zombie still holds. * * The child is spawned with detached:true, so it leads its own process group and * -child.pid signals the whole tree, not just the launcher. */ async function stopChild(child, graceMs = 30_000) { if (!child?.pid) return; // Fast path: already reaped (crashed mid-smoke, or exited before this call) — nothing // left to signal or wait for. if (hasExited(child)) return; let onSettled; const exited = new Promise((resolve) => { onSettled = () => resolve(); child.once("exit", onSettled); child.once("close", onSettled); }); // Race the exit/close promise against a timeout; then re-read authoritative state, so a // same-tick exit that lost the race still counts. Timer is always cleared. const waitForExit = (ms) => { let timer; return Promise.race([ exited, new Promise((resolve) => { timer = setTimeout(resolve, ms); }), ]) .finally(() => clearTimeout(timer)) .then(() => hasExited(child)); }; try { // Re-check AFTER attaching: if the process died in the gap between the fast path and // listener attach, once("exit") can never fire (event already emitted), and without // this waitForExit would burn the full grace window. if (hasExited(child)) return; try { process.kill(-child.pid, "SIGTERM"); } catch { /* group already gone */ } if (await waitForExit(graceMs)) return; try { process.kill(-child.pid, "SIGKILL"); } catch { /* group already gone */ } if (!(await waitForExit(5_000))) { throw new Error( `[pack-boot] server process group ${child.pid} still alive 5s after SIGKILL — ` + "refusing to reboot on the same port" ); } } finally { child.removeListener("exit", onSettled); child.removeListener("close", onSettled); } } /** * Boot the installed CLI once on an isolated DATA_DIR. The child is spawned detached:true * so it leads its own process group — stopChild() relies on that to SIGTERM the whole tree. * The caller owns shutdown so the graceful DB flush lands before teardown. */ function spawnServer(binPath, port, dataDir) { const child = spawn(binPath, ["serve", "--port", String(port), "--log", "--no-open"], { env: { ...process.env, PORT: String(port), DATA_DIR: dataDir, JWT_SECRET: "pack-boot-smoke-secret-with-sufficient-length-000", API_KEY_SECRET: "pack-boot-smoke-api-key-secret-long", DISABLE_SQLITE_AUTO_BACKUP: "true", OMNIROUTE_SKIP_SYSTEM_TRUST: "1", OMNIROUTE_PACK_BOOT_SMOKE: "1", OMNIROUTE_PACK_BOOT_FORCE_SQLJS: "1", }, stdio: ["ignore", "pipe", "pipe"], detached: true, }); const tail = []; const keepTail = (chunk) => { tail.push(String(chunk)); while (tail.length > 80) tail.shift(); }; child.stdout.on("data", keepTail); child.stderr.on("data", keepTail); return { child, tail }; } /** Poll /api/monitoring/health until the packed version answers or the boot deadline passes. */ async function waitForHealthy(port, child, expectedVersion) { // Seed from authoritative state (Node sets these synchronously at death), then attach a // named once-listener, then re-check: a child that died before this call, or in the gap // before the listener attached, would otherwise never fire "exit" and waste the deadline. const exitDescriptor = (code, signal) => (signal ? `signal ${signal}` : `code ${code ?? -1}`); let childExit = hasExited(child) ? exitDescriptor(child.exitCode, child.signalCode) : null; const onChildExit = (code, signal) => { childExit = exitDescriptor(code, signal); }; child.once("exit", onChildExit); if (hasExited(child)) { childExit = exitDescriptor(child.exitCode, child.signalCode); } const deadline = Date.now() + BOOT_DEADLINE_MS; let verdict = { ok: false, failures: ["never polled"] }; try { while (Date.now() < deadline) { if (childExit !== null) { return { ok: false, failures: [`process exited (${childExit}) before serving`] }; } try { const res = await fetch(`http://127.0.0.1:${port}/api/monitoring/health`); const body = await res.json().catch(() => null); verdict = evaluateBoot(res.status, body, expectedVersion); if (verdict.ok) return verdict; } catch { // not listening yet — keep polling } await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); } return verdict; } finally { child.removeListener("exit", onChildExit); } } /** * Read the current debugMode setting and return the EXACT boolean. A missing or non-boolean * field throws: coercing with `=== true` would read `false` for a malformed response and * could falsely "pass" persistence whenever the expected value happens to be false. */ async function readSettingsDebugMode(baseUrl) { const { response, body } = await readJsonResponse(`${baseUrl}/api/settings`); if (response.status !== 200 || !body || typeof body !== "object") { throw new Error(`settings GET HTTP ${response.status} or non-JSON body`); } if (typeof body.debugMode !== "boolean") { throw new Error(`settings debugMode is ${typeof body.debugMode} (expected boolean)`); } return body.debugMode; } async function main() { const ROOT = process.cwd(); if (!fs.existsSync(path.join(ROOT, "dist", "server.js"))) { console.error( "[pack-boot] dist/server.js missing — run `npm run build:cli` first (this is a --with-build gate)" ); process.exit(2); } const expectedVersion = JSON.parse( fs.readFileSync(path.join(ROOT, "package.json"), "utf8") ).version; const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-pack-boot-")); let child = null; let tail = []; let exitCode = 1; let primaryError = null; // a smoke-logic failure: boot/PATCH/GET/restart, or an in-flow stop let cleanupError = null; // recorded ONLY in finally, ONLY for a final stopChild failure let shutdownConfirmed = false; // process group confirmed stopped → safe to rm the workspace try { log(`packing v${expectedVersion}…`); const packOut = execFileSync("npm", ["pack", "--json", "--pack-destination", tmp], { cwd: ROOT, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, }); const tarball = path.join(tmp, pickTarball(packOut)); log(`installing ${path.basename(tarball)} into a clean prefix (postinstall runs for real)…`); const prefix = path.join(tmp, "prefix"); execFileSync("npm", ["install", "-g", "--prefix", prefix, tarball], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, }); const packageRoot = path.join(prefix, "lib", "node_modules", "omniroute"); const missingSqlJsFiles = findMissingSqlJsRuntimeFiles(packageRoot); if (missingSqlJsFiles.length > 0) { throw new Error( `installed package is missing the sql.js runtime contract: ${missingSqlJsFiles.join(", ")}` ); } log("installed package contains the complete sql.js WASM runtime"); const port = pickPort(); const dataDir = path.join(tmp, "data"); fs.mkdirSync(dataDir, { recursive: true }); const binPath = path.join(prefix, "bin", "omniroute"); // BOOT #1 — boot, prove the forced sql.js tier, PATCH a setting, then shut down cleanly // so the sql.js adapter's graceful persist actually lands on disk. The in-flow stopChild // THROWS on failure; that lands in catch as primaryError and boot #2 never starts. log(`boot #1: installed CLI on :${port} (DATA_DIR isolated)…`); ({ child, tail } = spawnServer(binPath, port, dataDir)); let verdict = await waitForHealthy(port, child, expectedVersion); if (verdict.ok) { log(`healthy: HTTP 200, version ${expectedVersion}`); const roundTrip = await verifySettingsRoundTrip(`http://127.0.0.1:${port}`, tail.join("")); if (roundTrip.ok) { log("settings write/read succeeded through the forced sql.js driver"); await stopChild(child); // throws here → primaryError; boot #2 is skipped child = null; // BOOT #2 — same DATA_DIR, fresh process: the value must be read back FROM DISK. log("boot #2: rebooting on the same DATA_DIR to prove disk persistence…"); ({ child, tail } = spawnServer(binPath, port, dataDir)); verdict = await waitForHealthy(port, child, expectedVersion); if (verdict.ok) { log(`healthy: HTTP 200, version ${expectedVersion}`); const restartValue = await readSettingsDebugMode(`http://127.0.0.1:${port}`); const persistence = evaluateRestartPersistence({ expectedValue: roundTrip.expectedValue, restartValue, }); if (persistence.ok) { log("value survived a clean shutdown + restart — disk persistence proven"); await stopChild(child); // throws here → primaryError child = null; exitCode = 0; } else { verdict = persistence; } } } else { verdict = roundTrip; } } if (!verdict.ok) { primaryError = new Error(verdict.failures.join("; ")); exitCode = 1; } } catch (e) { // Every smoke-logic failure — boot/PATCH/GET/restart AND in-flow stopChild throws. primaryError = e; exitCode = 1; } finally { // Tear down whatever is still running. This block records ONLY a stopChild failure, // and never overwrites primaryError. if (child) { try { await stopChild(child); shutdownConfirmed = true; } catch (e) { cleanupError = e; // still !shutdownConfirmed → workspace preserved below } child = null; } else { // Stopped in-flow (already confirmed) or never spawned — nothing left to confirm. shutdownConfirmed = true; } // Remove the workspace ONLY after confirmed shutdown; a process group that refused to // die keeps its DATA_DIR for diagnosis. if (shutdownConfirmed) { fs.rmSync(tmp, { recursive: true, force: true }); } } // Report primaryError as the smoke failure; report cleanupError separately. Either one // fails the gate. if (primaryError) { console.error(`[pack-boot] ❌ smoke FAILED: ${primaryError.message}`); if (tail.length) { console.error( "[pack-boot] last server output:\n" + tail.join("").split("\n").slice(-40).join("\n") ); } } if (cleanupError) { console.error(`[pack-boot] ❌ final shutdown FAILED: ${cleanupError.message}`); exitCode = 1; } if (exitCode === 0) { log("✅ the packed tarball boots AND persists — #7065 class gate green"); } if (!shutdownConfirmed) { console.error( `[pack-boot] ⚠ process group not confirmed stopped — workspace preserved for diagnosis: ${tmp}` ); } process.exit(exitCode); } const isDirectRun = process.argv[1] && path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname); if (isDirectRun) { main().catch((e) => { console.error("[pack-boot] fatal:", e.message); process.exit(1); }); }