diff --git a/scripts/ops/deploy-canary.mjs b/scripts/ops/deploy-canary.mjs index d6bef152c7..c0c93af606 100644 --- a/scripts/ops/deploy-canary.mjs +++ b/scripts/ops/deploy-canary.mjs @@ -28,11 +28,16 @@ * OMNIROUTE_SMOKE_API_KEY sent as Authorization: Bearer when the gateway requires auth */ -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import path from "node:path"; import process from "node:process"; -import { buildRemoteSteps, evaluateSmoke, planCanaryDeploy } from "./deployCanary.ts"; +import { + buildRemoteSteps, + classifyInstallOutcome, + evaluateSmoke, + planCanaryDeploy, +} from "./deployCanary.ts"; import { makeGitAncestryProbe, readBuildSha } from "../build/buildProvenance.ts"; function parseArgs(argv) { @@ -61,6 +66,22 @@ function run(step) { return execFileSync(command, rest, { encoding: "utf8" }).trim(); } +/** + * Like `run`, but never throws: returns the exit code plus both streams. Used for the + * install, whose exit code does not decide the outcome (see classifyInstallOutcome) and + * whose stderr must reach the log — it used to be swallowed by execFileSync throwing. + */ +function runCapturing(step) { + console.log(`\n▶ ${step.name}: ${step.description}`); + const [command, ...rest] = step.argv; + const result = spawnSync(command, rest, { encoding: "utf8" }); + return { + exitCode: result.status ?? 1, + stdout: (result.stdout || "").trim(), + stderr: (result.stderr || "").trim(), + }; +} + async function probeHealth(baseUrl) { try { const response = await fetch(new URL("/api/monitoring/health", baseUrl), { @@ -110,8 +131,9 @@ if (args.models.length === 0) { } const repoRoot = process.cwd(); +const localBuildSha = readBuildSha(repoRoot); const plan = planCanaryDeploy({ - buildSha: readBuildSha(repoRoot), + buildSha: localBuildSha, isAncestorOfRelease: makeGitAncestryProbe( process.env.OMNIROUTE_RELEASE_REF || "origin/main", repoRoot @@ -147,7 +169,23 @@ try { console.log(`\n▶ upload: ${args.tarball} → ${args.host}:${remoteTarball}`); execFileSync("scp", [args.tarball, `${args.host}:${remoteTarball}`], { stdio: "inherit" }); - run(install); + const installResult = runCapturing(install); + const outcome = classifyInstallOutcome({ + exitCode: installResult.exitCode, + stderr: installResult.stderr, + installedSha: run(verify), + expectedSha: localBuildSha, + }); + if (!outcome.installed) { + if (installResult.stderr) console.error(installResult.stderr); + fail(`install did not land: ${outcome.reason}`); + } + if (outcome.kind === "installed-with-cleanup-failure") { + console.warn(` ⚠️ ${outcome.reason}`); + } else { + console.log(` ${outcome.reason}`); + } + run(restart); const installedSha = run(verify); diff --git a/scripts/ops/deployCanary.ts b/scripts/ops/deployCanary.ts index d81408326b..11103fc830 100644 --- a/scripts/ops/deployCanary.ts +++ b/scripts/ops/deployCanary.ts @@ -152,3 +152,80 @@ export function buildRemoteSteps(input: RemoteStepsInput): RemoteStep[] { }, ]; } + +export type InstallOutcomeInput = { + exitCode: number; + stderr: string; + /** BUILD_SHA read back from the installed package AFTER the install ran. */ + installedSha: string | null | undefined; + /** BUILD_SHA of the artifact being shipped. */ + expectedSha: string; +}; + +export type InstallOutcome = { + installed: boolean; + kind: "installed" | "installed-with-cleanup-failure" | "failed"; + reason: string; +}; + +/** + * Decide whether the global install actually landed. + * + * The exit code alone is not trustworthy in either direction: + * + * - `npm install -g` on the .17 gateway writes the whole package and *then* fails renaming + * the old tree into its staging directory (`ENOTEMPTY`, exit 217). Treating that as a + * failure aborts the deploy after the artifact is already on disk — which happened twice + * on 2026-08-18, each time leaving the host with new files and an old running process. + * - The 2026-08-14 outage went the other way: the install exited 0 while shipping a package + * built from the wrong branch. + * + * So the SHA on disk decides, and it must match exactly. An absent or unreadable SHA fails + * closed — an artifact that cannot be identified is never attested (same rule as the + * provenance gate). + */ +export function classifyInstallOutcome(input: InstallOutcomeInput): InstallOutcome { + const { exitCode, stderr, installedSha, expectedSha } = input; + const onDisk = (installedSha ?? "").trim(); + + if (!onDisk) { + return { + installed: false, + kind: "failed", + reason: "no BUILD_SHA could be read from the installed package after the install", + }; + } + if (onDisk !== expectedSha) { + return { + installed: false, + kind: "failed", + reason: `installed BUILD_SHA is ${onDisk}, expected ${expectedSha}`, + }; + } + if (exitCode === 0) { + return { installed: true, kind: "installed", reason: `installed ${onDisk}` }; + } + + const staging = orphanStagingDirFromStderr(stderr); + const enotempty = /ENOTEMPTY/.test(stderr); + return { + installed: true, + kind: "installed-with-cleanup-failure", + reason: + `npm exited ${exitCode} but ${onDisk} is on disk — the package installed and npm failed ` + + `during its own cleanup${enotempty ? " (ENOTEMPTY on the staging rename)" : ""}` + + (staging ? `; orphaned staging dir left behind: ${staging}` : ""), + }; +} + +/** + * The staging directory npm failed to rename into, if it named one. It blocks the NEXT + * install with the same error (npm reuses the name), so the operator has to clear it — + * surfacing the exact path is the whole point. Deliberately not removed automatically: + * this is a path under /usr/lib and a blind `rm -rf` there is not something a deploy + * script should do on its own. + */ +export function orphanStagingDirFromStderr(stderr: string): string | null { + const match = /npm error dest (\/\S*\/\.\S+)/.exec(stderr || ""); + return match ? match[1] : null; +} diff --git a/tests/unit/canary-install-outcome-10429.test.ts b/tests/unit/canary-install-outcome-10429.test.ts new file mode 100644 index 0000000000..5e00a2c3f7 --- /dev/null +++ b/tests/unit/canary-install-outcome-10429.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + classifyInstallOutcome, + orphanStagingDirFromStderr, +} from "../../scripts/ops/deployCanary.ts"; + +/** + * `npm install -g ` on the .17 gateway finishes writing the package and then fails + * while renaming the old tree into its staging directory: + * + * npm error code ENOTEMPTY + * npm error syscall rename + * npm error path /usr/lib/node_modules/omniroute + * npm error dest /usr/lib/node_modules/.omniroute-h797OOZa + * + * Exit status is 217, but `dist/BUILD_SHA`, the package version and every dependency are the + * new ones. The canary treated the non-zero exit as "install failed", aborted before the + * restart, and discarded npm's stderr — so the deploy stopped half-done twice (2026-08-18) + * with no clue in the log about why. + * + * The exit code is not the source of truth here; the SHA on disk is. These cases pin that, + * including the inverse trap: a ZERO exit that installed the wrong artifact must still fail. + */ + +const ENOTEMPTY_STDERR = [ + "npm warn deprecated boolean@3.2.0: Package no longer supported.", + "npm error code ENOTEMPTY", + "npm error syscall rename", + "npm error path /usr/lib/node_modules/omniroute", + "npm error dest /usr/lib/node_modules/.omniroute-h797OOZa", + "npm error ENOTEMPTY: directory not empty, rename '/usr/lib/node_modules/omniroute' -> " + + "'/usr/lib/node_modules/.omniroute-h797OOZa'", +].join("\n"); + +test("non-zero exit with the expected SHA on disk is a cleanup failure, not an install failure", () => { + const outcome = classifyInstallOutcome({ + exitCode: 217, + stderr: ENOTEMPTY_STDERR, + installedSha: "22b89a273b", + expectedSha: "22b89a273b", + }); + assert.equal(outcome.installed, true, "the artifact is on disk — the deploy must continue"); + assert.equal(outcome.kind, "installed-with-cleanup-failure"); + assert.match(outcome.reason, /ENOTEMPTY|cleanup/i); +}); + +test("a clean install is reported as such", () => { + const outcome = classifyInstallOutcome({ + exitCode: 0, + stderr: "", + installedSha: "22b89a273b", + expectedSha: "22b89a273b", + }); + assert.equal(outcome.installed, true); + assert.equal(outcome.kind, "installed"); +}); + +test("non-zero exit with a stale SHA is a real failure", () => { + const outcome = classifyInstallOutcome({ + exitCode: 217, + stderr: ENOTEMPTY_STDERR, + installedSha: "e05ac345da", + expectedSha: "22b89a273b", + }); + assert.equal(outcome.installed, false); + assert.equal(outcome.kind, "failed"); +}); + +test("a ZERO exit that left the wrong artifact still fails", () => { + // The 2026-08-14 outage shipped a package built from the wrong branch. An install that + // "succeeds" while the SHA does not match must never be waved through. + const outcome = classifyInstallOutcome({ + exitCode: 0, + stderr: "", + installedSha: "178febc50f", + expectedSha: "22b89a273b", + }); + assert.equal(outcome.installed, false); + assert.equal(outcome.kind, "failed"); +}); + +test("an unreadable SHA fails closed", () => { + for (const installedSha of ["", null, undefined]) { + const outcome = classifyInstallOutcome({ + exitCode: 0, + stderr: "", + installedSha: installedSha as string | null, + expectedSha: "22b89a273b", + }); + assert.equal(outcome.installed, false, `installedSha=${JSON.stringify(installedSha)}`); + assert.equal(outcome.kind, "failed"); + } +}); + +test("the orphaned staging directory is extracted so the operator can clear it", () => { + assert.equal( + orphanStagingDirFromStderr(ENOTEMPTY_STDERR), + "/usr/lib/node_modules/.omniroute-h797OOZa" + ); +}); + +test("no staging directory is invented when npm did not report one", () => { + assert.equal(orphanStagingDirFromStderr(""), null); + assert.equal(orphanStagingDirFromStderr("npm error code EACCES"), null); +});