fix(ops): judge the canary install by the SHA on disk, not npm's exit code (#10699)

`npm install -g <tarball>` on the .17 gateway writes the whole package and then fails
renaming the old tree into its staging directory (ENOTEMPTY, exit 217). The canary read
that non-zero exit as "install failed", aborted before the restart, and discarded npm's
stderr through execFileSync throwing — so on 2026-08-18 the deploy stopped half-done
twice, each time leaving new files on disk under an old running process, with no clue in
the log.

The exit code is not trustworthy in either direction: the 2026-08-14 outage installed a
package built from the wrong branch and exited 0. classifyInstallOutcome() therefore
decides on the BUILD_SHA read back from the installed package, and fails closed when it
is absent or does not match — a zero exit with the wrong artifact is still a failure.

npm reuses the same staging directory name, so the orphan blocks the next install with
the same error; orphanStagingDirFromStderr() surfaces the exact path. It is not removed
automatically — that is an rm -rf under /usr/lib, not something a deploy script should
decide on its own.

Refs #10429

Co-authored-by: Xiangzhe <bakryun0718@proton.me>

scripts/ops/deploy-canary.mjs

@@ -28,11 +28,16 @@
  *   OMNIROUTE_SMOKE_API_KEY      sent as Authorization: Bearer when the gateway requires auth
  */
 
-import { execFileSync } from "node:child_process";
+import { execFileSync, spawnSync } from "node:child_process";
 import path from "node:path";
 import process from "node:process";
 
-import { buildRemoteSteps, evaluateSmoke, planCanaryDeploy } from "./deployCanary.ts";
+import {
+  buildRemoteSteps,
+  classifyInstallOutcome,
+  evaluateSmoke,
+  planCanaryDeploy,
+} from "./deployCanary.ts";
 import { makeGitAncestryProbe, readBuildSha } from "../build/buildProvenance.ts";
 
 function parseArgs(argv) {
@@ -61,6 +66,22 @@ function run(step) {
   return execFileSync(command, rest, { encoding: "utf8" }).trim();
 }
 
+/**
+ * Like `run`, but never throws: returns the exit code plus both streams. Used for the
+ * install, whose exit code does not decide the outcome (see classifyInstallOutcome) and
+ * whose stderr must reach the log — it used to be swallowed by execFileSync throwing.
+ */
+function runCapturing(step) {
+  console.log(`\n▶ ${step.name}: ${step.description}`);
+  const [command, ...rest] = step.argv;
+  const result = spawnSync(command, rest, { encoding: "utf8" });
+  return {
+    exitCode: result.status ?? 1,
+    stdout: (result.stdout || "").trim(),
+    stderr: (result.stderr || "").trim(),
+  };
+}
+
 async function probeHealth(baseUrl) {
   try {
     const response = await fetch(new URL("/api/monitoring/health", baseUrl), {
@@ -110,8 +131,9 @@ if (args.models.length === 0) {
 }
 
 const repoRoot = process.cwd();
+const localBuildSha = readBuildSha(repoRoot);
 const plan = planCanaryDeploy({
-  buildSha: readBuildSha(repoRoot),
+  buildSha: localBuildSha,
   isAncestorOfRelease: makeGitAncestryProbe(
     process.env.OMNIROUTE_RELEASE_REF || "origin/main",
     repoRoot
@@ -147,7 +169,23 @@ try {
   console.log(`\n▶ upload: ${args.tarball} → ${args.host}:${remoteTarball}`);
   execFileSync("scp", [args.tarball, `${args.host}:${remoteTarball}`], { stdio: "inherit" });
 
-  run(install);
+  const installResult = runCapturing(install);
+  const outcome = classifyInstallOutcome({
+    exitCode: installResult.exitCode,
+    stderr: installResult.stderr,
+    installedSha: run(verify),
+    expectedSha: localBuildSha,
+  });
+  if (!outcome.installed) {
+    if (installResult.stderr) console.error(installResult.stderr);
+    fail(`install did not land: ${outcome.reason}`);
+  }
+  if (outcome.kind === "installed-with-cleanup-failure") {
+    console.warn(`   ⚠️  ${outcome.reason}`);
+  } else {
+    console.log(`   ${outcome.reason}`);
+  }
+
   run(restart);
 
   const installedSha = run(verify);

scripts/ops/deployCanary.ts

@@ -152,3 +152,80 @@ export function buildRemoteSteps(input: RemoteStepsInput): RemoteStep[] {
     },
   ];
 }
+
+export type InstallOutcomeInput = {
+  exitCode: number;
+  stderr: string;
+  /** BUILD_SHA read back from the installed package AFTER the install ran. */
+  installedSha: string | null | undefined;
+  /** BUILD_SHA of the artifact being shipped. */
+  expectedSha: string;
+};
+
+export type InstallOutcome = {
+  installed: boolean;
+  kind: "installed" | "installed-with-cleanup-failure" | "failed";
+  reason: string;
+};
+
+/**
+ * Decide whether the global install actually landed.
+ *
+ * The exit code alone is not trustworthy in either direction:
+ *
+ *  - `npm install -g` on the .17 gateway writes the whole package and *then* fails renaming
+ *    the old tree into its staging directory (`ENOTEMPTY`, exit 217). Treating that as a
+ *    failure aborts the deploy after the artifact is already on disk — which happened twice
+ *    on 2026-08-18, each time leaving the host with new files and an old running process.
+ *  - The 2026-08-14 outage went the other way: the install exited 0 while shipping a package
+ *    built from the wrong branch.
+ *
+ * So the SHA on disk decides, and it must match exactly. An absent or unreadable SHA fails
+ * closed — an artifact that cannot be identified is never attested (same rule as the
+ * provenance gate).
+ */
+export function classifyInstallOutcome(input: InstallOutcomeInput): InstallOutcome {
+  const { exitCode, stderr, installedSha, expectedSha } = input;
+  const onDisk = (installedSha ?? "").trim();
+
+  if (!onDisk) {
+    return {
+      installed: false,
+      kind: "failed",
+      reason: "no BUILD_SHA could be read from the installed package after the install",
+    };
+  }
+  if (onDisk !== expectedSha) {
+    return {
+      installed: false,
+      kind: "failed",
+      reason: `installed BUILD_SHA is ${onDisk}, expected ${expectedSha}`,
+    };
+  }
+  if (exitCode === 0) {
+    return { installed: true, kind: "installed", reason: `installed ${onDisk}` };
+  }
+
+  const staging = orphanStagingDirFromStderr(stderr);
+  const enotempty = /ENOTEMPTY/.test(stderr);
+  return {
+    installed: true,
+    kind: "installed-with-cleanup-failure",
+    reason:
+      `npm exited ${exitCode} but ${onDisk} is on disk — the package installed and npm failed ` +
+      `during its own cleanup${enotempty ? " (ENOTEMPTY on the staging rename)" : ""}` +
+      (staging ? `; orphaned staging dir left behind: ${staging}` : ""),
+  };
+}
+
+/**
+ * The staging directory npm failed to rename into, if it named one. It blocks the NEXT
+ * install with the same error (npm reuses the name), so the operator has to clear it —
+ * surfacing the exact path is the whole point. Deliberately not removed automatically:
+ * this is a path under /usr/lib and a blind `rm -rf` there is not something a deploy
+ * script should do on its own.
+ */
+export function orphanStagingDirFromStderr(stderr: string): string | null {
+  const match = /npm error dest (\/\S*\/\.\S+)/.exec(stderr || "");
+  return match ? match[1] : null;
+}