#!/usr/bin/env node /** * scripts/ops/deploy-canary.mjs — ship a packaged artifact to a canary host and PROVE it works. * * Replaces the manual build → pack → scp → `npm i -g` → `pm2 restart` sequence that caused * the 2026-08-14 gateway outage (#10429): the package installed there had been built from a * feature branch predating #10373, the process came up healthy, and every request returned * `502 … Executor result must contain a Response` until a human noticed. * * The policy lives in `deployCanary.ts` (pure, unit-tested); this file is the thin shell * that performs the side effects and rolls back when the smoke fails. * * Usage: * node scripts/ops/deploy-canary.mjs --host root@192.168.0.17 --tarball ./omniroute-3.8.50.tgz \ * --base-url http://192.168.0.17:20128 --model cx/gpt-5.6-terra --model qct/deepseek-v4-flash-0731 * * Flags: * --host ssh target (required) * --tarball local tarball produced by `npm run build:release && npm pack` (required) * --base-url http base of the deployed gateway (required) * --model completion probe target; repeatable, at least one required * --pm2-app process-manager app name (default: omniroute) * --dry-run print the plan and the remote steps, change nothing * * Env: * OMNIROUTE_RELEASE_REF ref to check ancestry against (default origin/main) * OMNIROUTE_ALLOW_CANARY_BUILD set to 1 to deploy an artifact that is not on the release line * OMNIROUTE_SMOKE_API_KEY sent as Authorization: Bearer when the gateway requires auth */ import { execFileSync } from "node:child_process"; import path from "node:path"; import process from "node:process"; import { buildRemoteSteps, evaluateSmoke, planCanaryDeploy } from "./deployCanary.ts"; import { makeGitAncestryProbe, readBuildSha } from "../build/buildProvenance.ts"; function parseArgs(argv) { const args = { models: [], pm2App: "omniroute", dryRun: false }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; const value = argv[i + 1]; if (flag === "--host") args.host = value; else if (flag === "--tarball") args.tarball = value; else if (flag === "--base-url") args.baseUrl = value; else if (flag === "--model") args.models.push(value); else if (flag === "--pm2-app") args.pm2App = value; else if (flag === "--dry-run") args.dryRun = true; } return args; } function fail(message) { console.error(`\n❌ ${message}`); process.exit(1); } function run(step) { console.log(`\n▶ ${step.name}: ${step.description}`); const [command, ...rest] = step.argv; return execFileSync(command, rest, { encoding: "utf8" }).trim(); } async function probeHealth(baseUrl) { try { const response = await fetch(new URL("/api/monitoring/health", baseUrl), { signal: AbortSignal.timeout(20_000), }); if (!response.ok) return { ok: false, buildSha: null }; const body = await response.json(); return { ok: body?.status === "healthy", buildSha: body?.system?.buildSha ?? null, }; } catch { return { ok: false, buildSha: null }; } } async function probeCompletion(baseUrl, model, apiKey) { const headers = { "Content-Type": "application/json" }; if (apiKey) headers.Authorization = `Bearer ${apiKey}`; try { const response = await fetch(new URL("/v1/chat/completions", baseUrl), { method: "POST", headers, body: JSON.stringify({ model, messages: [{ role: "user", content: "reply with: ok" }], max_tokens: 16, }), signal: AbortSignal.timeout(120_000), }); // A 2xx alone is not enough: the outage this script exists for returned a body-level // failure. Require a parseable completion with at least one choice. const body = await response.json().catch(() => null); const ok = response.ok && Array.isArray(body?.choices) && body.choices.length > 0; return { model, ok, status: response.status }; } catch { return { model, ok: false, status: 0 }; } } const args = parseArgs(process.argv.slice(2)); if (!args.host) fail("--host is required"); if (!args.tarball) fail("--tarball is required"); if (!args.baseUrl) fail("--base-url is required"); if (args.models.length === 0) { fail("at least one --model is required — a health check cannot see a broken egress path"); } const repoRoot = process.cwd(); const plan = planCanaryDeploy({ buildSha: readBuildSha(repoRoot), isAncestorOfRelease: makeGitAncestryProbe( process.env.OMNIROUTE_RELEASE_REF || "origin/main", repoRoot ), allowCanary: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1", }); console.log(`[provenance] ${plan.reason}`); if (!plan.proceed) fail("refusing to deploy an artifact that cannot be traced to the release line"); const remoteTarball = path.posix.join("/root", path.basename(args.tarball)); const steps = buildRemoteSteps({ host: args.host, tarballPath: remoteTarball, pm2App: args.pm2App, }); if (args.dryRun) { console.log("\n--dry-run: nothing will be changed. Planned steps:"); console.log(` scp ${args.tarball} ${args.host}:${remoteTarball}`); for (const step of steps) console.log(` ${step.argv.join(" ")}`); console.log(` probes: health + ${args.models.join(", ")}`); process.exit(0); } let previousSha = null; try { const [capture, install, restart, verify] = steps; previousSha = run(capture); console.log(` previous BUILD_SHA: ${previousSha || "(none)"}`); console.log(`\n▶ upload: ${args.tarball} → ${args.host}:${remoteTarball}`); execFileSync("scp", [args.tarball, `${args.host}:${remoteTarball}`], { stdio: "inherit" }); run(install); run(restart); const installedSha = run(verify); console.log(` installed BUILD_SHA: ${installedSha}`); // Give the process a moment to bind before probing. await new Promise((resolve) => setTimeout(resolve, 15_000)); const health = await probeHealth(args.baseUrl); const completions = []; for (const model of args.models) { const probe = await probeCompletion(args.baseUrl, model, process.env.OMNIROUTE_SMOKE_API_KEY); console.log(` probe ${probe.model}: ${probe.ok ? "ok" : `FAILED (${probe.status})`}`); completions.push(probe); } const verdict = evaluateSmoke({ healthOk: health.ok, completions }); if (!verdict.ok) { console.error(`\n❌ smoke failed: ${verdict.reason}`); if (previousSha) { console.error( `\n⚠️ ROLLBACK REQUIRED — the previous artifact was ${previousSha}. This script does ` + "not keep old tarballs, so reinstall that build and restart:\n" + ` ssh ${args.host} npm install -g --no-audit --no-fund\n` + ` ssh ${args.host} pm2 restart ${args.pm2App} --update-env` ); } process.exit(1); } console.log(`\n✅ ${verdict.reason}`); console.log(` deployed BUILD_SHA: ${installedSha}`); if (health.buildSha && health.buildSha !== installedSha) { console.warn( `\n⚠️ health reports buildSha ${health.buildSha} but the package says ${installedSha} — ` + "the process may still be serving the old artifact." ); } } catch (error) { fail(`deploy aborted: ${error.message}`); }