From 1287b6a75debfa390d52e13eece494ce94fac660 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 15 Aug 2026 03:22:09 -0300 Subject: [PATCH] feat(ops): canary deploy with provenance gate, real smoke and rollback anchor (#10446) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploying the internal gateway was a manual build/pack/scp/npm-i/pm2-restart sequence with no record of what landed and no proof it served traffic. On 2026-08-14 that shipped a package built from a branch predating #10373: the process came up, health said 'healthy', and every request returned 502 until a human hit it. scripts/ops/deployCanary.ts holds the policy as pure functions — refuse an artifact that is not traceable to the release line (reusing #10427), and grade the deploy on health PLUS at least one real completion. Zero probes fails: 'no probe ran' must never read as 'everything is fine', which is exactly how a broken egress path hides behind a green health check. Remote steps are argv arrays, never shell strings (Hard Rule #13), ordered so the rollback anchor is captured before the install overwrites it. scripts/ops/deploy-canary.mjs performs the side effects, supports --dry-run, and prints the rollback command when the smoke fails. Closes #10429 --- .env.example | 7 +- docs/reference/ENVIRONMENT.md | 1 + scripts/ops/deploy-canary.mjs | 191 +++++++++++++++++++++++++ scripts/ops/deployCanary.ts | 154 ++++++++++++++++++++ tests/unit/deploy-canary-10429.test.ts | 134 +++++++++++++++++ 5 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 scripts/ops/deploy-canary.mjs create mode 100644 scripts/ops/deployCanary.ts create mode 100644 tests/unit/deploy-canary-10429.test.ts diff --git a/.env.example b/.env.example index 522aff82ef..80f60b55b8 100644 --- a/.env.example +++ b/.env.example @@ -54,7 +54,12 @@ INITIAL_PASSWORD=CHANGEME # Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git # SHA when the dist/BUILD_SHA sentinel is absent; it is also what `npm run build:release` # stamps. OMNIROUTE_RELEASE_REF is the ref the pack gate checks ancestry against, and -# OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of +# OMNIROUTE_ALLOW_CANARY_BUILD=1 + +# API key the canary-deploy smoke uses when the target gateway requires auth (#10429). +# Used by: scripts/ops/deploy-canary.mjs — sent as `Authorization: Bearer` on the +# /v1/chat/completions probe. Never needed by the server itself. +# OMNIROUTE_SMOKE_API_KEY=sk-... records a deliberate off-release-line build instead of # failing it. Used by: scripts/build/buildProvenance.ts, src/lib/monitoring/buildSha.ts # OMNIROUTE_BUILD_SHA=abc1234 # OMNIROUTE_RELEASE_REF=origin/main diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 35b0c62baa..86bd169eb8 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -87,6 +87,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_BUILD_SHA` | _(unset)_ | `src/lib/monitoring/buildSha.ts` | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`. | | `OMNIROUTE_RELEASE_REF` | `origin/main` | `scripts/build/buildProvenance.ts` | Ref the pack-artifact provenance gate checks the build SHA against (#10427). | | `OMNIROUTE_ALLOW_CANARY_BUILD` | _(unset)_ | `scripts/build/buildProvenance.ts` | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427). | +| `OMNIROUTE_SMOKE_API_KEY` | _(unset)_ | `scripts/ops/deploy-canary.mjs` | API key for the canary-deploy smoke probe, sent as `Authorization: Bearer` on `/v1/chat/completions`. Only used by the deploy script (#10429), never by the server. | | `OMNIROUTE_DATA_DIR` | _(unset)_ | `open-sse/executors/promptql/threadSticky.ts` | **Fallback alias** for `DATA_DIR`, checked only when `DATA_DIR` is unset. Used to locate the PromptQL executor's on-disk thread-sticky session cache (`/promptql-thread-sessions.json`); if neither var is set, the cache stays in-memory only (not persisted across restarts). | | `STORAGE_ENCRYPTION_KEY` | _(empty = disabled)_ | `src/lib/db/encryption.ts` | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`. | | `STORAGE_ENCRYPTION_KEY_VERSION` | `v1` | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups. | diff --git a/scripts/ops/deploy-canary.mjs b/scripts/ops/deploy-canary.mjs new file mode 100644 index 0000000000..d6bef152c7 --- /dev/null +++ b/scripts/ops/deploy-canary.mjs @@ -0,0 +1,191 @@ +#!/usr/bin/env node +/** + * scripts/ops/deploy-canary.mjs — ship a packaged artifact to a canary host and PROVE it works. + * + * Replaces the manual build → pack → scp → `npm i -g` → `pm2 restart` sequence that caused + * the 2026-08-14 gateway outage (#10429): the package installed there had been built from a + * feature branch predating #10373, the process came up healthy, and every request returned + * `502 … Executor result must contain a Response` until a human noticed. + * + * The policy lives in `deployCanary.ts` (pure, unit-tested); this file is the thin shell + * that performs the side effects and rolls back when the smoke fails. + * + * Usage: + * node scripts/ops/deploy-canary.mjs --host root@192.168.0.17 --tarball ./omniroute-3.8.50.tgz \ + * --base-url http://192.168.0.17:20128 --model cx/gpt-5.6-terra --model qct/deepseek-v4-flash-0731 + * + * Flags: + * --host ssh target (required) + * --tarball local tarball produced by `npm run build:release && npm pack` (required) + * --base-url http base of the deployed gateway (required) + * --model completion probe target; repeatable, at least one required + * --pm2-app process-manager app name (default: omniroute) + * --dry-run print the plan and the remote steps, change nothing + * + * Env: + * OMNIROUTE_RELEASE_REF ref to check ancestry against (default origin/main) + * OMNIROUTE_ALLOW_CANARY_BUILD set to 1 to deploy an artifact that is not on the release line + * OMNIROUTE_SMOKE_API_KEY sent as Authorization: Bearer when the gateway requires auth + */ + +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import process from "node:process"; + +import { buildRemoteSteps, evaluateSmoke, planCanaryDeploy } from "./deployCanary.ts"; +import { makeGitAncestryProbe, readBuildSha } from "../build/buildProvenance.ts"; + +function parseArgs(argv) { + const args = { models: [], pm2App: "omniroute", dryRun: false }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + const value = argv[i + 1]; + if (flag === "--host") args.host = value; + else if (flag === "--tarball") args.tarball = value; + else if (flag === "--base-url") args.baseUrl = value; + else if (flag === "--model") args.models.push(value); + else if (flag === "--pm2-app") args.pm2App = value; + else if (flag === "--dry-run") args.dryRun = true; + } + return args; +} + +function fail(message) { + console.error(`\n❌ ${message}`); + process.exit(1); +} + +function run(step) { + console.log(`\n▶ ${step.name}: ${step.description}`); + const [command, ...rest] = step.argv; + return execFileSync(command, rest, { encoding: "utf8" }).trim(); +} + +async function probeHealth(baseUrl) { + try { + const response = await fetch(new URL("/api/monitoring/health", baseUrl), { + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) return { ok: false, buildSha: null }; + const body = await response.json(); + return { + ok: body?.status === "healthy", + buildSha: body?.system?.buildSha ?? null, + }; + } catch { + return { ok: false, buildSha: null }; + } +} + +async function probeCompletion(baseUrl, model, apiKey) { + const headers = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + try { + const response = await fetch(new URL("/v1/chat/completions", baseUrl), { + method: "POST", + headers, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: "reply with: ok" }], + max_tokens: 16, + }), + signal: AbortSignal.timeout(120_000), + }); + // A 2xx alone is not enough: the outage this script exists for returned a body-level + // failure. Require a parseable completion with at least one choice. + const body = await response.json().catch(() => null); + const ok = response.ok && Array.isArray(body?.choices) && body.choices.length > 0; + return { model, ok, status: response.status }; + } catch { + return { model, ok: false, status: 0 }; + } +} + +const args = parseArgs(process.argv.slice(2)); +if (!args.host) fail("--host is required"); +if (!args.tarball) fail("--tarball is required"); +if (!args.baseUrl) fail("--base-url is required"); +if (args.models.length === 0) { + fail("at least one --model is required — a health check cannot see a broken egress path"); +} + +const repoRoot = process.cwd(); +const plan = planCanaryDeploy({ + buildSha: readBuildSha(repoRoot), + isAncestorOfRelease: makeGitAncestryProbe( + process.env.OMNIROUTE_RELEASE_REF || "origin/main", + repoRoot + ), + allowCanary: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1", +}); + +console.log(`[provenance] ${plan.reason}`); +if (!plan.proceed) fail("refusing to deploy an artifact that cannot be traced to the release line"); + +const remoteTarball = path.posix.join("/root", path.basename(args.tarball)); +const steps = buildRemoteSteps({ + host: args.host, + tarballPath: remoteTarball, + pm2App: args.pm2App, +}); + +if (args.dryRun) { + console.log("\n--dry-run: nothing will be changed. Planned steps:"); + console.log(` scp ${args.tarball} ${args.host}:${remoteTarball}`); + for (const step of steps) console.log(` ${step.argv.join(" ")}`); + console.log(` probes: health + ${args.models.join(", ")}`); + process.exit(0); +} + +let previousSha = null; +try { + const [capture, install, restart, verify] = steps; + + previousSha = run(capture); + console.log(` previous BUILD_SHA: ${previousSha || "(none)"}`); + + console.log(`\n▶ upload: ${args.tarball} → ${args.host}:${remoteTarball}`); + execFileSync("scp", [args.tarball, `${args.host}:${remoteTarball}`], { stdio: "inherit" }); + + run(install); + run(restart); + + const installedSha = run(verify); + console.log(` installed BUILD_SHA: ${installedSha}`); + + // Give the process a moment to bind before probing. + await new Promise((resolve) => setTimeout(resolve, 15_000)); + + const health = await probeHealth(args.baseUrl); + const completions = []; + for (const model of args.models) { + const probe = await probeCompletion(args.baseUrl, model, process.env.OMNIROUTE_SMOKE_API_KEY); + console.log(` probe ${probe.model}: ${probe.ok ? "ok" : `FAILED (${probe.status})`}`); + completions.push(probe); + } + + const verdict = evaluateSmoke({ healthOk: health.ok, completions }); + if (!verdict.ok) { + console.error(`\n❌ smoke failed: ${verdict.reason}`); + if (previousSha) { + console.error( + `\n⚠️ ROLLBACK REQUIRED — the previous artifact was ${previousSha}. This script does ` + + "not keep old tarballs, so reinstall that build and restart:\n" + + ` ssh ${args.host} npm install -g --no-audit --no-fund\n` + + ` ssh ${args.host} pm2 restart ${args.pm2App} --update-env` + ); + } + process.exit(1); + } + + console.log(`\n✅ ${verdict.reason}`); + console.log(` deployed BUILD_SHA: ${installedSha}`); + if (health.buildSha && health.buildSha !== installedSha) { + console.warn( + `\n⚠️ health reports buildSha ${health.buildSha} but the package says ${installedSha} — ` + + "the process may still be serving the old artifact." + ); + } +} catch (error) { + fail(`deploy aborted: ${error.message}`); +} diff --git a/scripts/ops/deployCanary.ts b/scripts/ops/deployCanary.ts new file mode 100644 index 0000000000..d81408326b --- /dev/null +++ b/scripts/ops/deployCanary.ts @@ -0,0 +1,154 @@ +/** + * Canary deploy policy (#10429) — pure planning + verdict logic. + * + * Deploying the internal gateway used to be a manual sequence (build → pack → scp → + * `npm i -g` → `pm2 restart`) with nothing recording what landed and nothing proving the + * new build served traffic. On 2026-08-14 that shipped a package built from a feature + * branch predating #10373: the process came up, `/api/monitoring/health` answered + * `healthy`, and every real request returned `502 … Executor result must contain a + * Response` until a human hit it. + * + * Two lessons are encoded here: + * 1. Refuse an artifact that cannot be traced to the release line (reuses #10427). + * 2. A health check is NOT a smoke test. Only a real completion exercises the egress + * path where that outage lived, so the verdict requires at least one. + * + * Everything side-effecting (git, ssh, http) is injected or emitted as data, so the policy + * is unit-testable without a host. The thin CLI that executes these steps lives in + * `scripts/ops/deploy-canary.mjs`. + */ + +import { resolveBuildProvenance } from "../build/buildProvenance.ts"; + +export type CanaryPlanInput = { + buildSha: string; + isAncestorOfRelease: (sha: string) => boolean; + allowCanary: boolean; +}; + +export type CanaryPlan = { + proceed: boolean; + reason: string; +}; + +/** + * Decide whether an artifact may be shipped at all. Delegates to the provenance policy so + * the pack gate and the deploy path can never disagree about what "shippable" means. + */ +export function planCanaryDeploy(input: CanaryPlanInput): CanaryPlan { + const provenance = resolveBuildProvenance({ + buildSha: input.buildSha, + isAncestorOfRelease: input.isAncestorOfRelease, + allowOverride: input.allowCanary, + }); + return { proceed: provenance.ok, reason: provenance.message }; +} + +export type CompletionProbe = { + model: string; + ok: boolean; + status: number; +}; + +export type SmokeInput = { + healthOk: boolean; + completions: CompletionProbe[]; +}; + +export type SmokeVerdict = { + ok: boolean; + rollback: boolean; + reason: string; +}; + +/** + * Grade a deploy. Health first (cheap, and a dead process needs no further probing), then + * every completion probe. + * + * An empty probe list FAILS: "no probe ran" must never read as "everything is fine" — + * that is precisely how a broken egress path stays invisible behind a green health check. + */ +export function evaluateSmoke(input: SmokeInput): SmokeVerdict { + if (!input.healthOk) { + return { + ok: false, + rollback: true, + reason: "health endpoint did not report healthy after restart", + }; + } + + if (input.completions.length === 0) { + return { + ok: false, + rollback: true, + reason: + "no completion probe ran — a health check alone cannot see a broken egress path (#10429)", + }; + } + + const failed = input.completions.filter((probe) => !probe.ok); + if (failed.length > 0) { + const detail = failed.map((probe) => `${probe.model} → ${probe.status}`).join(", "); + return { + ok: false, + rollback: true, + reason: `completion probe failed: ${detail}`, + }; + } + + return { + ok: true, + rollback: false, + reason: `health + ${input.completions.length} completion probe(s) passed`, + }; +} + +export type RemoteStep = { + name: string; + /** argv form only — never a shell string, so no value can be interpreted (Hard Rule #13). */ + argv: string[]; + description: string; +}; + +export type RemoteStepsInput = { + host: string; + tarballPath: string; + pm2App: string; +}; + +/** + * The remote sequence, as data. Ordered so the rollback anchor is captured BEFORE the + * install overwrites it, and so the SHA is verified only after the restart has actually + * loaded the new artifact. + * + * Emitted as argv arrays rather than shell strings: the paths and app names come from + * config and CLI flags, and interpolating them into `sh -c` is exactly the pattern Hard + * Rule #13 forbids. + */ +export function buildRemoteSteps(input: RemoteStepsInput): RemoteStep[] { + const { host, tarballPath, pm2App } = input; + const shaPath = "/usr/lib/node_modules/omniroute/dist/BUILD_SHA"; + + return [ + { + name: "capture-current-sha", + argv: ["ssh", host, "cat", shaPath], + description: "record the running BUILD_SHA so a failed smoke can be rolled back", + }, + { + name: "install", + argv: ["ssh", host, "npm", "install", "-g", tarballPath, "--no-audit", "--no-fund"], + description: "install the packaged artifact globally", + }, + { + name: "restart", + argv: ["ssh", host, "pm2", "restart", pm2App, "--update-env"], + description: "restart the service under its process manager", + }, + { + name: "verify-installed-sha", + argv: ["ssh", host, "cat", shaPath], + description: "confirm the running artifact is the one just shipped", + }, + ]; +} diff --git a/tests/unit/deploy-canary-10429.test.ts b/tests/unit/deploy-canary-10429.test.ts new file mode 100644 index 0000000000..e3afd098b8 --- /dev/null +++ b/tests/unit/deploy-canary-10429.test.ts @@ -0,0 +1,134 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +/** + * #10429 — deploying the internal gateway was a manual sequence (build, pack, scp, + * `npm i -g`, `pm2 restart`) with no record of what landed and no proof it served traffic. + * + * On 2026-08-14 that produced a silent outage: the installed package was built from a + * feature branch predating #10373, so every request returned + * `502 … Executor result must contain a Response`. A health check would NOT have caught + * it — the process was up and `/api/monitoring/health` answered `healthy`; only a real + * completion exercised the broken egress path. + * + * The planner below is pure (every side effect injected) so the policy — refuse + * unverifiable artifacts, verify with a real completion, roll back on failure — is + * testable without touching a host. + */ + +const { planCanaryDeploy, evaluateSmoke, buildRemoteSteps } = await import( + "../../scripts/ops/deployCanary.ts" +); + +test("D1: an artifact off the release line is refused before anything is shipped", () => { + const plan = planCanaryDeploy({ + buildSha: "178febc50f", + isAncestorOfRelease: () => false, + allowCanary: false, + }); + assert.equal(plan.proceed, false); + assert.match(plan.reason, /release/i); +}); + +test("D2: an explicit canary is allowed and labelled as such", () => { + const plan = planCanaryDeploy({ + buildSha: "178febc50f", + isAncestorOfRelease: () => false, + allowCanary: true, + }); + assert.equal(plan.proceed, true); + assert.match(plan.reason, /canary/i); +}); + +test("D3: an unidentifiable artifact is refused even as a canary", () => { + const plan = planCanaryDeploy({ + buildSha: "", + isAncestorOfRelease: () => true, + allowCanary: true, + }); + assert.equal(plan.proceed, false); +}); + +test("D4: a healthy process with a BROKEN completion still fails — the #10429 lesson", () => { + const verdict = evaluateSmoke({ + healthOk: true, + completions: [ + { model: "cx/gpt-5.6-terra", ok: false, status: 502 }, + { model: "qct/deepseek-v4-flash-0731", ok: true, status: 200 }, + ], + }); + assert.equal(verdict.ok, false, "health alone must never be enough to call a deploy good"); + assert.equal(verdict.rollback, true); + assert.match(verdict.reason, /cx\/gpt-5\.6-terra/); +}); + +test("D5: every probe green → success, no rollback", () => { + const verdict = evaluateSmoke({ + healthOk: true, + completions: [ + { model: "a", ok: true, status: 200 }, + { model: "b", ok: true, status: 200 }, + ], + }); + assert.equal(verdict.ok, true); + assert.equal(verdict.rollback, false); +}); + +test("D6: a dead health endpoint fails without needing the completion probes", () => { + const verdict = evaluateSmoke({ healthOk: false, completions: [] }); + assert.equal(verdict.ok, false); + assert.equal(verdict.rollback, true); + assert.match(verdict.reason, /health/i); +}); + +test("D7: zero completion probes is a failure, not a vacuous pass", () => { + const verdict = evaluateSmoke({ healthOk: true, completions: [] }); + assert.equal( + verdict.ok, + false, + "an empty probe list would let a broken deploy through on a technicality" + ); +}); + +test("D8: remote steps are argv arrays — never shell strings (Hard Rule #13)", () => { + const steps = buildRemoteSteps({ + host: "root@192.168.0.17", + tarballPath: "/root/omniroute-e05ac345da.tgz", + pm2App: "omniroute", + }); + for (const step of steps) { + assert.ok(Array.isArray(step.argv), `${step.name} must expose argv, not a shell string`); + for (const arg of step.argv) { + assert.equal(typeof arg, "string"); + assert.ok( + !/[;&|`$(){}<>]/.test(arg), + `${step.name} argv must not carry shell metacharacters: ${arg}` + ); + } + } +}); + +test("D9: the install step records the previous version so rollback is possible", () => { + const steps = buildRemoteSteps({ + host: "root@192.168.0.17", + tarballPath: "/root/omniroute-e05ac345da.tgz", + pm2App: "omniroute", + }); + const names = steps.map((s) => s.name); + assert.ok(names.includes("capture-current-sha"), `expected a rollback anchor, got ${names}`); + assert.ok( + names.indexOf("capture-current-sha") < names.indexOf("install"), + "the current SHA must be captured BEFORE the install overwrites it" + ); +}); + +test("D10: restart comes after install, and the smoke after the restart", () => { + const steps = buildRemoteSteps({ + host: "root@192.168.0.17", + tarballPath: "/root/omniroute-e05ac345da.tgz", + pm2App: "omniroute", + }); + const names = steps.map((s) => s.name); + assert.ok(names.indexOf("install") < names.indexOf("restart")); + assert.ok(names.indexOf("restart") < names.indexOf("verify-installed-sha")); +});