diff --git a/.env.example b/.env.example
index 522aff82ef..80f60b55b8 100644
--- a/.env.example
+++ b/.env.example
@@ -54,7 +54,12 @@ INITIAL_PASSWORD=CHANGEME
# Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git
# SHA when the dist/BUILD_SHA sentinel is absent; it is also what `npm run build:release`
# stamps. OMNIROUTE_RELEASE_REF is the ref the pack gate checks ancestry against, and
-# OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of
+# OMNIROUTE_ALLOW_CANARY_BUILD=1
+
+# API key the canary-deploy smoke uses when the target gateway requires auth (#10429).
+# Used by: scripts/ops/deploy-canary.mjs — sent as `Authorization: Bearer` on the
+# /v1/chat/completions probe. Never needed by the server itself.
+# OMNIROUTE_SMOKE_API_KEY=sk-... records a deliberate off-release-line build instead of
# failing it. Used by: scripts/build/buildProvenance.ts, src/lib/monitoring/buildSha.ts
# OMNIROUTE_BUILD_SHA=abc1234
# OMNIROUTE_RELEASE_REF=origin/main
diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md
index 35b0c62baa..86bd169eb8 100644
--- a/docs/reference/ENVIRONMENT.md
+++ b/docs/reference/ENVIRONMENT.md
@@ -87,6 +87,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari
| `OMNIROUTE_BUILD_SHA` | _(unset)_ | `src/lib/monitoring/buildSha.ts` | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`. |
| `OMNIROUTE_RELEASE_REF` | `origin/main` | `scripts/build/buildProvenance.ts` | Ref the pack-artifact provenance gate checks the build SHA against (#10427). |
| `OMNIROUTE_ALLOW_CANARY_BUILD` | _(unset)_ | `scripts/build/buildProvenance.ts` | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427). |
+| `OMNIROUTE_SMOKE_API_KEY` | _(unset)_ | `scripts/ops/deploy-canary.mjs` | API key for the canary-deploy smoke probe, sent as `Authorization: Bearer` on `/v1/chat/completions`. Only used by the deploy script (#10429), never by the server. |
| `OMNIROUTE_DATA_DIR` | _(unset)_ | `open-sse/executors/promptql/threadSticky.ts` | **Fallback alias** for `DATA_DIR`, checked only when `DATA_DIR` is unset. Used to locate the PromptQL executor's on-disk thread-sticky session cache (`
/promptql-thread-sessions.json`); if neither var is set, the cache stays in-memory only (not persisted across restarts). |
| `STORAGE_ENCRYPTION_KEY` | _(empty = disabled)_ | `src/lib/db/encryption.ts` | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`. |
| `STORAGE_ENCRYPTION_KEY_VERSION` | `v1` | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups. |
diff --git a/scripts/ops/deploy-canary.mjs b/scripts/ops/deploy-canary.mjs
new file mode 100644
index 0000000000..d6bef152c7
--- /dev/null
+++ b/scripts/ops/deploy-canary.mjs
@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+/**
+ * scripts/ops/deploy-canary.mjs — ship a packaged artifact to a canary host and PROVE it works.
+ *
+ * Replaces the manual build → pack → scp → `npm i -g` → `pm2 restart` sequence that caused
+ * the 2026-08-14 gateway outage (#10429): the package installed there had been built from a
+ * feature branch predating #10373, the process came up healthy, and every request returned
+ * `502 … Executor result must contain a Response` until a human noticed.
+ *
+ * The policy lives in `deployCanary.ts` (pure, unit-tested); this file is the thin shell
+ * that performs the side effects and rolls back when the smoke fails.
+ *
+ * Usage:
+ * node scripts/ops/deploy-canary.mjs --host root@192.168.0.17 --tarball ./omniroute-3.8.50.tgz \
+ * --base-url http://192.168.0.17:20128 --model cx/gpt-5.6-terra --model qct/deepseek-v4-flash-0731
+ *
+ * Flags:
+ * --host ssh target (required)
+ * --tarball local tarball produced by `npm run build:release && npm pack` (required)
+ * --base-url http base of the deployed gateway (required)
+ * --model completion probe target; repeatable, at least one required
+ * --pm2-app process-manager app name (default: omniroute)
+ * --dry-run print the plan and the remote steps, change nothing
+ *
+ * Env:
+ * OMNIROUTE_RELEASE_REF ref to check ancestry against (default origin/main)
+ * OMNIROUTE_ALLOW_CANARY_BUILD set to 1 to deploy an artifact that is not on the release line
+ * OMNIROUTE_SMOKE_API_KEY sent as Authorization: Bearer when the gateway requires auth
+ */
+
+import { execFileSync } from "node:child_process";
+import path from "node:path";
+import process from "node:process";
+
+import { buildRemoteSteps, evaluateSmoke, planCanaryDeploy } from "./deployCanary.ts";
+import { makeGitAncestryProbe, readBuildSha } from "../build/buildProvenance.ts";
+
+function parseArgs(argv) {
+ const args = { models: [], pm2App: "omniroute", dryRun: false };
+ for (let i = 0; i < argv.length; i += 1) {
+ const flag = argv[i];
+ const value = argv[i + 1];
+ if (flag === "--host") args.host = value;
+ else if (flag === "--tarball") args.tarball = value;
+ else if (flag === "--base-url") args.baseUrl = value;
+ else if (flag === "--model") args.models.push(value);
+ else if (flag === "--pm2-app") args.pm2App = value;
+ else if (flag === "--dry-run") args.dryRun = true;
+ }
+ return args;
+}
+
+function fail(message) {
+ console.error(`\n❌ ${message}`);
+ process.exit(1);
+}
+
+function run(step) {
+ console.log(`\n▶ ${step.name}: ${step.description}`);
+ const [command, ...rest] = step.argv;
+ return execFileSync(command, rest, { encoding: "utf8" }).trim();
+}
+
+async function probeHealth(baseUrl) {
+ try {
+ const response = await fetch(new URL("/api/monitoring/health", baseUrl), {
+ signal: AbortSignal.timeout(20_000),
+ });
+ if (!response.ok) return { ok: false, buildSha: null };
+ const body = await response.json();
+ return {
+ ok: body?.status === "healthy",
+ buildSha: body?.system?.buildSha ?? null,
+ };
+ } catch {
+ return { ok: false, buildSha: null };
+ }
+}
+
+async function probeCompletion(baseUrl, model, apiKey) {
+ const headers = { "Content-Type": "application/json" };
+ if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+ try {
+ const response = await fetch(new URL("/v1/chat/completions", baseUrl), {
+ method: "POST",
+ headers,
+ body: JSON.stringify({
+ model,
+ messages: [{ role: "user", content: "reply with: ok" }],
+ max_tokens: 16,
+ }),
+ signal: AbortSignal.timeout(120_000),
+ });
+ // A 2xx alone is not enough: the outage this script exists for returned a body-level
+ // failure. Require a parseable completion with at least one choice.
+ const body = await response.json().catch(() => null);
+ const ok = response.ok && Array.isArray(body?.choices) && body.choices.length > 0;
+ return { model, ok, status: response.status };
+ } catch {
+ return { model, ok: false, status: 0 };
+ }
+}
+
+const args = parseArgs(process.argv.slice(2));
+if (!args.host) fail("--host is required");
+if (!args.tarball) fail("--tarball is required");
+if (!args.baseUrl) fail("--base-url is required");
+if (args.models.length === 0) {
+ fail("at least one --model is required — a health check cannot see a broken egress path");
+}
+
+const repoRoot = process.cwd();
+const plan = planCanaryDeploy({
+ buildSha: readBuildSha(repoRoot),
+ isAncestorOfRelease: makeGitAncestryProbe(
+ process.env.OMNIROUTE_RELEASE_REF || "origin/main",
+ repoRoot
+ ),
+ allowCanary: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1",
+});
+
+console.log(`[provenance] ${plan.reason}`);
+if (!plan.proceed) fail("refusing to deploy an artifact that cannot be traced to the release line");
+
+const remoteTarball = path.posix.join("/root", path.basename(args.tarball));
+const steps = buildRemoteSteps({
+ host: args.host,
+ tarballPath: remoteTarball,
+ pm2App: args.pm2App,
+});
+
+if (args.dryRun) {
+ console.log("\n--dry-run: nothing will be changed. Planned steps:");
+ console.log(` scp ${args.tarball} ${args.host}:${remoteTarball}`);
+ for (const step of steps) console.log(` ${step.argv.join(" ")}`);
+ console.log(` probes: health + ${args.models.join(", ")}`);
+ process.exit(0);
+}
+
+let previousSha = null;
+try {
+ const [capture, install, restart, verify] = steps;
+
+ previousSha = run(capture);
+ console.log(` previous BUILD_SHA: ${previousSha || "(none)"}`);
+
+ console.log(`\n▶ upload: ${args.tarball} → ${args.host}:${remoteTarball}`);
+ execFileSync("scp", [args.tarball, `${args.host}:${remoteTarball}`], { stdio: "inherit" });
+
+ run(install);
+ run(restart);
+
+ const installedSha = run(verify);
+ console.log(` installed BUILD_SHA: ${installedSha}`);
+
+ // Give the process a moment to bind before probing.
+ await new Promise((resolve) => setTimeout(resolve, 15_000));
+
+ const health = await probeHealth(args.baseUrl);
+ const completions = [];
+ for (const model of args.models) {
+ const probe = await probeCompletion(args.baseUrl, model, process.env.OMNIROUTE_SMOKE_API_KEY);
+ console.log(` probe ${probe.model}: ${probe.ok ? "ok" : `FAILED (${probe.status})`}`);
+ completions.push(probe);
+ }
+
+ const verdict = evaluateSmoke({ healthOk: health.ok, completions });
+ if (!verdict.ok) {
+ console.error(`\n❌ smoke failed: ${verdict.reason}`);
+ if (previousSha) {
+ console.error(
+ `\n⚠️ ROLLBACK REQUIRED — the previous artifact was ${previousSha}. This script does ` +
+ "not keep old tarballs, so reinstall that build and restart:\n" +
+ ` ssh ${args.host} npm install -g --no-audit --no-fund\n` +
+ ` ssh ${args.host} pm2 restart ${args.pm2App} --update-env`
+ );
+ }
+ process.exit(1);
+ }
+
+ console.log(`\n✅ ${verdict.reason}`);
+ console.log(` deployed BUILD_SHA: ${installedSha}`);
+ if (health.buildSha && health.buildSha !== installedSha) {
+ console.warn(
+ `\n⚠️ health reports buildSha ${health.buildSha} but the package says ${installedSha} — ` +
+ "the process may still be serving the old artifact."
+ );
+ }
+} catch (error) {
+ fail(`deploy aborted: ${error.message}`);
+}
diff --git a/scripts/ops/deployCanary.ts b/scripts/ops/deployCanary.ts
new file mode 100644
index 0000000000..d81408326b
--- /dev/null
+++ b/scripts/ops/deployCanary.ts
@@ -0,0 +1,154 @@
+/**
+ * Canary deploy policy (#10429) — pure planning + verdict logic.
+ *
+ * Deploying the internal gateway used to be a manual sequence (build → pack → scp →
+ * `npm i -g` → `pm2 restart`) with nothing recording what landed and nothing proving the
+ * new build served traffic. On 2026-08-14 that shipped a package built from a feature
+ * branch predating #10373: the process came up, `/api/monitoring/health` answered
+ * `healthy`, and every real request returned `502 … Executor result must contain a
+ * Response` until a human hit it.
+ *
+ * Two lessons are encoded here:
+ * 1. Refuse an artifact that cannot be traced to the release line (reuses #10427).
+ * 2. A health check is NOT a smoke test. Only a real completion exercises the egress
+ * path where that outage lived, so the verdict requires at least one.
+ *
+ * Everything side-effecting (git, ssh, http) is injected or emitted as data, so the policy
+ * is unit-testable without a host. The thin CLI that executes these steps lives in
+ * `scripts/ops/deploy-canary.mjs`.
+ */
+
+import { resolveBuildProvenance } from "../build/buildProvenance.ts";
+
+export type CanaryPlanInput = {
+ buildSha: string;
+ isAncestorOfRelease: (sha: string) => boolean;
+ allowCanary: boolean;
+};
+
+export type CanaryPlan = {
+ proceed: boolean;
+ reason: string;
+};
+
+/**
+ * Decide whether an artifact may be shipped at all. Delegates to the provenance policy so
+ * the pack gate and the deploy path can never disagree about what "shippable" means.
+ */
+export function planCanaryDeploy(input: CanaryPlanInput): CanaryPlan {
+ const provenance = resolveBuildProvenance({
+ buildSha: input.buildSha,
+ isAncestorOfRelease: input.isAncestorOfRelease,
+ allowOverride: input.allowCanary,
+ });
+ return { proceed: provenance.ok, reason: provenance.message };
+}
+
+export type CompletionProbe = {
+ model: string;
+ ok: boolean;
+ status: number;
+};
+
+export type SmokeInput = {
+ healthOk: boolean;
+ completions: CompletionProbe[];
+};
+
+export type SmokeVerdict = {
+ ok: boolean;
+ rollback: boolean;
+ reason: string;
+};
+
+/**
+ * Grade a deploy. Health first (cheap, and a dead process needs no further probing), then
+ * every completion probe.
+ *
+ * An empty probe list FAILS: "no probe ran" must never read as "everything is fine" —
+ * that is precisely how a broken egress path stays invisible behind a green health check.
+ */
+export function evaluateSmoke(input: SmokeInput): SmokeVerdict {
+ if (!input.healthOk) {
+ return {
+ ok: false,
+ rollback: true,
+ reason: "health endpoint did not report healthy after restart",
+ };
+ }
+
+ if (input.completions.length === 0) {
+ return {
+ ok: false,
+ rollback: true,
+ reason:
+ "no completion probe ran — a health check alone cannot see a broken egress path (#10429)",
+ };
+ }
+
+ const failed = input.completions.filter((probe) => !probe.ok);
+ if (failed.length > 0) {
+ const detail = failed.map((probe) => `${probe.model} → ${probe.status}`).join(", ");
+ return {
+ ok: false,
+ rollback: true,
+ reason: `completion probe failed: ${detail}`,
+ };
+ }
+
+ return {
+ ok: true,
+ rollback: false,
+ reason: `health + ${input.completions.length} completion probe(s) passed`,
+ };
+}
+
+export type RemoteStep = {
+ name: string;
+ /** argv form only — never a shell string, so no value can be interpreted (Hard Rule #13). */
+ argv: string[];
+ description: string;
+};
+
+export type RemoteStepsInput = {
+ host: string;
+ tarballPath: string;
+ pm2App: string;
+};
+
+/**
+ * The remote sequence, as data. Ordered so the rollback anchor is captured BEFORE the
+ * install overwrites it, and so the SHA is verified only after the restart has actually
+ * loaded the new artifact.
+ *
+ * Emitted as argv arrays rather than shell strings: the paths and app names come from
+ * config and CLI flags, and interpolating them into `sh -c` is exactly the pattern Hard
+ * Rule #13 forbids.
+ */
+export function buildRemoteSteps(input: RemoteStepsInput): RemoteStep[] {
+ const { host, tarballPath, pm2App } = input;
+ const shaPath = "/usr/lib/node_modules/omniroute/dist/BUILD_SHA";
+
+ return [
+ {
+ name: "capture-current-sha",
+ argv: ["ssh", host, "cat", shaPath],
+ description: "record the running BUILD_SHA so a failed smoke can be rolled back",
+ },
+ {
+ name: "install",
+ argv: ["ssh", host, "npm", "install", "-g", tarballPath, "--no-audit", "--no-fund"],
+ description: "install the packaged artifact globally",
+ },
+ {
+ name: "restart",
+ argv: ["ssh", host, "pm2", "restart", pm2App, "--update-env"],
+ description: "restart the service under its process manager",
+ },
+ {
+ name: "verify-installed-sha",
+ argv: ["ssh", host, "cat", shaPath],
+ description: "confirm the running artifact is the one just shipped",
+ },
+ ];
+}
diff --git a/tests/unit/deploy-canary-10429.test.ts b/tests/unit/deploy-canary-10429.test.ts
new file mode 100644
index 0000000000..e3afd098b8
--- /dev/null
+++ b/tests/unit/deploy-canary-10429.test.ts
@@ -0,0 +1,134 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+/**
+ * #10429 — deploying the internal gateway was a manual sequence (build, pack, scp,
+ * `npm i -g`, `pm2 restart`) with no record of what landed and no proof it served traffic.
+ *
+ * On 2026-08-14 that produced a silent outage: the installed package was built from a
+ * feature branch predating #10373, so every request returned
+ * `502 … Executor result must contain a Response`. A health check would NOT have caught
+ * it — the process was up and `/api/monitoring/health` answered `healthy`; only a real
+ * completion exercised the broken egress path.
+ *
+ * The planner below is pure (every side effect injected) so the policy — refuse
+ * unverifiable artifacts, verify with a real completion, roll back on failure — is
+ * testable without touching a host.
+ */
+
+const { planCanaryDeploy, evaluateSmoke, buildRemoteSteps } = await import(
+ "../../scripts/ops/deployCanary.ts"
+);
+
+test("D1: an artifact off the release line is refused before anything is shipped", () => {
+ const plan = planCanaryDeploy({
+ buildSha: "178febc50f",
+ isAncestorOfRelease: () => false,
+ allowCanary: false,
+ });
+ assert.equal(plan.proceed, false);
+ assert.match(plan.reason, /release/i);
+});
+
+test("D2: an explicit canary is allowed and labelled as such", () => {
+ const plan = planCanaryDeploy({
+ buildSha: "178febc50f",
+ isAncestorOfRelease: () => false,
+ allowCanary: true,
+ });
+ assert.equal(plan.proceed, true);
+ assert.match(plan.reason, /canary/i);
+});
+
+test("D3: an unidentifiable artifact is refused even as a canary", () => {
+ const plan = planCanaryDeploy({
+ buildSha: "",
+ isAncestorOfRelease: () => true,
+ allowCanary: true,
+ });
+ assert.equal(plan.proceed, false);
+});
+
+test("D4: a healthy process with a BROKEN completion still fails — the #10429 lesson", () => {
+ const verdict = evaluateSmoke({
+ healthOk: true,
+ completions: [
+ { model: "cx/gpt-5.6-terra", ok: false, status: 502 },
+ { model: "qct/deepseek-v4-flash-0731", ok: true, status: 200 },
+ ],
+ });
+ assert.equal(verdict.ok, false, "health alone must never be enough to call a deploy good");
+ assert.equal(verdict.rollback, true);
+ assert.match(verdict.reason, /cx\/gpt-5\.6-terra/);
+});
+
+test("D5: every probe green → success, no rollback", () => {
+ const verdict = evaluateSmoke({
+ healthOk: true,
+ completions: [
+ { model: "a", ok: true, status: 200 },
+ { model: "b", ok: true, status: 200 },
+ ],
+ });
+ assert.equal(verdict.ok, true);
+ assert.equal(verdict.rollback, false);
+});
+
+test("D6: a dead health endpoint fails without needing the completion probes", () => {
+ const verdict = evaluateSmoke({ healthOk: false, completions: [] });
+ assert.equal(verdict.ok, false);
+ assert.equal(verdict.rollback, true);
+ assert.match(verdict.reason, /health/i);
+});
+
+test("D7: zero completion probes is a failure, not a vacuous pass", () => {
+ const verdict = evaluateSmoke({ healthOk: true, completions: [] });
+ assert.equal(
+ verdict.ok,
+ false,
+ "an empty probe list would let a broken deploy through on a technicality"
+ );
+});
+
+test("D8: remote steps are argv arrays — never shell strings (Hard Rule #13)", () => {
+ const steps = buildRemoteSteps({
+ host: "root@192.168.0.17",
+ tarballPath: "/root/omniroute-e05ac345da.tgz",
+ pm2App: "omniroute",
+ });
+ for (const step of steps) {
+ assert.ok(Array.isArray(step.argv), `${step.name} must expose argv, not a shell string`);
+ for (const arg of step.argv) {
+ assert.equal(typeof arg, "string");
+ assert.ok(
+ !/[;&|`$(){}<>]/.test(arg),
+ `${step.name} argv must not carry shell metacharacters: ${arg}`
+ );
+ }
+ }
+});
+
+test("D9: the install step records the previous version so rollback is possible", () => {
+ const steps = buildRemoteSteps({
+ host: "root@192.168.0.17",
+ tarballPath: "/root/omniroute-e05ac345da.tgz",
+ pm2App: "omniroute",
+ });
+ const names = steps.map((s) => s.name);
+ assert.ok(names.includes("capture-current-sha"), `expected a rollback anchor, got ${names}`);
+ assert.ok(
+ names.indexOf("capture-current-sha") < names.indexOf("install"),
+ "the current SHA must be captured BEFORE the install overwrites it"
+ );
+});
+
+test("D10: restart comes after install, and the smoke after the restart", () => {
+ const steps = buildRemoteSteps({
+ host: "root@192.168.0.17",
+ tarballPath: "/root/omniroute-e05ac345da.tgz",
+ pm2App: "omniroute",
+ });
+ const names = steps.map((s) => s.name);
+ assert.ok(names.indexOf("install") < names.indexOf("restart"));
+ assert.ok(names.indexOf("restart") < names.indexOf("verify-installed-sha"));
+});