fix(build): verify artifact provenance and expose buildSha on health (#10444)

The packaged artifact stamped dist/BUILD_SHA but nothing verified the SHA belonged to the release line, so a tarball built from a feature branch installed and served traffic indistinguishably from a release build. That is how the internal gateway ended up running a build that predated #10373 and answered every request with 502 'Executor result must contain a Response' — identifying it required SSH plus grepping the compiled chunks.

scripts/build/buildProvenance.ts classifies a build SHA against the release ref (pure functions, injected git probe). A missing SHA fails even with the canary override: an unidentifiable artifact cannot be vouched for. validate-pack-artifact enforces it on real packs (skipped under --policy-only, which runs without a build); OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of failing it. /api/monitoring/health now exposes system.buildSha — absent when unknown, never fabricated.

Closes #10427

.env.example

@@ -51,6 +51,15 @@ INITIAL_PASSWORD=CHANGEME
 # DATA_DIR — never for CI. Used by: src/lib/dataPaths.ts
 # OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1
 
+# Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git
+# SHA when the dist/BUILD_SHA sentinel is absent; it is also what `npm run build:release`
+# stamps. OMNIROUTE_RELEASE_REF is the ref the pack gate checks ancestry against, and
+# OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of
+# failing it. Used by: scripts/build/buildProvenance.ts, src/lib/monitoring/buildSha.ts
+# OMNIROUTE_BUILD_SHA=abc1234
+# OMNIROUTE_RELEASE_REF=origin/main
+# OMNIROUTE_ALLOW_CANARY_BUILD=1
+
 # Encryption key for SQLite database encryption at rest.
 # Used by: src/lib/db/encryption.ts — encrypts the entire SQLite database.
 # Generate: openssl rand -hex 32  |  Leave empty to disable DB encryption.

docs/reference/ENVIRONMENT.md

@@ -84,6 +84,9 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari
 | -------------------------------------- | -------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `DATA_DIR`                             | `~/.omniroute/`      | `src/lib/db/core.ts`                                  | Root directory for SQLite DB, backups, and data files. Override for Docker volumes or custom paths.                                                                                                                                             |
 | `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR`      | _(unset)_            | `src/lib/dataPaths.ts`                                | Escape hatch for the test-context DATA_DIR guard (#10428). Test runs with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory.                  |
+| `OMNIROUTE_BUILD_SHA`                   | _(unset)_            | `src/lib/monitoring/buildSha.ts`                      | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`.                                          |
+| `OMNIROUTE_RELEASE_REF`                 | `origin/main`        | `scripts/build/buildProvenance.ts`                    | Ref the pack-artifact provenance gate checks the build SHA against (#10427).                                                                                                                                                                     |
+| `OMNIROUTE_ALLOW_CANARY_BUILD`          | _(unset)_            | `scripts/build/buildProvenance.ts`                    | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427).                                                                                               |
 | `OMNIROUTE_DATA_DIR`                   | _(unset)_            | `open-sse/executors/promptql/threadSticky.ts`         | **Fallback alias** for `DATA_DIR`, checked only when `DATA_DIR` is unset. Used to locate the PromptQL executor's on-disk thread-sticky session cache (`<dir>/promptql-thread-sessions.json`); if neither var is set, the cache stays in-memory only (not persisted across restarts).                                                                                                                                             |
 | `STORAGE_ENCRYPTION_KEY`               | _(empty = disabled)_ | `src/lib/db/encryption.ts`                            | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`.                                                                                                                                                      |
 | `STORAGE_ENCRYPTION_KEY_VERSION`       | `v1`                 | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups.                                                                                                                              |

scripts/build/buildProvenance.ts

@@ -0,0 +1,120 @@
+/**
+ * Build provenance — is this artifact actually built from the release line? (#10427)
+ *
+ * `scripts/build/write-build-sha.mjs` stamps `dist/BUILD_SHA` into every packaged build,
+ * but nothing ever verified that the SHA belongs to the release branch. A tarball built
+ * from a feature branch installs and serves traffic indistinguishably from a release one.
+ *
+ * That gap took down the internal gateway on 2026-08-14: the installed package carried
+ * `BUILD_SHA = 178febc50f`, a commit on `fix/9603-qwen-token-plan-quota` that predated
+ * #10373, so it shipped the nominal `instanceof Response` guard from #10256 and answered
+ * every request with `502 … Executor result must contain a Response`.
+ *
+ * Kept as pure functions (the ancestry probe is injected) so the policy is unit-testable
+ * without a git fixture, and so the caller decides how strict to be per environment.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+
+export type BuildProvenanceReason =
+  | "on-release-line"
+  | "off-release-line"
+  | "canary-override"
+  | "missing-sha";
+
+export type BuildProvenanceResult = {
+  ok: boolean;
+  reason: BuildProvenanceReason;
+  message: string;
+};
+
+export type BuildProvenanceInput = {
+  /** Contents of `dist/BUILD_SHA` (empty when the sentinel is absent). */
+  buildSha: string;
+  /** Whether `buildSha` is an ancestor of the release ref. Injected so this stays pure. */
+  isAncestorOfRelease: (sha: string) => boolean;
+  /** Deliberate canary build — allowed, but always reported. */
+  allowOverride: boolean;
+};
+
+/**
+ * Read `dist/BUILD_SHA` from a package root. Returns "" when absent — an unstamped build
+ * is a policy decision for the caller, not an exception here.
+ */
+export function readBuildSha(packageRoot: string): string {
+  try {
+    return fs.readFileSync(path.join(packageRoot, "dist", "BUILD_SHA"), "utf8").trim();
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Classify a build SHA against the release line.
+ *
+ * A missing SHA fails even with the override on: an artifact that cannot be identified
+ * cannot be vouched for, and "canary" is a statement about a KNOWN commit.
+ */
+export function resolveBuildProvenance(input: BuildProvenanceInput): BuildProvenanceResult {
+  const { buildSha, isAncestorOfRelease, allowOverride } = input;
+
+  if (!buildSha) {
+    return {
+      ok: false,
+      reason: "missing-sha",
+      message:
+        "dist/BUILD_SHA is missing — the artifact cannot be traced to a commit. " +
+        "Build with `npm run build:release` (or run scripts/build/write-build-sha.mjs).",
+    };
+  }
+
+  if (isAncestorOfRelease(buildSha)) {
+    return {
+      ok: true,
+      reason: "on-release-line",
+      message: `BUILD_SHA ${buildSha} is on the release line.`,
+    };
+  }
+
+  if (allowOverride) {
+    return {
+      ok: true,
+      reason: "canary-override",
+      message:
+        `BUILD_SHA ${buildSha} is NOT on the release line — allowed as a canary build ` +
+        "because OMNIROUTE_ALLOW_CANARY_BUILD=1 was set.",
+    };
+  }
+
+  return {
+    ok: false,
+    reason: "off-release-line",
+    message:
+      `BUILD_SHA ${buildSha} is not an ancestor of the release branch. Shipping it means ` +
+      "serving code that never passed the release gates (see #10427). Rebuild from the " +
+      "release tip, or set OMNIROUTE_ALLOW_CANARY_BUILD=1 to record this as a deliberate canary.",
+  };
+}
+
+/**
+ * Default ancestry probe: `git merge-base --is-ancestor <sha> <releaseRef>`.
+ *
+ * Any git failure (shallow clone, unknown ref, SHA not fetched) resolves to `false` —
+ * "cannot prove it is on the release line" is the safe answer for a gate whose whole
+ * purpose is to refuse unverifiable artifacts.
+ */
+export function makeGitAncestryProbe(releaseRef: string, cwd: string): (sha: string) => boolean {
+  return (sha: string) => {
+    try {
+      execFileSync("git", ["merge-base", "--is-ancestor", sha, releaseRef], {
+        cwd,
+        stdio: "ignore",
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  };
+}

scripts/build/validate-pack-artifact.ts

@@ -4,6 +4,11 @@ import { execFileSync, spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import {
+  makeGitAncestryProbe,
+  readBuildSha,
+  resolveBuildProvenance,
+} from "./buildProvenance.ts";
 
 import {
   MCP_CLOSURE_SPOT_CHECK_PATH,
@@ -204,6 +209,26 @@ try {
     process.exit(1);
   }
 
+  // #10427: an artifact is only shippable if it can be traced to the release line. The
+  // 2026-08-14 gateway outage was a package built from a feature branch that predated the
+  // fix it was supposed to carry — nothing in this gate noticed. Skipped under
+  // --policy-only, which deliberately runs without a build (no dist/BUILD_SHA to check).
+  if (!POLICY_ONLY) {
+    const provenance = resolveBuildProvenance({
+      buildSha: readBuildSha(process.cwd()),
+      isAncestorOfRelease: makeGitAncestryProbe(
+        process.env.OMNIROUTE_RELEASE_REF || "origin/main",
+        process.cwd()
+      ),
+      allowOverride: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1",
+    });
+    console.log(`\n[provenance] ${provenance.message}`);
+    if (!provenance.ok) {
+      console.error("\n❌ Build provenance check failed.");
+      process.exit(1);
+    }
+  }
+
   console.log("\n✅ Pack artifact policy check passed.");
 } catch (error) {
   console.error(`\n❌ Pack artifact validation failed: ${error.message}`);

src/app/api/monitoring/health/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from "next/server";
 import { getProviderConnections, getCachedSettings } from "@/lib/localDb";
 import { buildHealthPayload } from "@/lib/monitoring/observability";
+import { readRunningBuildSha } from "@/lib/monitoring/buildSha";
 import { APP_CONFIG } from "@/shared/constants/config";
 import { AI_PROVIDERS } from "@/shared/constants/providers";
 import { isAuthenticated } from "@/shared/utils/apiAuth";
@@ -158,6 +159,9 @@ export async function GET() {
 
     const payload = buildHealthPayload({
       appVersion: APP_CONFIG.version,
+      // #10427: surface the artifact's git SHA so a deployment can be audited over HTTP
+      // instead of SSH + grepping compiled chunks (the 2026-08-14 gateway outage).
+      buildSha: readRunningBuildSha(),
       catalogCount: Object.keys(AI_PROVIDERS).length,
       settings,
       connections,

src/lib/monitoring/buildSha.ts

@@ -0,0 +1,52 @@
+/**
+ * Runtime build identity (#10427).
+ *
+ * `scripts/build/write-build-sha.mjs` stamps the git SHA into `dist/BUILD_SHA` and
+ * `.build/next/standalone/BUILD_SHA` at release-build time. Reading it back at runtime is
+ * what lets an operator answer "what code is this box actually running?" over HTTP.
+ *
+ * Before this existed, answering that question during the 2026-08-14 gateway outage meant
+ * SSH-ing into the host and grepping the compiled Next chunks — the deployed package
+ * turned out to be built from a feature branch that predated the fix it was supposed to
+ * carry.
+ *
+ * Resolution order: explicit env var (containers can inject it without the sentinel file),
+ * then the sentinel files relative to the working directory. Unknown → `null`, never a
+ * fabricated or guessed value.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+const SENTINEL_PATHS = [
+  ["dist", "BUILD_SHA"],
+  [".build", "next", "standalone", "BUILD_SHA"],
+  ["BUILD_SHA"],
+];
+
+let cached: string | null | undefined;
+
+export function readRunningBuildSha(cwd: string = process.cwd()): string | null {
+  if (cached !== undefined) return cached;
+
+  const fromEnv = process.env.OMNIROUTE_BUILD_SHA?.trim();
+  if (fromEnv) {
+    cached = fromEnv;
+    return cached;
+  }
+
+  for (const segments of SENTINEL_PATHS) {
+    try {
+      const value = fs.readFileSync(path.join(cwd, ...segments), "utf8").trim();
+      if (value) {
+        cached = value;
+        return cached;
+      }
+    } catch {
+      // Sentinel absent at this location — try the next one. A dev run has none of them.
+    }
+  }
+
+  cached = null;
+  return cached;
+}

src/lib/monitoring/observability.ts

@@ -126,6 +126,8 @@ interface BuildTelemetryPayloadOptions {
 
 interface BuildHealthPayloadOptions {
   appVersion: string;
+  /** #10427: git SHA the running artifact was built from, so a deploy is auditable over HTTP. */
+  buildSha?: string | null;
   catalogCount?: number;
   settings: { setupComplete?: boolean } | null | undefined;
   connections: Array<{ provider?: string; isActive?: boolean | null; rateLimitedUntil?: unknown }>;
@@ -288,10 +290,14 @@ export function buildHealthPayload({
   activeSessionsByKey = {},
   credentialHealth,
   adaptiveAdmission = null,
+  buildSha = null,
 }: BuildHealthPayloadOptions) {
   const timestamp = new Date().toISOString();
   const system = {
     version: appVersion,
+    // #10427: identifying a bad deploy previously required SSH + grepping compiled chunks.
+    // Absent/empty when unknown (dev runs) — never a fabricated value.
+    ...(buildSha ? { buildSha } : {}),
     nodeVersion: process.version,
     uptime: process.uptime(),
     memoryUsage: process.memoryUsage(),

tests/unit/build-sha-provenance-10427.test.ts

@@ -0,0 +1,135 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+/**
+ * #10427 — the packaged artifact carries `dist/BUILD_SHA`, but nothing ever checked that
+ * the SHA belongs to the release line. A tarball built from a feature branch installs and
+ * runs indistinguishably from a release build.
+ *
+ * That is exactly how the internal gateway ended up serving a build from
+ * `fix/9603-qwen-token-plan-quota` (SHA 178febc50f) that predated #10373: every request
+ * died with `502 … Executor result must contain a Response`, and the only way to find out
+ * what was actually deployed was SSH + grepping the compiled chunks.
+ *
+ * Two defenses, both covered here:
+ *  - `resolveBuildProvenance()` classifies a build SHA against the release line.
+ *  - the health payload exposes `buildSha`, so what is deployed is auditable over HTTP.
+ */
+
+const { resolveBuildProvenance } = await import("../../scripts/build/buildProvenance.ts");
+const { buildHealthPayload } = await import("../../src/lib/monitoring/observability.ts");
+
+function makeRepo(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-prov-"));
+  return dir;
+}
+
+test("P1: a SHA on the release line is accepted", () => {
+  const result = resolveBuildProvenance({
+    buildSha: "abc1234",
+    isAncestorOfRelease: () => true,
+    allowOverride: false,
+  });
+  assert.equal(result.ok, true);
+  assert.equal(result.reason, "on-release-line");
+});
+
+test("P2: a SHA from an unrelated branch is REJECTED — the #10427 incident", () => {
+  const result = resolveBuildProvenance({
+    buildSha: "178febc50f",
+    isAncestorOfRelease: () => false,
+    allowOverride: false,
+  });
+  assert.equal(result.ok, false);
+  assert.equal(result.reason, "off-release-line");
+  assert.match(
+    result.message,
+    /release/i,
+    "the failure must say what is wrong so a human can act on it"
+  );
+});
+
+test("P3: an explicit canary override is allowed but recorded, never silent", () => {
+  const result = resolveBuildProvenance({
+    buildSha: "178febc50f",
+    isAncestorOfRelease: () => false,
+    allowOverride: true,
+  });
+  assert.equal(result.ok, true);
+  assert.equal(result.reason, "canary-override");
+  assert.match(result.message, /canary/i);
+});
+
+test("P4: a missing BUILD_SHA is a failure, not a pass-by-default", () => {
+  const result = resolveBuildProvenance({
+    buildSha: "",
+    isAncestorOfRelease: () => true,
+    allowOverride: false,
+  });
+  assert.equal(result.ok, false);
+  assert.equal(result.reason, "missing-sha");
+});
+
+test("P5: a missing BUILD_SHA is not excused by the canary override either", () => {
+  const result = resolveBuildProvenance({
+    buildSha: "",
+    isAncestorOfRelease: () => false,
+    allowOverride: true,
+  });
+  assert.equal(result.ok, false, "an unidentifiable artifact can never be validated");
+  assert.equal(result.reason, "missing-sha");
+});
+
+test("P6: readBuildSha returns the trimmed sentinel, or empty when absent", async () => {
+  const { readBuildSha } = await import("../../scripts/build/buildProvenance.ts");
+  const repo = makeRepo();
+  try {
+    assert.equal(readBuildSha(repo), "", "no dist/BUILD_SHA → empty, never a throw");
+    fs.mkdirSync(path.join(repo, "dist"), { recursive: true });
+    fs.writeFileSync(path.join(repo, "dist", "BUILD_SHA"), "e05ac345da\n");
+    assert.equal(readBuildSha(repo), "e05ac345da");
+  } finally {
+    fs.rmSync(repo, { recursive: true, force: true });
+  }
+});
+
+/** Minimal but complete options for buildHealthPayload — only `buildSha` varies below. */
+function healthOptions(buildSha?: string) {
+  return {
+    appVersion: "3.8.50",
+    ...(buildSha === undefined ? {} : { buildSha }),
+    settings: { setupComplete: true },
+    connections: [],
+    circuitBreakers: [],
+    rateLimitStatus: {},
+    learnedLimits: {},
+    lockouts: {},
+    localProviders: {},
+    inflightRequests: 0,
+    quotaMonitorSummary: {},
+    quotaMonitorMonitors: [],
+    activeSessions: [],
+  } as unknown as Parameters<typeof buildHealthPayload>[0];
+}
+
+test("P7: the health payload exposes buildSha so deployments are auditable over HTTP", () => {
+  const payload = buildHealthPayload(healthOptions("e05ac345da"));
+  assert.equal(
+    (payload.system as { buildSha?: string }).buildSha,
+    "e05ac345da",
+    "without this, identifying a bad deploy needs SSH + grepping compiled chunks"
+  );
+});
+
+test("P8: health stays valid when no buildSha is known (dev runs)", () => {
+  const payload = buildHealthPayload(healthOptions());
+  const system = payload.system as { version?: string; buildSha?: string };
+  assert.equal(system.version, "3.8.50", "the existing contract must not regress");
+  assert.ok(
+    system.buildSha === undefined || system.buildSha === null || system.buildSha === "",
+    "an unknown build SHA must be absent/empty, never a fabricated value"
+  );
+});