diff --git a/.env.example b/.env.example index 5d1f674607..522aff82ef 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,15 @@ INITIAL_PASSWORD=CHANGEME # DATA_DIR — never for CI. Used by: src/lib/dataPaths.ts # OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1 +# Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git +# SHA when the dist/BUILD_SHA sentinel is absent; it is also what `npm run build:release` +# stamps. OMNIROUTE_RELEASE_REF is the ref the pack gate checks ancestry against, and +# OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of +# failing it. Used by: scripts/build/buildProvenance.ts, src/lib/monitoring/buildSha.ts +# OMNIROUTE_BUILD_SHA=abc1234 +# OMNIROUTE_RELEASE_REF=origin/main +# OMNIROUTE_ALLOW_CANARY_BUILD=1 + # Encryption key for SQLite database encryption at rest. # Used by: src/lib/db/encryption.ts — encrypts the entire SQLite database. # Generate: openssl rand -hex 32 | Leave empty to disable DB encryption. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 2cab810ee9..35b0c62baa 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -84,6 +84,9 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | -------------------------------------- | -------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `DATA_DIR` | `~/.omniroute/` | `src/lib/db/core.ts` | Root directory for SQLite DB, backups, and data files. Override for Docker volumes or custom paths. | | `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR` | _(unset)_ | `src/lib/dataPaths.ts` | Escape hatch for the test-context DATA_DIR guard (#10428). Test runs with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory. | +| `OMNIROUTE_BUILD_SHA` | _(unset)_ | `src/lib/monitoring/buildSha.ts` | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`. | +| `OMNIROUTE_RELEASE_REF` | `origin/main` | `scripts/build/buildProvenance.ts` | Ref the pack-artifact provenance gate checks the build SHA against (#10427). | +| `OMNIROUTE_ALLOW_CANARY_BUILD` | _(unset)_ | `scripts/build/buildProvenance.ts` | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427). | | `OMNIROUTE_DATA_DIR` | _(unset)_ | `open-sse/executors/promptql/threadSticky.ts` | **Fallback alias** for `DATA_DIR`, checked only when `DATA_DIR` is unset. Used to locate the PromptQL executor's on-disk thread-sticky session cache (`/promptql-thread-sessions.json`); if neither var is set, the cache stays in-memory only (not persisted across restarts). | | `STORAGE_ENCRYPTION_KEY` | _(empty = disabled)_ | `src/lib/db/encryption.ts` | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`. | | `STORAGE_ENCRYPTION_KEY_VERSION` | `v1` | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups. | diff --git a/scripts/build/buildProvenance.ts b/scripts/build/buildProvenance.ts new file mode 100644 index 0000000000..27d818021c --- /dev/null +++ b/scripts/build/buildProvenance.ts @@ -0,0 +1,120 @@ +/** + * Build provenance — is this artifact actually built from the release line? (#10427) + * + * `scripts/build/write-build-sha.mjs` stamps `dist/BUILD_SHA` into every packaged build, + * but nothing ever verified that the SHA belongs to the release branch. A tarball built + * from a feature branch installs and serves traffic indistinguishably from a release one. + * + * That gap took down the internal gateway on 2026-08-14: the installed package carried + * `BUILD_SHA = 178febc50f`, a commit on `fix/9603-qwen-token-plan-quota` that predated + * #10373, so it shipped the nominal `instanceof Response` guard from #10256 and answered + * every request with `502 … Executor result must contain a Response`. + * + * Kept as pure functions (the ancestry probe is injected) so the policy is unit-testable + * without a git fixture, and so the caller decides how strict to be per environment. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; + +export type BuildProvenanceReason = + | "on-release-line" + | "off-release-line" + | "canary-override" + | "missing-sha"; + +export type BuildProvenanceResult = { + ok: boolean; + reason: BuildProvenanceReason; + message: string; +}; + +export type BuildProvenanceInput = { + /** Contents of `dist/BUILD_SHA` (empty when the sentinel is absent). */ + buildSha: string; + /** Whether `buildSha` is an ancestor of the release ref. Injected so this stays pure. */ + isAncestorOfRelease: (sha: string) => boolean; + /** Deliberate canary build — allowed, but always reported. */ + allowOverride: boolean; +}; + +/** + * Read `dist/BUILD_SHA` from a package root. Returns "" when absent — an unstamped build + * is a policy decision for the caller, not an exception here. + */ +export function readBuildSha(packageRoot: string): string { + try { + return fs.readFileSync(path.join(packageRoot, "dist", "BUILD_SHA"), "utf8").trim(); + } catch { + return ""; + } +} + +/** + * Classify a build SHA against the release line. + * + * A missing SHA fails even with the override on: an artifact that cannot be identified + * cannot be vouched for, and "canary" is a statement about a KNOWN commit. + */ +export function resolveBuildProvenance(input: BuildProvenanceInput): BuildProvenanceResult { + const { buildSha, isAncestorOfRelease, allowOverride } = input; + + if (!buildSha) { + return { + ok: false, + reason: "missing-sha", + message: + "dist/BUILD_SHA is missing — the artifact cannot be traced to a commit. " + + "Build with `npm run build:release` (or run scripts/build/write-build-sha.mjs).", + }; + } + + if (isAncestorOfRelease(buildSha)) { + return { + ok: true, + reason: "on-release-line", + message: `BUILD_SHA ${buildSha} is on the release line.`, + }; + } + + if (allowOverride) { + return { + ok: true, + reason: "canary-override", + message: + `BUILD_SHA ${buildSha} is NOT on the release line — allowed as a canary build ` + + "because OMNIROUTE_ALLOW_CANARY_BUILD=1 was set.", + }; + } + + return { + ok: false, + reason: "off-release-line", + message: + `BUILD_SHA ${buildSha} is not an ancestor of the release branch. Shipping it means ` + + "serving code that never passed the release gates (see #10427). Rebuild from the " + + "release tip, or set OMNIROUTE_ALLOW_CANARY_BUILD=1 to record this as a deliberate canary.", + }; +} + +/** + * Default ancestry probe: `git merge-base --is-ancestor `. + * + * Any git failure (shallow clone, unknown ref, SHA not fetched) resolves to `false` — + * "cannot prove it is on the release line" is the safe answer for a gate whose whole + * purpose is to refuse unverifiable artifacts. + */ +export function makeGitAncestryProbe(releaseRef: string, cwd: string): (sha: string) => boolean { + return (sha: string) => { + try { + execFileSync("git", ["merge-base", "--is-ancestor", sha, releaseRef], { + cwd, + stdio: "ignore", + }); + return true; + } catch { + return false; + } + }; +} diff --git a/scripts/build/validate-pack-artifact.ts b/scripts/build/validate-pack-artifact.ts index 4c0085846a..d0d30fbbd6 100644 --- a/scripts/build/validate-pack-artifact.ts +++ b/scripts/build/validate-pack-artifact.ts @@ -4,6 +4,11 @@ import { execFileSync, spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { + makeGitAncestryProbe, + readBuildSha, + resolveBuildProvenance, +} from "./buildProvenance.ts"; import { MCP_CLOSURE_SPOT_CHECK_PATH, @@ -204,6 +209,26 @@ try { process.exit(1); } + // #10427: an artifact is only shippable if it can be traced to the release line. The + // 2026-08-14 gateway outage was a package built from a feature branch that predated the + // fix it was supposed to carry — nothing in this gate noticed. Skipped under + // --policy-only, which deliberately runs without a build (no dist/BUILD_SHA to check). + if (!POLICY_ONLY) { + const provenance = resolveBuildProvenance({ + buildSha: readBuildSha(process.cwd()), + isAncestorOfRelease: makeGitAncestryProbe( + process.env.OMNIROUTE_RELEASE_REF || "origin/main", + process.cwd() + ), + allowOverride: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1", + }); + console.log(`\n[provenance] ${provenance.message}`); + if (!provenance.ok) { + console.error("\n❌ Build provenance check failed."); + process.exit(1); + } + } + console.log("\n✅ Pack artifact policy check passed."); } catch (error) { console.error(`\n❌ Pack artifact validation failed: ${error.message}`); diff --git a/src/app/api/monitoring/health/route.ts b/src/app/api/monitoring/health/route.ts index 8d7f5f9b2c..144d0a3ce0 100644 --- a/src/app/api/monitoring/health/route.ts +++ b/src/app/api/monitoring/health/route.ts @@ -1,6 +1,7 @@ import { NextResponse } from "next/server"; import { getProviderConnections, getCachedSettings } from "@/lib/localDb"; import { buildHealthPayload } from "@/lib/monitoring/observability"; +import { readRunningBuildSha } from "@/lib/monitoring/buildSha"; import { APP_CONFIG } from "@/shared/constants/config"; import { AI_PROVIDERS } from "@/shared/constants/providers"; import { isAuthenticated } from "@/shared/utils/apiAuth"; @@ -158,6 +159,9 @@ export async function GET() { const payload = buildHealthPayload({ appVersion: APP_CONFIG.version, + // #10427: surface the artifact's git SHA so a deployment can be audited over HTTP + // instead of SSH + grepping compiled chunks (the 2026-08-14 gateway outage). + buildSha: readRunningBuildSha(), catalogCount: Object.keys(AI_PROVIDERS).length, settings, connections, diff --git a/src/lib/monitoring/buildSha.ts b/src/lib/monitoring/buildSha.ts new file mode 100644 index 0000000000..c43fba7749 --- /dev/null +++ b/src/lib/monitoring/buildSha.ts @@ -0,0 +1,52 @@ +/** + * Runtime build identity (#10427). + * + * `scripts/build/write-build-sha.mjs` stamps the git SHA into `dist/BUILD_SHA` and + * `.build/next/standalone/BUILD_SHA` at release-build time. Reading it back at runtime is + * what lets an operator answer "what code is this box actually running?" over HTTP. + * + * Before this existed, answering that question during the 2026-08-14 gateway outage meant + * SSH-ing into the host and grepping the compiled Next chunks — the deployed package + * turned out to be built from a feature branch that predated the fix it was supposed to + * carry. + * + * Resolution order: explicit env var (containers can inject it without the sentinel file), + * then the sentinel files relative to the working directory. Unknown → `null`, never a + * fabricated or guessed value. + */ + +import fs from "node:fs"; +import path from "node:path"; + +const SENTINEL_PATHS = [ + ["dist", "BUILD_SHA"], + [".build", "next", "standalone", "BUILD_SHA"], + ["BUILD_SHA"], +]; + +let cached: string | null | undefined; + +export function readRunningBuildSha(cwd: string = process.cwd()): string | null { + if (cached !== undefined) return cached; + + const fromEnv = process.env.OMNIROUTE_BUILD_SHA?.trim(); + if (fromEnv) { + cached = fromEnv; + return cached; + } + + for (const segments of SENTINEL_PATHS) { + try { + const value = fs.readFileSync(path.join(cwd, ...segments), "utf8").trim(); + if (value) { + cached = value; + return cached; + } + } catch { + // Sentinel absent at this location — try the next one. A dev run has none of them. + } + } + + cached = null; + return cached; +} diff --git a/src/lib/monitoring/observability.ts b/src/lib/monitoring/observability.ts index 9974c48e55..44acfdffc6 100644 --- a/src/lib/monitoring/observability.ts +++ b/src/lib/monitoring/observability.ts @@ -126,6 +126,8 @@ interface BuildTelemetryPayloadOptions { interface BuildHealthPayloadOptions { appVersion: string; + /** #10427: git SHA the running artifact was built from, so a deploy is auditable over HTTP. */ + buildSha?: string | null; catalogCount?: number; settings: { setupComplete?: boolean } | null | undefined; connections: Array<{ provider?: string; isActive?: boolean | null; rateLimitedUntil?: unknown }>; @@ -288,10 +290,14 @@ export function buildHealthPayload({ activeSessionsByKey = {}, credentialHealth, adaptiveAdmission = null, + buildSha = null, }: BuildHealthPayloadOptions) { const timestamp = new Date().toISOString(); const system = { version: appVersion, + // #10427: identifying a bad deploy previously required SSH + grepping compiled chunks. + // Absent/empty when unknown (dev runs) — never a fabricated value. + ...(buildSha ? { buildSha } : {}), nodeVersion: process.version, uptime: process.uptime(), memoryUsage: process.memoryUsage(), diff --git a/tests/unit/build-sha-provenance-10427.test.ts b/tests/unit/build-sha-provenance-10427.test.ts new file mode 100644 index 0000000000..e7b3afb675 --- /dev/null +++ b/tests/unit/build-sha-provenance-10427.test.ts @@ -0,0 +1,135 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +/** + * #10427 — the packaged artifact carries `dist/BUILD_SHA`, but nothing ever checked that + * the SHA belongs to the release line. A tarball built from a feature branch installs and + * runs indistinguishably from a release build. + * + * That is exactly how the internal gateway ended up serving a build from + * `fix/9603-qwen-token-plan-quota` (SHA 178febc50f) that predated #10373: every request + * died with `502 … Executor result must contain a Response`, and the only way to find out + * what was actually deployed was SSH + grepping the compiled chunks. + * + * Two defenses, both covered here: + * - `resolveBuildProvenance()` classifies a build SHA against the release line. + * - the health payload exposes `buildSha`, so what is deployed is auditable over HTTP. + */ + +const { resolveBuildProvenance } = await import("../../scripts/build/buildProvenance.ts"); +const { buildHealthPayload } = await import("../../src/lib/monitoring/observability.ts"); + +function makeRepo(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-prov-")); + return dir; +} + +test("P1: a SHA on the release line is accepted", () => { + const result = resolveBuildProvenance({ + buildSha: "abc1234", + isAncestorOfRelease: () => true, + allowOverride: false, + }); + assert.equal(result.ok, true); + assert.equal(result.reason, "on-release-line"); +}); + +test("P2: a SHA from an unrelated branch is REJECTED — the #10427 incident", () => { + const result = resolveBuildProvenance({ + buildSha: "178febc50f", + isAncestorOfRelease: () => false, + allowOverride: false, + }); + assert.equal(result.ok, false); + assert.equal(result.reason, "off-release-line"); + assert.match( + result.message, + /release/i, + "the failure must say what is wrong so a human can act on it" + ); +}); + +test("P3: an explicit canary override is allowed but recorded, never silent", () => { + const result = resolveBuildProvenance({ + buildSha: "178febc50f", + isAncestorOfRelease: () => false, + allowOverride: true, + }); + assert.equal(result.ok, true); + assert.equal(result.reason, "canary-override"); + assert.match(result.message, /canary/i); +}); + +test("P4: a missing BUILD_SHA is a failure, not a pass-by-default", () => { + const result = resolveBuildProvenance({ + buildSha: "", + isAncestorOfRelease: () => true, + allowOverride: false, + }); + assert.equal(result.ok, false); + assert.equal(result.reason, "missing-sha"); +}); + +test("P5: a missing BUILD_SHA is not excused by the canary override either", () => { + const result = resolveBuildProvenance({ + buildSha: "", + isAncestorOfRelease: () => false, + allowOverride: true, + }); + assert.equal(result.ok, false, "an unidentifiable artifact can never be validated"); + assert.equal(result.reason, "missing-sha"); +}); + +test("P6: readBuildSha returns the trimmed sentinel, or empty when absent", async () => { + const { readBuildSha } = await import("../../scripts/build/buildProvenance.ts"); + const repo = makeRepo(); + try { + assert.equal(readBuildSha(repo), "", "no dist/BUILD_SHA → empty, never a throw"); + fs.mkdirSync(path.join(repo, "dist"), { recursive: true }); + fs.writeFileSync(path.join(repo, "dist", "BUILD_SHA"), "e05ac345da\n"); + assert.equal(readBuildSha(repo), "e05ac345da"); + } finally { + fs.rmSync(repo, { recursive: true, force: true }); + } +}); + +/** Minimal but complete options for buildHealthPayload — only `buildSha` varies below. */ +function healthOptions(buildSha?: string) { + return { + appVersion: "3.8.50", + ...(buildSha === undefined ? {} : { buildSha }), + settings: { setupComplete: true }, + connections: [], + circuitBreakers: [], + rateLimitStatus: {}, + learnedLimits: {}, + lockouts: {}, + localProviders: {}, + inflightRequests: 0, + quotaMonitorSummary: {}, + quotaMonitorMonitors: [], + activeSessions: [], + } as unknown as Parameters[0]; +} + +test("P7: the health payload exposes buildSha so deployments are auditable over HTTP", () => { + const payload = buildHealthPayload(healthOptions("e05ac345da")); + assert.equal( + (payload.system as { buildSha?: string }).buildSha, + "e05ac345da", + "without this, identifying a bad deploy needs SSH + grepping compiled chunks" + ); +}); + +test("P8: health stays valid when no buildSha is known (dev runs)", () => { + const payload = buildHealthPayload(healthOptions()); + const system = payload.system as { version?: string; buildSha?: string }; + assert.equal(system.version, "3.8.50", "the existing contract must not regress"); + assert.ok( + system.buildSha === undefined || system.buildSha === null || system.buildSha === "", + "an unknown build SHA must be absent/empty, never a fabricated value" + ); +});