diff --git a/.env.example b/.env.example index c1a1f06e67..e6c8f35c8f 100644 --- a/.env.example +++ b/.env.example @@ -1023,6 +1023,13 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # to disable the check. Used by: src/lib/db/migrationRunner.ts. Default: 50. #OMNIROUTE_MAX_PENDING_MIGRATIONS=50 +# Working directory for the check:install-upgrade release gate. It builds two ~3 GB +# install trees plus a ~275 MB tarball, so it needs roughly 12 GB — more than the +# 12 GB RAM-backed tmpfs that /tmp is on the self-hosted runner, where it exhausted +# the tmpfs and npm silently truncated the package. Defaults to /.install-upgrade +# on real disk. Used by: scripts/check/check-install-upgrade.mjs. Default: /.install-upgrade. +#OMNIROUTE_INSTALL_UPGRADE_WORKDIR=/var/tmp/omniroute-install-upgrade + # Trust user-managed RTK project filter rules without strict signature checks. # Used by: open-sse/services/compression/engines/rtk/filterLoader.ts. Default: 0. #OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=0 diff --git a/.gitignore b/.gitignore index 07545f2a37..31c9e3b96a 100644 --- a/.gitignore +++ b/.gitignore @@ -293,3 +293,6 @@ docker-compose.yml.bak # Ad-hoc test sandboxes (never tracked — may contain local DBs) /.sandbox/ .aider* + +# check:install-upgrade work trees (~12 GB, disposable) +/.install-upgrade/ diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index b383b5696f..72db4a2aae 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -103,6 +103,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_MIGRATIONS_DIR` | _(auto-detect)_ | `src/lib/db/migrationRunner.ts` | Override the directory that the migration runner scans. Useful when shipping bundled migrations in custom builds. | | `OMNIROUTE_EXTRA_MIGRATIONS_DIRS` | _(unset)_ | `src/lib/db/migrationRunner/extraDirs.ts` | Additional migration directories as `namespace=dir` entries separated by the platform path delimiter (e.g. `ee=/opt/app/enterprise/db/migrations`). Files found there are recorded as `-`, so a distribution shipping its own migrations never collides with the upstream numeric slots. A malformed entry, an invalid namespace or a missing directory throws at startup instead of silently skipping the schema. | | `OMNIROUTE_MAX_PENDING_MIGRATIONS` | `50` | `src/lib/db/migrationRunner.ts` | Mass-pending-migrations safety threshold (#3416). Startup aborts if more than this many migrations are pending on an existing DB (guards against a wiped tracking table). Raise it to restore an older backup; set to `0` to disable the check. | +| `OMNIROUTE_INSTALL_UPGRADE_WORKDIR` | _(`/.install-upgrade`)_ | `scripts/check/check-install-upgrade.mjs` | Working directory for the `check:install-upgrade` release gate. It needs roughly 12 GB (two ~3 GB install trees plus the tarball), so it must not run on a small tmpfs — on the self-hosted runner `/tmp` is a 12 GB RAM-backed tmpfs and the gate exhausted it, truncating the package. | | `OMNIROUTE_SPEND_FLUSH_INTERVAL_MS` | _(default in code)_ | `src/lib/spend/batchWriter.ts` | Flush interval (ms) for the batched spend/cost writer. Lower values reduce write coalescing; higher values reduce DB contention. | | `OMNIROUTE_SPEND_MAX_BUFFER_SIZE` | _(default in code)_ | `src/lib/spend/batchWriter.ts` | Max buffered spend entries before a forced flush. Raise on high-QPS deployments; lower when bounded memory matters more. | | `OMNIROUTE_PROXY_FETCH_DEBUG` | _(unset)_ | `open-sse/utils/proxyFetch.ts` | Set to `"true"` to emit `[ProxyFetch]` debug logs on the Vercel relay path. Off by default to avoid leaking routing hints. | diff --git a/scripts/check/check-install-upgrade.mjs b/scripts/check/check-install-upgrade.mjs index 65b89d7e49..a71ccdb1fe 100644 --- a/scripts/check/check-install-upgrade.mjs +++ b/scripts/check/check-install-upgrade.mjs @@ -302,7 +302,7 @@ export function assertNoDiskExhaustion(output, label) { throw new Error( `${label}: the install ran out of disk space (${count} ENOSPC error(s) from npm). ` + `The package tree is truncated, so anything measured from it — boot, schema, ` + - `migrations — is meaningless. Free space in ${os.tmpdir()} (each install tree is ` + + `migrations — is meaningless. Free space in ${workDirForMessages} (each install tree is ` + `~3 GB) and re-run. This is an environment failure, NOT a schema divergence.` ); } @@ -318,6 +318,11 @@ function freeBytes(dir) { const GB = 1024 ** 3; +// Set once the work directory exists, so the ENOSPC message names the filesystem that +// actually ran out — pointing at /tmp when the gate works elsewhere sends the reader to +// free space on the wrong volume (which is what happened during the v3.8.50 publish). +let workDirForMessages = os.tmpdir(); + function resolvePreviousVersion(current, explicit) { if (explicit) return explicit; const out = execFileSync("npm", ["view", "omniroute", "dist-tags.latest"], { encoding: "utf8" }); @@ -348,7 +353,16 @@ async function main() { } const version = JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8")).version; const allowlist = loadAllowlist(ROOT); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-install-upgrade-")); + // NOT os.tmpdir(): on the self-hosted runner /tmp is a 12 GB tmpfs backed by RAM, while + // the root filesystem has ~66 GB free. This gate needs ~12 GB, so it exhausted the tmpfs + // and npm truncated the package — 58269 ENOSPC errors on the v3.8.50 publish, which the + // previous code could only report as a crash. Freeing disk did not help because the disk + // was never the constraint. Work on real disk beside the repo instead. + const workRoot = + process.env.OMNIROUTE_INSTALL_UPGRADE_WORKDIR || path.join(ROOT, ".install-upgrade"); + fs.mkdirSync(workRoot, { recursive: true }); + const tmp = fs.mkdtempSync(path.join(workRoot, "omniroute-install-upgrade-")); + workDirForMessages = tmp; const failures = []; const warnings = []; @@ -374,7 +388,7 @@ async function main() { // exited 0, and every later measurement was taken from a broken tree. const availableBytes = freeBytes(tmp); if (availableBytes !== null) { - log(`free space in ${os.tmpdir()}: ${(availableBytes / GB).toFixed(1)} GB`); + log(`free space in ${tmp}: ${(availableBytes / GB).toFixed(1)} GB`); if (availableBytes < 12 * GB) { warn( `only ${(availableBytes / GB).toFixed(1)} GB free — this gate needs roughly 12 GB ` +