From 15a786b56f8f311f5176e06ef2ee0fca8ece35e4 Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Wed, 26 Aug 2026 12:51:43 -0300 Subject: [PATCH] fix(docker): re-derive build memory budget from measured per-process RSS (#7518) PR #11419 lowered OMNIROUTE_BUILD_WORKERS 8->3 using an INFERRED per-worker peak (WORKER_PEAK_MB=2560) and treated the parent `next build` process's RSS as bounded by OMNIROUTE_BUILD_MEMORY_MB (the V8 heap ceiling). Both assumptions were wrong: the issue owner's own VPS reproduction (dmesg OOM-killer report) measured the real per-process RSS directly at ~4.5 GB, independent of NODE_OPTIONS, since Turbopack compiles in native/Rust memory outside the V8 heap -- and that applies to the parent process too. With OMNIROUTE_BUILD_WORKERS=3 (1 parent + 2 workers), the measured worst case is 3 x 4500MB = 13500MB against a 12288MB (75% of 16GB) budget -- still over, matching the still-live "Publish to Docker Hub" CI failures after #11419 merged (run 32907937950, 2026-08-25). Lowers OMNIROUTE_BUILD_WORKERS to 2 (1 parent + 1 worker = 9000MB, fits with headroom) and re-derives tests/unit/docker-build-memory-budget.test.ts's model on the measured figure, applied uniformly to every process instead of conflating the parent process with the V8 heap ceiling. --- Dockerfile | 36 +++++++----- .../fixes/7518-docker-build-memory-budget.md | 1 + docs/guides/DOCKER_GUIDE.md | 19 ++++--- tests/unit/docker-build-memory-budget.test.ts | 57 +++++++++++-------- 4 files changed, 70 insertions(+), 43 deletions(-) create mode 100644 changelog.d/fixes/7518-docker-build-memory-budget.md diff --git a/Dockerfile b/Dockerfile index a35f57e280..97a4b75d50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -184,19 +184,29 @@ ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_BUILD_MEMORY_MB}" # silently leaving no standalone bundle. Next derives the worker count from # CIRCLE_NODE_TOTAL (workers = N-1). (#10060) # -# Lowered 8 → 3 (7 workers → 2). Every page-data worker inherits NODE_OPTIONS -# above, so the ceiling is per PROCESS, not per build: 7 workers on a 16 GB -# GitHub runner (ubuntu-24.04 / ubuntu-24.04-arm, 4 vCPU) exhausted the host and -# buildkit failed the whole step with `ResourceExhausted: ... cannot allocate -# memory`. The compile phase always finished ("✓ Compiled successfully in -# 4.2min"); the kernel killed the build right after "Collecting page data using -# 7 workers". It was intermittent for a while and went 100% on 2026-08-22, which -# is what a threshold being crossed by ordinary codebase growth looks like. -# tests/unit/docker-build-memory-budget.test.ts does the arithmetic and fails if -# either knob is raised past what a 16 GB runner holds. 2 workers also stops -# oversubscribing the runner's 4 vCPU, which 7 did. Override for a big builder: -# `--build-arg OMNIROUTE_BUILD_WORKERS=8`. -ARG OMNIROUTE_BUILD_WORKERS=3 +# Lowered 8 → 3 (7 workers → 2) in #11419, then 3 → 2 (2 workers → 1) in #7518. +# Every page-data worker inherits NODE_OPTIONS above, so the ceiling is per +# PROCESS, not per build: 7 workers on a 16 GB GitHub runner (ubuntu-24.04 / +# ubuntu-24.04-arm, 4 vCPU) exhausted the host and buildkit failed the whole +# step with `ResourceExhausted: ... cannot allocate memory`. The compile phase +# always finished ("✓ Compiled successfully in 4.2min"); the kernel killed the +# build right after "Collecting page data using N workers". +# +# #11419's first fix (8 → 3) modeled the per-worker peak as an INFERENCE +# (2560 MB, guessed from "7 workers didn't fit") and assumed the parent +# process's RSS tracked the V8 heap ceiling. Both assumptions were wrong: a +# live VPS reproduction (issue #7518, dmesg OOM-killer report) measured the +# real per-process RSS directly at ~4.5 GB, independent of the NODE_OPTIONS +# heap flag (Turbopack itself is native/Rust, outside the V8 heap) — and it +# applies to the parent process too, not just workers. 2 workers (3 processes +# × 4.5 GB = 13.5 GB) still didn't fit the 12.288 GB (75%) budget on a 16 GB +# runner, matching the still-live publish failures after #11419 merged. 1 +# worker (2 processes × 4.5 GB = 9 GB) fits with headroom to spare. +# tests/unit/docker-build-memory-budget.test.ts does the arithmetic against +# the measured figure and fails if either knob is raised past what a 16 GB +# runner holds. Override for a big builder: `--build-arg +# OMNIROUTE_BUILD_WORKERS=8`. +ARG OMNIROUTE_BUILD_WORKERS=2 ENV CIRCLE_NODE_TOTAL=${OMNIROUTE_BUILD_WORKERS} COPY . ./ diff --git a/changelog.d/fixes/7518-docker-build-memory-budget.md b/changelog.d/fixes/7518-docker-build-memory-budget.md new file mode 100644 index 0000000000..9db73b9049 --- /dev/null +++ b/changelog.d/fixes/7518-docker-build-memory-budget.md @@ -0,0 +1 @@ +- **fix(docker):** re-derive the Docker build's worker-pool memory budget from the MEASURED ~4.5 GB per-process RSS (the issue owner's own VPS dmesg OOM-killer reproduction) instead of the stale 2560 MB/worker inference #11419 shipped, and lower `OMNIROUTE_BUILD_WORKERS` 3 → 2 so 1 parent + 1 page-data worker (2 processes × 4.5 GB = 9 GB) fits the 12.288 GB (75%) budget on a 16 GB GitHub Actions runner — the previous default (1 parent + 2 workers = 13.5 GB) still overcommitted the runner and kept "Publish to Docker Hub" failing with `cannot allocate memory` after #11419 merged (#7518). diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index 295d8aa800..ec5affb381 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -226,16 +226,21 @@ Three build args control what the `builder` stage costs. They are build-time onl | --------------------------- | ------- | ----------------------------------------------------------------------------------- | | `OMNIROUTE_USE_TURBOPACK` | `1` | `0` builds with webpack instead. Lower peak memory, slower. | | `OMNIROUTE_BUILD_MEMORY_MB` | `6144` | V8 heap ceiling (`--max-old-space-size`) for the spawned `next build`. | -| `OMNIROUTE_BUILD_WORKERS` | `3` | Feeds `CIRCLE_NODE_TOTAL`; Next derives `workers = N - 1` for page-data collection. | +| `OMNIROUTE_BUILD_WORKERS` | `2` | Feeds `CIRCLE_NODE_TOTAL`; Next derives `workers = N - 1` for page-data collection. | `OMNIROUTE_BUILD_WORKERS` is the one to raise on a big builder and the one to suspect when a constrained build dies **after** `✓ Compiled successfully`. Each -page-data worker is its own process and inherits `NODE_OPTIONS`, so the heap -ceiling is per process, not per build: the default of `3` (→ 2 workers) is sized -for the 16 GB / 4 vCPU GitHub-hosted runners the publish pipeline uses. At `8` -(→ 7 workers) that runner ran out of memory and buildkit failed the step with -`ResourceExhausted: ... cannot allocate memory`. `tests/unit/docker-build-memory-budget.test.ts` -does the arithmetic and fails if either knob outgrows the runner. +page-data worker is its own process, and so is the parent `next build` itself; +a live VPS reproduction (issue #7518) measured each process's peak RSS at +~4.5 GB independent of the `NODE_OPTIONS` heap flag (Turbopack compiles in +native/Rust memory outside the V8 heap). The default of `2` (→ 1 worker, 2 +processes total) is sized for the 16 GB / 4 vCPU GitHub-hosted runners the +publish pipeline uses. At `8` (→ 7 workers) that runner ran out of memory and +buildkit failed the step with `ResourceExhausted: ... cannot allocate memory`; +`3` (→ 2 workers) still didn't fit once the per-process RSS was measured +directly instead of inferred. `tests/unit/docker-build-memory-budget.test.ts` +does the arithmetic against the measured figure and fails if either knob +outgrows the runner. Turbopack compiles in native Rust memory that lives **outside** the V8 heap, so `OMNIROUTE_BUILD_MEMORY_MB` does not bound it. On a host with a memory ceiling the diff --git a/tests/unit/docker-build-memory-budget.test.ts b/tests/unit/docker-build-memory-budget.test.ts index 4c33386eb3..0d01e8aae1 100644 --- a/tests/unit/docker-build-memory-budget.test.ts +++ b/tests/unit/docker-build-memory-budget.test.ts @@ -4,28 +4,37 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; // The Docker publish workflow builds on GitHub-hosted runners (ubuntu-24.04 and -// ubuntu-24.04-arm): 4 vCPU, 16 GB RAM. Every Next page-data worker is its own -// process and inherits NODE_OPTIONS, so the V8 ceiling is per PROCESS: the -// build's worst case is roughly `workers × OMNIROUTE_BUILD_MEMORY_MB`. +// ubuntu-24.04-arm): 4 vCPU, 16 GB RAM. Every Next page-data worker AND the +// parent `next build` process are separate OS processes, so the budget has to +// cover all of them, not just the workers. // // With 7 workers × 6144 MB the runner ran out and buildkit failed the step with // `ResourceExhausted: ... cannot allocate memory`, right after "Collecting page // data using 7 workers" — every Docker publish since 2026-08-22 23:14 UTC. +// Lowering to 2 workers (#10060 / PR #11419) was not enough: it modeled the +// per-process peak as an INFERENCE (`WORKER_PEAK_MB = 2560`, derived only from +// "7 workers didn't fit") and assumed the parent process tracked the V8 heap +// ceiling (`OMNIROUTE_BUILD_MEMORY_MB`) rather than its own RSS. The owner's +// live VPS reproduction (issue #7518, dmesg OOM-killer report, 2026-08-24) +// measured the real number directly: `next-build (v16) ... anon-rss:4522744kB` +// (~4.5 GB) per process, independent of the NODE_OPTIONS heap flag — Turbopack +// itself is native/Rust and compiles outside the V8 heap. With 2 workers that +// keeps the publish pipeline failing at "Collecting page data using 2 workers" +// (run 32907937950, 2026-08-25). // -// This pins the budget so raising either knob has to be a deliberate change -// that re-does the arithmetic, not a one-line bump that silently reds the -// publish pipeline again. +// This pins the budget on the MEASURED figure, applied uniformly to every +// process (parent + workers), so raising the worker count has to be a +// deliberate change that re-does the arithmetic, not a one-line bump that +// silently reds the publish pipeline again. const RUNNER_MEMORY_MB = 16 * 1024; // Leave room for buildkit, the snapshotter and page cache. const HEADROOM_FRACTION = 0.75; -// Planning figure for one page-data worker's peak RSS. It is an INFERENCE, not -// a measurement: 7 workers did not fit in 16 GB alongside the parent, which -// puts the per-worker peak somewhere north of ~1.8 GB. 2.5 GB is that bound -// rounded up, so the budget below stays conservative. If a future build OOMs -// again with a worker count this test accepts, raise this number — do not -// weaken the budget. -const WORKER_PEAK_MB = 2560; +// Measured (not inferred) peak RSS for a single Next/Turbopack build process — +// parent or page-data worker alike — from the dmesg OOM-killer report above. +// If a future build OOMs again, re-measure via dmesg before raising this +// number — do not weaken the budget with another guess. +const MEASURED_PROCESS_RSS_MB = 4500; const dockerfile = readFileSync( fileURLToPath(new URL("../../Dockerfile", import.meta.url)), @@ -51,25 +60,27 @@ test("the Docker build's worker pool is derived from OMNIROUTE_BUILD_WORKERS", ( ); }); -test("worker count × per-process heap fits a 16 GB GitHub runner", () => { +test("worker count × measured per-process RSS fits a 16 GB GitHub runner", () => { const workerPool = readArgDefault("OMNIROUTE_BUILD_WORKERS"); - const heapMb = readArgDefault("OMNIROUTE_BUILD_MEMORY_MB"); // Next derives `workers = CIRCLE_NODE_TOTAL - 1`. const workers = workerPool - 1; assert.ok(workers >= 1, `CIRCLE_NODE_TOTAL=${workerPool} leaves no build workers`); - // The parent `next build` process is the one that genuinely needs the raised - // ceiling (the webpack/turbopack production pass, #4076); the workers are - // budgeted at their inferred peak instead. - const worstCaseMb = heapMb + workers * WORKER_PEAK_MB; + // Every process — the parent `next build` process AND each page-data + // worker — is budgeted at the measured per-process RSS floor (see the file + // banner comment). The V8 heap ceiling (OMNIROUTE_BUILD_MEMORY_MB) bounds + // JS allocations but not Turbopack's native/Rust memory, so it cannot stand + // in for the parent process's real RSS. + const processes = workers + 1; + const worstCaseMb = processes * MEASURED_PROCESS_RSS_MB; const budgetMb = RUNNER_MEMORY_MB * HEADROOM_FRACTION; assert.ok( worstCaseMb <= budgetMb, - `parent ${heapMb} MB + ${workers} workers × ${WORKER_PEAK_MB} MB = ${worstCaseMb} MB ` + - `exceeds the ${budgetMb} MB budget on a ${RUNNER_MEMORY_MB} MB runner — the Docker ` + - `publish step dies with "ResourceExhausted: cannot allocate memory" during page-data ` + - `collection` + `${processes} processes (1 parent + ${workers} workers) × ${MEASURED_PROCESS_RSS_MB} MB ` + + `measured RSS = ${worstCaseMb} MB exceeds the ${budgetMb} MB budget on a ` + + `${RUNNER_MEMORY_MB} MB runner — the Docker publish step dies with "ResourceExhausted: ` + + `cannot allocate memory" during page-data collection` ); });