From f907b5ea8e137fa6c2b8e87b8518bcd2ddb929e1 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Fri, 28 Aug 2026 17:19:45 -0300 Subject: [PATCH] fix(ci): cap heavy builds at two runners with the omni-build label (#11932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .113 box (31 GB) holds one next-build (14–16 GB RSS) comfortably and two at the edge; on 2026-08-28 the kernel killed main's build twice while PR builds ran beside it. Labels are the runner-side cap: only omniroute-113-5 and omniroute-113-6 carry omni-build (added through the runners API, no re-registration), and every job that runs a next build — ci.yml build, npm-publish.yml publish, both nightly-release-green validations — now asks for that label. A third heavy job queues on GitHub instead of racing for memory. The six other runners keep omni-release and no longer take builds. Pairs with the heavy-build-* concurrency lanes (#11901); documented in docs/ops/RUNNER_BOX.md. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/nightly-release-green.yml | 4 ++-- .github/workflows/npm-publish.yml | 2 +- .../maintenance/ci-omni-build-runner-label.md | 4 ++++ docs/ops/RUNNER_BOX.md | 14 +++++++++----- 5 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 changelog.d/maintenance/ci-omni-build-runner-label.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c4bdca2e9..34874b0c7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -606,12 +606,12 @@ jobs: # Dynamic runner: when the release captain flips the USE_VPS_RUNNER repo var to # 'true' (scripts/vps/release-runner-up.sh does it after the self-hosted VM is # online), the heavy jobs run on the dedicated 32-core VPS runners (label - # omni-release) instead of queueing on the 20-concurrent-job hosted pool. + # omni-build) instead of queueing on the 20-concurrent-job hosted pool. # Safety: fork PRs NEVER reach the self-hosted runner — the expression falls # back to ubuntu-latest unless the PR head repo is this repository (push / # dispatch events are own-origin by definition). Any failure path (VM down, # var unset/false) also falls back to ubuntu-latest. - runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }} + runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-latest' }} needs: changes # The .113 pool runs ONE next-build with room to spare and two at the edge: the # box has 31 GB and a single next-build peaks at 14–16 GB RSS. On 2026-08-28 diff --git a/.github/workflows/nightly-release-green.yml b/.github/workflows/nightly-release-green.yml index 65d12db4de..4aa3fec847 100644 --- a/.github/workflows/nightly-release-green.yml +++ b/.github/workflows/nightly-release-green.yml @@ -68,7 +68,7 @@ jobs: # this runs on the dedicated VPS runner — clean env (no operator OMNIROUTE_API_KEY, # no local noauth CLIs => zero machine-specific false positives) and no contention. # Nightly cron normally finds the var false (VM off) and falls back to hosted. - runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-release"]')) || 'ubuntu-latest' }} + runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]')) || 'ubuntu-latest' }} env: JWT_SECRET: ci-nightly-secret-with-sufficient-length-for-validation API_KEY_SECRET: ci-nightly-api-key-secret-long @@ -217,7 +217,7 @@ jobs: # On a push, only run for a push to main — a push to release/* is handled by # release-green above. Schedule/dispatch always run (they also sweep main). if: ${{ github.event_name != 'push' || github.ref_name == 'main' }} - runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-release"]')) || 'ubuntu-latest' }} + runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]')) || 'ubuntu-latest' }} env: JWT_SECRET: ci-nightly-secret-with-sufficient-length-for-validation API_KEY_SECRET: ci-nightly-api-key-secret-long diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index e7a92bb2f4..b945017cc1 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -62,7 +62,7 @@ jobs: # mid-"Creating an optimized production build" while v3.8.48 had still fit in 16min. # This job never runs on `pull_request`, so the fork-safety clause is always true here; # it is kept verbatim so the expression stays greppable against ci.yml. - runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }} + runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-latest' }} outputs: version: ${{ steps.resolve.outputs.version }} tag: ${{ steps.resolve.outputs.tag }} diff --git a/changelog.d/maintenance/ci-omni-build-runner-label.md b/changelog.d/maintenance/ci-omni-build-runner-label.md new file mode 100644 index 0000000000..50dfd41d24 --- /dev/null +++ b/changelog.d/maintenance/ci-omni-build-runner-label.md @@ -0,0 +1,4 @@ +- Every CI job that runs a `next build` (`build`, the npm `publish`, both release-green + validations) now targets the `omni-build` runner label, which only two of the eight + self-hosted runners carry. The box holds one build comfortably and two at the edge; a + third now queues on GitHub instead of being OOM-killed by the kernel. diff --git a/docs/ops/RUNNER_BOX.md b/docs/ops/RUNNER_BOX.md index 2012742e7e..ce77f66f14 100644 --- a/docs/ops/RUNNER_BOX.md +++ b/docs/ops/RUNNER_BOX.md @@ -4,7 +4,7 @@ title: Self-Hosted Runner Box Operations # Self-Hosted Runner Box Operations (.113 pool) -The self-hosted pool (`self-hosted, omni-release` labels) runs on the **.113** box. +The self-hosted pool (`self-hosted, omni-release` on all eight runners; `omni-build` on two) runs on the **.113** box. Measured 2026-08-28 (v3.8.50 postmortem, Parte III): | resource | value | what it means for scheduling | @@ -49,10 +49,14 @@ a time, only when idle**, with the idle check and the restart in the same comman ## Operating rules -- **Heavy-build ceiling: 2 at a time.** The listener ceiling (`MAX_ACTIVE_RUNNERS=8` - in cron) is a proxy until jobs are split by label — `omni-build` on 2 runners for - Build/publish/heavy shards, `omni-light` on the rest — which is an operator - decision, not something cron should enforce by killing listeners. +- **Heavy-build ceiling: 2 at a time — enforced by label.** Every job that runs a + `next build` (`ci.yml` `build`, `npm-publish.yml` `publish`, both `nightly-release-green` + validations) targets `[self-hosted, omni-build]`, and only **two** runners carry that + label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no + re-registration). The other six keep `omni-release` and take nothing heavy; GitHub + queues a third build instead of the kernel killing one. Pair with the `heavy-build-*` + concurrency lanes in `ci.yml`. To add capacity, label another runner — never raise + the count past what 31 GB holds (one next-build ≈ 14–16 GB). - **Never clean `/tmp` or `_work` by hand while any runner is busy.** A check-then-delete with a gap between the two is how a live Build job lost its `_work` on 2026-08-27. The janitor does the check and the removal in one step;