From 38e2616464fac4681c1f7a4e05dc9974e99e1dde Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Sat, 29 Aug 2026 09:52:00 -0300 Subject: [PATCH] fix(ci): stop hosted docker-publish OOM and unpaint Build (advisory) (#12021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ci): stop hosted docker-publish OOM and unpaint Build (advisory) docker-publish was firing 8 concurrent hosted builds on every merge storm; each died ResourceExhausted in npm run build (#11976). One publish per ref, webpack instead of Turbopack so native RSS stays inside the V8 heap we can cap. Build (advisory) is skipped: continue-on-error still reports FAILURE and was painting every fork PR red. Closes #11976 * fix(ci): run docker-publish amd64 on omni-build and share the heavy lane The .113 box is 31 GB / 32 cores — enough for one next-build. Hosted ubuntu-24.04 is ~7 GB and ResourceExhausted every publish (#11976). amd64 now targets [self-hosted, omni-build] (Turbopack) when USE_VPS_RUNNER is on, joins the existing heavy-build-main group so it queues beside ci.yml Build instead of becoming a third heavy, and falls back to hosted + webpack if the VPS is off. arm64 stays on ubuntu-24.04-arm with webpack (no ARM box). * test(ci): align the advisory-build contract with the hosted-OOM skip if: ${{ false }} tripped zizmor obfuscation (194→195). Bare if: false skips the job without a new finding. The #7307 test now pins the skip and keeps the job body as the restore recipe. --- .github/workflows/ci.yml | 6 +-- .github/workflows/docker-publish.yml | 37 +++++++++++++++++-- .github/workflows/quality.yml | 11 +++++- ...cker-publish-webpack-and-advisory-build.md | 1 + docs/ops/RUNNER_BOX.md | 26 +++++++++---- tests/unit/build/check-workflows.test.ts | 26 +++++-------- 6 files changed, 77 insertions(+), 30 deletions(-) create mode 100644 changelog.d/maintenance/11976-docker-publish-webpack-and-advisory-build.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 441d19160e..c15bf1ca12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -621,9 +621,9 @@ jobs: # 13:50Z the kernel OOM-killed main's build while a PR build ran beside it # (five Build jobs had been queued by a burst of PRs). Two lanes: main keeps # its own so a release is never queued behind PR traffic; PR builds serialize - # among themselves. GitHub keeps one running + one pending per group and - # CANCELS older pendings — a cancelled PR build is re-runnable; a dead main - # build costs the publish its artefact and a 40-minute rebuild that OOMs. + # among themselves. docker-publish.yml's amd64 leg joins `heavy-build-main` + # so a :next image build waits beside this artefact instead of becoming the + # third heavy (#11976). GitHub keeps one running + one pending per group. concurrency: group: heavy-build-${{ github.ref == 'refs/heads/main' && 'main' || 'pr' }} cancel-in-progress: false diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 37578d6e4f..fc288a9cb7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -26,6 +26,14 @@ on: type: boolean default: false +# One publish per ref. A merge storm used to fan out 8 concurrent hosted builds, +# every one OOM-killing `npm run build` inside BuildKit (#11976). The :next +# channel only needs the newest SHA; cancel-in-progress is the same pattern as +# quality.yml / nightly-release-green. +concurrency: + group: docker-publish-${{ github.ref }} + cancel-in-progress: true + # Least-privilege default: read-only at the top level; the build and merge jobs that # push to GHCR grant packages: write themselves (Scorecard TokenPermissions). permissions: @@ -118,7 +126,19 @@ jobs: name: Build Docker (${{ matrix.platform }}) needs: prepare if: needs.prepare.outputs.skip != 'true' - runs-on: ${{ matrix.runner }} + # amd64: the .113 omni-build pool (31 GB / 32 cores, two listeners). A + # next-build peaks at 14–16 GB; hosted ubuntu-24.04 is ~7 GB and dies + # ResourceExhausted (#11976). Falls back to hosted + webpack when + # USE_VPS_RUNNER is off. arm64: no ARM box — stay on GitHub's ubuntu-24.04-arm + # with webpack (Turbopack native RSS does not fit 7 GB). + runs-on: ${{ matrix.arch == 'amd64' && (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-24.04') || 'ubuntu-24.04-arm' }} + # Share the 2-slot omni-build ceiling with ci.yml `Build` / npm-publish. + # Same group as main's Build so a :next publish waits beside the artefact + # instead of becoming the third heavy and getting kernel-OOM'd (2026-08-28). + # arm64 is hosted — its own group, cancelled by the workflow-level concurrency. + concurrency: + group: ${{ matrix.arch == 'amd64' && 'heavy-build-main' || format('docker-publish-arm-{0}', github.ref) }} + cancel-in-progress: ${{ matrix.arch != 'amd64' }} permissions: contents: read packages: write @@ -127,14 +147,14 @@ jobs: matrix: include: - platform: linux/amd64 - runner: ubuntu-24.04 arch: amd64 - platform: linux/arm64 - runner: ubuntu-24.04-arm arch: arm64 env: IMAGE_NAME: diegosouzapw/omniroute GHCR_IMAGE_NAME: ghcr.io/diegosouzapw/omniroute + # Turbopack on the 31 GB box; webpack on any hosted 7 GB fallback / ARM. + USE_TURBOPACK: ${{ (matrix.arch == 'amd64' && vars.USE_VPS_RUNNER == 'true') && '1' || '0' }} steps: - name: Checkout uses: actions/checkout@v7 @@ -143,6 +163,9 @@ jobs: ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/v{0}', inputs.version) || '' }} fetch-depth: 0 + - name: Assert Docker Engine + run: docker info + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -166,6 +189,8 @@ jobs: context: . target: runner-base platforms: ${{ matrix.platform }} + build-args: | + OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -183,6 +208,8 @@ jobs: context: . target: runner-web platforms: ${{ matrix.platform }} + build-args: | + OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -208,6 +235,8 @@ jobs: file: Dockerfile.bun target: runner-base platforms: ${{ matrix.platform }} + build-args: | + OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} @@ -233,6 +262,8 @@ jobs: file: Dockerfile.bun target: runner-web platforms: ${{ matrix.platform }} + build-args: | + OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true tags: | ${{ env.IMAGE_NAME }} diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f4006490ff..a21597e8ba 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -70,7 +70,16 @@ jobs: # 2026-08-14: 72 of the last 100 PRs into release/** came from forks, so the fork case is # the majority of the traffic, not the exception — this job earns its place, it just should # not duplicate build.yml for the own-origin 28%. - if: ${{ github.event_name != 'pull_request' || ((github.event.pull_request.draft == false || startsWith(github.head_ref, 'mergify/merge-queue/')) && needs.changes.outputs.code == 'true' && github.event.pull_request.head.repo.full_name != github.repository) }} + # Disabled 2026-08-29 (#11976 follow-up). `continue-on-error: true` still + # reports a GitHub check FAILURE, so every fork PR into release/** was born + # with a red "Build (advisory)" even when every required gate was green + # (sweep-reds, 41 PRs). Hosted ubuntu-latest cannot finish `npm run build` + # on this tree — VM shutdown ~6 min in, same class as build.yml going + # workflow_dispatch-only in #11962. Pre-merge build signal for release/** + # is nightly-release-green (omni-build); for main it is ci.yml `Build`. + # Restore this job when a runner that actually fits the tree is wired here. + # Bare `false` (not `${{ false }}`) — zizmor obfuscation flags the expression form. + if: false # PINNED to hosted — this was the last job in THIS workflow still on the USE_VPS_RUNNER # switch (ci.yml's Build, nightly-release-green and npm-publish keep it, so the variable # stays meaningful), and with USE_VPS_RUNNER=true it produced NO signal at all here. diff --git a/changelog.d/maintenance/11976-docker-publish-webpack-and-advisory-build.md b/changelog.d/maintenance/11976-docker-publish-webpack-and-advisory-build.md new file mode 100644 index 0000000000..ac3c86ae82 --- /dev/null +++ b/changelog.d/maintenance/11976-docker-publish-webpack-and-advisory-build.md @@ -0,0 +1 @@ +- Stop painting every fork PR into `release/**` red: `quality.yml` `Build (advisory)` is skipped (GitHub still reports `continue-on-error` failures as check FAILURE). Hosted `ubuntu-latest` cannot finish `npm run build` on this tree — same class as #11962 taking `build.yml` off the PR rail. `docker-publish.yml` amd64 now runs on the `.113` `omni-build` pool (31 GB / 32 cores, two listeners) with Turbopack, shares the `heavy-build-main` lane with `ci.yml` `Build` so it queues instead of becoming a third heavy, and keeps per-ref concurrency (a merge storm was starting 8 concurrent OOM builds). arm64 stays on `ubuntu-24.04-arm` with webpack — there is no ARM box. Fallback when `USE_VPS_RUNNER` is off: hosted amd64 + webpack (#11976). diff --git a/docs/ops/RUNNER_BOX.md b/docs/ops/RUNNER_BOX.md index c9719bb053..4b6fef353d 100644 --- a/docs/ops/RUNNER_BOX.md +++ b/docs/ops/RUNNER_BOX.md @@ -50,13 +50,25 @@ a time, only when idle**, with the idle check and the restart in the same comman ## Operating rules - **Heavy-build ceiling: 2 at a time — enforced by label.** Every job that runs a - `next build` (`ci.yml` `build`, `npm-publish.yml` `publish`, both `nightly-release-green` - validations) targets `[self-hosted, omni-build]`, and only **two** runners carry that - label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no - re-registration). The other six keep `omni-release` and take nothing heavy; GitHub - queues a third build instead of the kernel killing one. Pair with the `heavy-build-*` - concurrency lanes in `ci.yml`. To add capacity, label another runner — never raise - the count past what 31 GB holds (one next-build ≈ 14–16 GB). + full `next build` targets `[self-hosted, omni-build]`, and only **two** runners carry + that label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no + re-registration): + - `ci.yml` `Build` and `npm-publish.yml` `publish` + - both `nightly-release-green` validations + - `docker-publish.yml` **amd64** (the hosted 7 GB runner ResourceExhausted this + tree — #11976). The arm64 leg stays on `ubuntu-24.04-arm` (no ARM box) with + webpack. + The other listeners keep `omni-release` / `omni-light` and take nothing heavy; + GitHub queues a third build instead of the kernel killing one. Pair with the + `heavy-build-*` concurrency lanes: `ci.yml` `Build` on `main` and docker-publish + amd64 share `heavy-build-main` (`cancel-in-progress: false`) so a `:next` publish + waits beside the artefact instead of sitting next to it. PR builds use + `heavy-build-pr`. To add capacity, label another runner — never raise the count + past what 31 GB holds (one next-build ≈ 14–16 GB; a Docker amd64 build is the + same class plus the daemon — still one slot). + Docker Engine must be on those two units (`docker info` is the first step of + the publish job). If it is missing, the job fails closed instead of hanging on + `setup-buildx`. - **Light pool: `omni-light` (2026-08-29, #11965).** `omniroute-113` and `omniroute-113-2` carry `omni-light` for jobs that need a backend-only `next build` (~5–6 GB) but not a full one: the nightly Schemathesis, promptfoo, garak and axe-a11y jobs. They ran on the hosted 7 GB runner and diff --git a/tests/unit/build/check-workflows.test.ts b/tests/unit/build/check-workflows.test.ts index fea3f3476e..245ca625c6 100644 --- a/tests/unit/build/check-workflows.test.ts +++ b/tests/unit/build/check-workflows.test.ts @@ -318,7 +318,7 @@ test("readBaselineZizmorValue: invalid JSON returns null (does not throw)", () = // quality.yml — release PR build gate regression coverage (#7307) // ───────────────────────────────────────────────────────────────────────────── -test("#7307 quality.yml adds an advisory production build for release PR code changes", () => { +test("#7307 quality.yml keeps the advisory production build (disabled: hosted 7 GB cannot finish it)", () => { const source = readQualityWorkflow(); const buildJob = source.match(/\n build:\n[\s\S]*?\n # Docs\/OpenAPI contract gates only/); @@ -326,17 +326,15 @@ test("#7307 quality.yml adds an advisory production build for release PR code ch assert.ok(buildJob, "quality.yml must define the build job before docs-gates"); assert.match(buildJob[0], /name: Build \(advisory\)/); assert.match(buildJob[0], /needs: changes/); - assert.match(buildJob[0], /needs\.changes\.outputs\.code == 'true'/); - assert.match(buildJob[0], /github\.event\.pull_request\.draft == false/); - assert.match(buildJob[0], /startsWith\(github\.head_ref, 'mergify\/merge-queue\/'\)/); - // FORK PRs ONLY (2026-08-14). build.yml's `Fast Production Build` fires on - // `push: branches: ["**"]` and runs the superset `build:release`, so own-origin branches - // were building twice; a fork's push never reaches this repo, making this their only - // pre-merge build signal — and forks are 72 of the last 100 PRs into release/**. - assert.match( - buildJob[0], - /github\.event\.pull_request\.head\.repo\.full_name != github\.repository/ - ); + // Live predicate is a static skip. `continue-on-error` still paints a GitHub + // check FAILURE, and the hosted runner ResourceExhausts this tree (#11976) — + // so the job must not run. Bare `if: false` (zizmor obfuscation flags + // `${{ false }}`). The body below is the restore recipe, not dead code. + const buildDirectives = buildJob[0] + .split("\n") + .filter((line) => !/^\s*#/.test(line)) + .join("\n"); + assert.match(buildDirectives, /\n {4}if: false\n/); // Runner PINNED to hosted. The self-hosted pool is 2 permanently-busy runners, where this // job either queued for hours or was killed by cancel-in-progress — ~10-15% of runs ever // reached a conclusion across 2026-08-13/14. It must NOT go back on the USE_VPS_RUNNER @@ -345,10 +343,6 @@ test("#7307 quality.yml adds an advisory production build for release PR code ch // Check the DIRECTIVES, not the prose: the comment above legitimately explains why the // self-hosted pool was abandoned, so a naive /self-hosted/ scan over the whole block would // match its own rationale. - const buildDirectives = buildJob[0] - .split("\n") - .filter((line) => !/^\s*#/.test(line)) - .join("\n"); assert.doesNotMatch(buildDirectives, /self-hosted/); assert.doesNotMatch(buildDirectives, /USE_VPS_RUNNER/); // Memory provisioning mirrored from build.yml: --max-old-space-size bounds only V8's heap,