fix(ci): stop hosted docker-publish OOM and unpaint Build (advisory) (#12021)

* fix(ci): stop hosted docker-publish OOM and unpaint Build (advisory)

docker-publish was firing 8 concurrent hosted builds on every merge
storm; each died ResourceExhausted in npm run build (#11976). One
publish per ref, webpack instead of Turbopack so native RSS stays
inside the V8 heap we can cap. Build (advisory) is skipped: continue-on-error
still reports FAILURE and was painting every fork PR red.

Closes #11976

* fix(ci): run docker-publish amd64 on omni-build and share the heavy lane

The .113 box is 31 GB / 32 cores — enough for one next-build. Hosted
ubuntu-24.04 is ~7 GB and ResourceExhausted every publish (#11976).
amd64 now targets [self-hosted, omni-build] (Turbopack) when
USE_VPS_RUNNER is on, joins the existing heavy-build-main group so it
queues beside ci.yml Build instead of becoming a third heavy, and
falls back to hosted + webpack if the VPS is off. arm64 stays on
ubuntu-24.04-arm with webpack (no ARM box).

* test(ci): align the advisory-build contract with the hosted-OOM skip

if: ${{ false }} tripped zizmor obfuscation (194→195). Bare if: false
skips the job without a new finding. The #7307 test now pins the skip
and keeps the job body as the restore recipe.

.github/workflows/ci.yml

@@ -621,9 +621,9 @@ jobs:
     # 13:50Z the kernel OOM-killed main's build while a PR build ran beside it
     # (five Build jobs had been queued by a burst of PRs). Two lanes: main keeps
     # its own so a release is never queued behind PR traffic; PR builds serialize
-    # among themselves. GitHub keeps one running + one pending per group and
-    # CANCELS older pendings — a cancelled PR build is re-runnable; a dead main
-    # build costs the publish its artefact and a 40-minute rebuild that OOMs.
+    # among themselves. docker-publish.yml's amd64 leg joins `heavy-build-main`
+    # so a :next image build waits beside this artefact instead of becoming the
+    # third heavy (#11976). GitHub keeps one running + one pending per group.
     concurrency:
       group: heavy-build-${{ github.ref == 'refs/heads/main' && 'main' || 'pr' }}
       cancel-in-progress: false

.github/workflows/docker-publish.yml

@@ -26,6 +26,14 @@ on:
         type: boolean
         default: false
 
+# One publish per ref. A merge storm used to fan out 8 concurrent hosted builds,
+# every one OOM-killing `npm run build` inside BuildKit (#11976). The :next
+# channel only needs the newest SHA; cancel-in-progress is the same pattern as
+# quality.yml / nightly-release-green.
+concurrency:
+  group: docker-publish-${{ github.ref }}
+  cancel-in-progress: true
+
 # Least-privilege default: read-only at the top level; the build and merge jobs that
 # push to GHCR grant packages: write themselves (Scorecard TokenPermissions).
 permissions:
@@ -118,7 +126,19 @@ jobs:
     name: Build Docker (${{ matrix.platform }})
     needs: prepare
     if: needs.prepare.outputs.skip != 'true'
-    runs-on: ${{ matrix.runner }}
+    # amd64: the .113 omni-build pool (31 GB / 32 cores, two listeners). A
+    # next-build peaks at 14–16 GB; hosted ubuntu-24.04 is ~7 GB and dies
+    # ResourceExhausted (#11976). Falls back to hosted + webpack when
+    # USE_VPS_RUNNER is off. arm64: no ARM box — stay on GitHub's ubuntu-24.04-arm
+    # with webpack (Turbopack native RSS does not fit 7 GB).
+    runs-on: ${{ matrix.arch == 'amd64' && (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-24.04') || 'ubuntu-24.04-arm' }}
+    # Share the 2-slot omni-build ceiling with ci.yml `Build` / npm-publish.
+    # Same group as main's Build so a :next publish waits beside the artefact
+    # instead of becoming the third heavy and getting kernel-OOM'd (2026-08-28).
+    # arm64 is hosted — its own group, cancelled by the workflow-level concurrency.
+    concurrency:
+      group: ${{ matrix.arch == 'amd64' && 'heavy-build-main' || format('docker-publish-arm-{0}', github.ref) }}
+      cancel-in-progress: ${{ matrix.arch != 'amd64' }}
     permissions:
       contents: read
       packages: write
@@ -127,14 +147,14 @@ jobs:
       matrix:
         include:
           - platform: linux/amd64
-            runner: ubuntu-24.04
             arch: amd64
           - platform: linux/arm64
-            runner: ubuntu-24.04-arm
             arch: arm64
     env:
       IMAGE_NAME: diegosouzapw/omniroute
       GHCR_IMAGE_NAME: ghcr.io/diegosouzapw/omniroute
+      # Turbopack on the 31 GB box; webpack on any hosted 7 GB fallback / ARM.
+      USE_TURBOPACK: ${{ (matrix.arch == 'amd64' && vars.USE_VPS_RUNNER == 'true') && '1' || '0' }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -143,6 +163,9 @@ jobs:
           ref: ${{ github.event_name == 'workflow_dispatch' && format('refs/tags/v{0}', inputs.version) || '' }}
           fetch-depth: 0
 
+      - name: Assert Docker Engine
+        run: docker info
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -166,6 +189,8 @@ jobs:
           context: .
           target: runner-base
           platforms: ${{ matrix.platform }}
+          build-args: |
+            OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
           tags: |
             ${{ env.IMAGE_NAME }}
@@ -183,6 +208,8 @@ jobs:
           context: .
           target: runner-web
           platforms: ${{ matrix.platform }}
+          build-args: |
+            OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
           tags: |
             ${{ env.IMAGE_NAME }}
@@ -208,6 +235,8 @@ jobs:
           file: Dockerfile.bun
           target: runner-base
           platforms: ${{ matrix.platform }}
+          build-args: |
+            OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
           tags: |
             ${{ env.IMAGE_NAME }}
@@ -233,6 +262,8 @@ jobs:
           file: Dockerfile.bun
           target: runner-web
           platforms: ${{ matrix.platform }}
+          build-args: |
+            OMNIROUTE_USE_TURBOPACK=${{ env.USE_TURBOPACK }}
           outputs: type=image,push-by-digest=true,name-canonical=true,push=true
           tags: |
             ${{ env.IMAGE_NAME }}

.github/workflows/quality.yml

@@ -70,7 +70,16 @@ jobs:
     # 2026-08-14: 72 of the last 100 PRs into release/** came from forks, so the fork case is
     # the majority of the traffic, not the exception — this job earns its place, it just should
     # not duplicate build.yml for the own-origin 28%.
-    if: ${{ github.event_name != 'pull_request' || ((github.event.pull_request.draft == false || startsWith(github.head_ref, 'mergify/merge-queue/')) && needs.changes.outputs.code == 'true' && github.event.pull_request.head.repo.full_name != github.repository) }}
+    # Disabled 2026-08-29 (#11976 follow-up). `continue-on-error: true` still
+    # reports a GitHub check FAILURE, so every fork PR into release/** was born
+    # with a red "Build (advisory)" even when every required gate was green
+    # (sweep-reds, 41 PRs). Hosted ubuntu-latest cannot finish `npm run build`
+    # on this tree — VM shutdown ~6 min in, same class as build.yml going
+    # workflow_dispatch-only in #11962. Pre-merge build signal for release/**
+    # is nightly-release-green (omni-build); for main it is ci.yml `Build`.
+    # Restore this job when a runner that actually fits the tree is wired here.
+    # Bare `false` (not `${{ false }}`) — zizmor obfuscation flags the expression form.
+    if: false
     # PINNED to hosted — this was the last job in THIS workflow still on the USE_VPS_RUNNER
     # switch (ci.yml's Build, nightly-release-green and npm-publish keep it, so the variable
     # stays meaningful), and with USE_VPS_RUNNER=true it produced NO signal at all here.

changelog.d/maintenance/11976-docker-publish-webpack-and-advisory-build.md

@@ -0,0 +1 @@
+- Stop painting every fork PR into `release/**` red: `quality.yml` `Build (advisory)` is skipped (GitHub still reports `continue-on-error` failures as check FAILURE). Hosted `ubuntu-latest` cannot finish `npm run build` on this tree — same class as #11962 taking `build.yml` off the PR rail. `docker-publish.yml` amd64 now runs on the `.113` `omni-build` pool (31 GB / 32 cores, two listeners) with Turbopack, shares the `heavy-build-main` lane with `ci.yml` `Build` so it queues instead of becoming a third heavy, and keeps per-ref concurrency (a merge storm was starting 8 concurrent OOM builds). arm64 stays on `ubuntu-24.04-arm` with webpack — there is no ARM box. Fallback when `USE_VPS_RUNNER` is off: hosted amd64 + webpack (#11976).

docs/ops/RUNNER_BOX.md

@@ -50,13 +50,25 @@ a time, only when idle**, with the idle check and the restart in the same comman
 ## Operating rules
 
 - **Heavy-build ceiling: 2 at a time — enforced by label.** Every job that runs a
-  `next build` (`ci.yml` `build`, `npm-publish.yml` `publish`, both `nightly-release-green`
-  validations) targets `[self-hosted, omni-build]`, and only **two** runners carry that
-  label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no
-  re-registration). The other six keep `omni-release` and take nothing heavy; GitHub
-  queues a third build instead of the kernel killing one. Pair with the `heavy-build-*`
-  concurrency lanes in `ci.yml`. To add capacity, label another runner — never raise
-  the count past what 31 GB holds (one next-build ≈ 14–16 GB).
+  full `next build` targets `[self-hosted, omni-build]`, and only **two** runners carry
+  that label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no
+  re-registration):
+  - `ci.yml` `Build` and `npm-publish.yml` `publish`
+  - both `nightly-release-green` validations
+  - `docker-publish.yml` **amd64** (the hosted 7 GB runner ResourceExhausted this
+    tree — #11976). The arm64 leg stays on `ubuntu-24.04-arm` (no ARM box) with
+    webpack.
+  The other listeners keep `omni-release` / `omni-light` and take nothing heavy;
+  GitHub queues a third build instead of the kernel killing one. Pair with the
+  `heavy-build-*` concurrency lanes: `ci.yml` `Build` on `main` and docker-publish
+  amd64 share `heavy-build-main` (`cancel-in-progress: false`) so a `:next` publish
+  waits beside the artefact instead of sitting next to it. PR builds use
+  `heavy-build-pr`. To add capacity, label another runner — never raise the count
+  past what 31 GB holds (one next-build ≈ 14–16 GB; a Docker amd64 build is the
+  same class plus the daemon — still one slot).
+  Docker Engine must be on those two units (`docker info` is the first step of
+  the publish job). If it is missing, the job fails closed instead of hanging on
+  `setup-buildx`.
 - **Light pool: `omni-light` (2026-08-29, #11965).** `omniroute-113` and `omniroute-113-2` carry
   `omni-light` for jobs that need a backend-only `next build` (~5–6 GB) but not a full one: the
   nightly Schemathesis, promptfoo, garak and axe-a11y jobs. They ran on the hosted 7 GB runner and

tests/unit/build/check-workflows.test.ts

@@ -318,7 +318,7 @@ test("readBaselineZizmorValue: invalid JSON returns null (does not throw)", () =
 // quality.yml — release PR build gate regression coverage (#7307)
 // ─────────────────────────────────────────────────────────────────────────────
 
-test("#7307 quality.yml adds an advisory production build for release PR code changes", () => {
+test("#7307 quality.yml keeps the advisory production build (disabled: hosted 7 GB cannot finish it)", () => {
   const source = readQualityWorkflow();
   const buildJob = source.match(/\n  build:\n[\s\S]*?\n  # Docs\/OpenAPI contract gates only/);
 
@@ -326,17 +326,15 @@ test("#7307 quality.yml adds an advisory production build for release PR code ch
   assert.ok(buildJob, "quality.yml must define the build job before docs-gates");
   assert.match(buildJob[0], /name: Build \(advisory\)/);
   assert.match(buildJob[0], /needs: changes/);
-  assert.match(buildJob[0], /needs\.changes\.outputs\.code == 'true'/);
-  assert.match(buildJob[0], /github\.event\.pull_request\.draft == false/);
-  assert.match(buildJob[0], /startsWith\(github\.head_ref, 'mergify\/merge-queue\/'\)/);
-  // FORK PRs ONLY (2026-08-14). build.yml's `Fast Production Build` fires on
-  // `push: branches: ["**"]` and runs the superset `build:release`, so own-origin branches
-  // were building twice; a fork's push never reaches this repo, making this their only
-  // pre-merge build signal — and forks are 72 of the last 100 PRs into release/**.
-  assert.match(
-    buildJob[0],
-    /github\.event\.pull_request\.head\.repo\.full_name != github\.repository/
-  );
+  // Live predicate is a static skip. `continue-on-error` still paints a GitHub
+  // check FAILURE, and the hosted runner ResourceExhausts this tree (#11976) —
+  // so the job must not run. Bare `if: false` (zizmor obfuscation flags
+  // `${{ false }}`). The body below is the restore recipe, not dead code.
+  const buildDirectives = buildJob[0]
+    .split("\n")
+    .filter((line) => !/^\s*#/.test(line))
+    .join("\n");
+  assert.match(buildDirectives, /\n {4}if: false\n/);
   // Runner PINNED to hosted. The self-hosted pool is 2 permanently-busy runners, where this
   // job either queued for hours or was killed by cancel-in-progress — ~10-15% of runs ever
   // reached a conclusion across 2026-08-13/14. It must NOT go back on the USE_VPS_RUNNER
@@ -345,10 +343,6 @@ test("#7307 quality.yml adds an advisory production build for release PR code ch
   // Check the DIRECTIVES, not the prose: the comment above legitimately explains why the
   // self-hosted pool was abandoned, so a naive /self-hosted/ scan over the whole block would
   // match its own rationale.
-  const buildDirectives = buildJob[0]
-    .split("\n")
-    .filter((line) => !/^\s*#/.test(line))
-    .join("\n");
   assert.doesNotMatch(buildDirectives, /self-hosted/);
   assert.doesNotMatch(buildDirectives, /USE_VPS_RUNNER/);
   // Memory provisioning mirrored from build.yml: --max-old-space-size bounds only V8's heap,