Compare commits

..

2 Commits

Author SHA1 Message Date
diegosouzapw
7c33560d22 docs(env): document OMNIROUTE_INSTALL_UPGRADE_WORKDIR
The workdir override introduced in this branch is a new `process.env.*` read, and
two gates caught it immediately: `issue #7793: real .env.example is in sync with
process.env.* reads in code` and `check:env-doc-sync` (Docs Sync STRICT).

Both were right — an env var that exists only in code is an env var nobody can
find. Documented in `.env.example` and `docs/reference/ENVIRONMENT.md` with the
reason it exists: the gate needs ~12 GB and must not land on a small tmpfs.
2026-08-27 23:38:15 -03:00
diegosouzapw
bc54433915 fix(ci): run the install-upgrade gate on disk, not on the /tmp tmpfs
The v3.8.50 publish failed this gate again, and this time it said why:

  free space in /tmp: 2.9 GB
  ⚠️  only 2.9 GB free — this gate needs roughly 12 GB
  crashed: upgrade install ran out of disk space (58269 ENOSPC errors)

On the self-hosted runner `/tmp` is a **12 GB tmpfs backed by RAM**, while the
root filesystem had 66 GB free. The gate builds two ~3 GB install trees, installs
the second one over twice, and packs a 275 MB tarball — roughly 12 GB, all of it
demanded from the wrong filesystem.

This is why freeing disk never fixed it: 84 GB were freed on `/`, and none of it
ever reached the volume the gate was using. The check even measured the right
number and reported it against the wrong path, so the warning read as "the disk
is full" when the disk was fine.

- work in `<repo>/.install-upgrade/` (gitignored) instead of `os.tmpdir()`,
  overridable with `OMNIROUTE_INSTALL_UPGRADE_WORKDIR`
- the free-space log and the ENOSPC crash message now name the directory the run
  actually uses, so the next reader is sent to the filesystem that ran out

Phase A already passes on the current main: clean install healthy, version
reported correctly, 130 tables — the authentication fix and migration 163 from
#11845 both hold. Only Phase B was starved.
2026-08-27 23:12:49 -03:00
46 changed files with 210 additions and 1914 deletions

View File

@@ -606,24 +606,13 @@ jobs:
# Dynamic runner: when the release captain flips the USE_VPS_RUNNER repo var to
# 'true' (scripts/vps/release-runner-up.sh does it after the self-hosted VM is
# online), the heavy jobs run on the dedicated 32-core VPS runners (label
# omni-build) instead of queueing on the 20-concurrent-job hosted pool.
# omni-release) instead of queueing on the 20-concurrent-job hosted pool.
# Safety: fork PRs NEVER reach the self-hosted runner — the expression falls
# back to ubuntu-latest unless the PR head repo is this repository (push /
# dispatch events are own-origin by definition). Any failure path (VM down,
# var unset/false) also falls back to ubuntu-latest.
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-latest' }}
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }}
needs: changes
# The .113 pool runs ONE next-build with room to spare and two at the edge: the
# box has 31 GB and a single next-build peaks at 1416 GB RSS. On 2026-08-28
# 13:50Z the kernel OOM-killed main's build while a PR build ran beside it
# (five Build jobs had been queued by a burst of PRs). Two lanes: main keeps
# its own so a release is never queued behind PR traffic; PR builds serialize
# among themselves. GitHub keeps one running + one pending per group and
# CANCELS older pendings — a cancelled PR build is re-runnable; a dead main
# build costs the publish its artefact and a 40-minute rebuild that OOMs.
concurrency:
group: heavy-build-${{ github.ref == 'refs/heads/main' && 'main' || 'pr' }}
cancel-in-progress: false
if: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.code == 'true' && github.event.pull_request.draft == false) }}
steps:
- uses: actions/checkout@v7
@@ -657,14 +646,14 @@ jobs:
# Keep standalone/node_modules intact: package/electron jobs consume the
# Next-traced standalone tree and must not replace it with root node_modules.
run: |
tar -czf "$RUNNER_TEMP/e2e-build.tar.gz" \
tar -czf /tmp/e2e-build.tar.gz \
--exclude='.build/next/cache' \
.build/next
- name: Upload Next.js build for downstream jobs
uses: actions/upload-artifact@v7
with:
name: next-build
path: ${{ runner.temp }}/e2e-build.tar.gz
path: /tmp/e2e-build.tar.gz
retention-days: 1
package-artifact:
@@ -687,14 +676,10 @@ jobs:
uses: actions/download-artifact@v8
with:
name: next-build
# Workspace-relative on purpose: the matrix below includes windows-latest, whose
# default shell is pwsh, where $RUNNER_TEMP is empty (it is $env:RUNNER_TEMP) —
# #11896's first cut broke the Electron smoke on exactly that. A relative path
# works in bash and pwsh alike; hosted workspaces are ephemeral.
path: next-build-artifact
path: /tmp/
- name: Extract Next.js build artifact
run: |
tar -xzf next-build-artifact/e2e-build.tar.gz
tar -xzf /tmp/e2e-build.tar.gz
# build:cli consumes the downloaded .build/next standalone artifact and assembles dist/;
# it only rebuilds if the downloaded standalone artifact is missing.
- run: npm run build:cli
@@ -782,14 +767,10 @@ jobs:
uses: actions/download-artifact@v8
with:
name: next-build
# Workspace-relative on purpose: the matrix below includes windows-latest, whose
# default shell is pwsh, where $RUNNER_TEMP is empty (it is $env:RUNNER_TEMP) —
# #11896's first cut broke the Electron smoke on exactly that. A relative path
# works in bash and pwsh alike; hosted workspaces are ephemeral.
path: next-build-artifact
path: /tmp/
- name: Extract Next.js build artifact
run: |
tar -xzf next-build-artifact/e2e-build.tar.gz
tar -xzf /tmp/e2e-build.tar.gz
- name: Install Electron dependencies
working-directory: electron
run: npm install --no-audit --no-fund
@@ -976,11 +957,7 @@ jobs:
# 10min was sized before #7114 added the lcov reporter (Codecov/Sonar need it);
# merging 8 shard JSONs + text+json+lcov now takes ~10-12min — three consecutive
# release-tip runs died at exactly 10m as job-timeout "cancelled" (2026-07-15/16).
# 30, not 20 (2026-08-29): the informational Codecov upload below hung for the rest of
# the budget on two consecutive main runs (33207760653, 33215115341); the job ended
# `cancelled` and dragged the whole run's conclusion to `cancelled` although every
# blocking job was green. The upload step now has its own ceiling; this is headroom.
timeout-minutes: 30
timeout-minutes: 20
needs: test-unit
if: ${{ !cancelled() && needs.test-unit.result == 'success' && !contains(github.event.pull_request.labels.*.name, 'hotfix') }}
env:
@@ -1059,10 +1036,6 @@ jobs:
# (if-no-files-found: warn) — Sonar consumes the same file.
- name: Upload coverage to Codecov (informational)
if: always()
# Informational means informational: its own ceiling and continue-on-error, so a
# stalled upload can neither eat the job's budget nor turn a green job cancelled.
timeout-minutes: 5
continue-on-error: true
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
files: coverage/lcov.info
@@ -1257,14 +1230,10 @@ jobs:
uses: actions/download-artifact@v8
with:
name: next-build
# Workspace-relative on purpose: the matrix below includes windows-latest, whose
# default shell is pwsh, where $RUNNER_TEMP is empty (it is $env:RUNNER_TEMP) —
# #11896's first cut broke the Electron smoke on exactly that. A relative path
# works in bash and pwsh alike; hosted workspaces are ephemeral.
path: next-build-artifact
path: /tmp/
- name: Extract Next.js build artifact
run: |
tar -xzf next-build-artifact/e2e-build.tar.gz
tar -xzf /tmp/e2e-build.tar.gz
# WS4.1: duration-balanced shards (LPT over config/quality/e2e-timings.json).
# Measured skew of plain --shard was 14× (24m47s vs 1m47s) — E2E was the CI
# critical path. The balancer self-verifies completeness and exits non-zero on

View File

@@ -485,9 +485,6 @@ jobs:
severity: CRITICAL
ignore-unfixed: true
exit-code: "1"
# Explicit: the advisory scan above already points at it, and the blocking
# gate must honour the same accepted-risk list (#12084).
trivyignores: .trivyignore
- name: Upload Trivy SARIF to Security tab
if: needs.prepare.outputs.version != 'main'

View File

@@ -4,21 +4,12 @@ on:
push:
tags:
- "v*"
# A dispatch builds the ref it is dispatched ON (`gh workflow run … --ref v3.8.50` rebuilds
# that tag; `--ref main` builds the repaired line). The ref is deliberately NOT an input:
# CodeQL flags an input-controlled checkout next to the npm cache on the default branch as
# cache poisoning (actions/cache-poisoning/poisonable-step), and `github.ref` is trusted.
workflow_dispatch:
inputs:
version:
description: "Release version (e.g., v1.6.8)"
required: true
type: string
publish_npm:
description: "Also run the npm publish leg (turn off when re-attaching desktop assets to a release whose npm package already shipped)"
required: false
default: true
type: boolean
# Least-privilege default: read-only at the top level; each job grants the writes it
# needs (build/release upload assets, publish-npm forwards npm provenance / packages
@@ -413,14 +404,7 @@ jobs:
tag_name: ${{ needs.validate.outputs.version }}
draft: false
prerelease: false
# NEVER. Phase 3 of the release flow creates the GitHub Release with the curated
# notes seconds after pushing the tag, so by the time this step runs (1-2 h of
# builds later) the body already exists — and `true` APPENDS GitHub's
# auto-generated "What's Changed" block to it (v3.8.48 shipped that way; the
# v3.8.50 re-attach dispatch added +1,416 chars to a 121 KB body, run
# 33238093090). A curated body sits ~3 KB under the 125,000-char cap, so the
# append can also turn this step RED and leave the release with no assets.
generate_release_notes: false
generate_release_notes: true
fail_on_unmatched_files: false
files: |
release-assets/*.dmg
@@ -478,20 +462,11 @@ jobs:
publish-npm:
name: Publish to npm
needs: [validate, release]
# A re-dispatch that only re-attaches desktop assets must not publish the npm package again.
if: ${{ github.event_name != 'workflow_dispatch' || inputs.publish_npm }}
permissions:
# Must be `write`, not `read`: this job calls the reusable npm-publish.yml whose
# `publish` job needs `contents: write` (gh release upload — attach the SBOM, #3874).
# A reusable workflow's job cannot request more permission than the caller grants,
# so a `read` here makes GitHub reject the run at startup (startup_failure).
#
# `actions: read` for the same reason: the called `publish` job downloads the next-build
# artefact and requests it. v3.8.50 (run 33005490476) died at startup with "The nested
# job 'publish' is requesting 'actions: read', but is only allowed 'actions: none'" — and
# because `release` lives in this same workflow, the tag shipped with ZERO assets. Keep
# this block a superset of every job's permissions in npm-publish.yml.
actions: read
contents: write
id-token: write # npm provenance (forwarded to the reusable workflow)
packages: write # publish to npm.pkg.github.com

View File

@@ -68,7 +68,7 @@ jobs:
# this runs on the dedicated VPS runner — clean env (no operator OMNIROUTE_API_KEY,
# no local noauth CLIs => zero machine-specific false positives) and no contention.
# Nightly cron normally finds the var false (VM off) and falls back to hosted.
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]')) || 'ubuntu-latest' }}
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-release"]')) || 'ubuntu-latest' }}
env:
JWT_SECRET: ci-nightly-secret-with-sufficient-length-for-validation
API_KEY_SECRET: ci-nightly-api-key-secret-long
@@ -196,26 +196,6 @@ jobs:
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --label base-red --body-file issue-body.md
fi
- name: Close tracking issue when the branch is green again
if: steps.validate.outputs.exit == '0'
env:
GH_TOKEN: ${{ github.token }}
TARGET: ${{ steps.branch.outputs.target }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
# The open/update step above is the UPWARD half of the loop; without this
# step a stale "not green" issue outlives the fix and every base-green check
# (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited.
TITLE="🔴 Release branch not green: ${TARGET}"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
--search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "")
if [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \
--comment "✅ \`${TARGET}\` is release-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)."
echo "Closed issue #$EXISTING"
fi
- name: Upload report artifact
if: always()
uses: actions/upload-artifact@v7
@@ -237,7 +217,7 @@ jobs:
# On a push, only run for a push to main — a push to release/* is handled by
# release-green above. Schedule/dispatch always run (they also sweep main).
if: ${{ github.event_name != 'push' || github.ref_name == 'main' }}
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-build"]')) || 'ubuntu-latest' }}
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && fromJSON('["self-hosted","omni-release"]')) || 'ubuntu-latest' }}
env:
JWT_SECRET: ci-nightly-secret-with-sufficient-length-for-validation
API_KEY_SECRET: ci-nightly-api-key-secret-long
@@ -314,25 +294,6 @@ jobs:
gh issue create --repo "$GITHUB_REPOSITORY" --title "$TITLE" --label base-red --body-file issue-body.md
fi
- name: Close tracking issue when the branch is green again
if: steps.validate.outputs.exit == '0'
env:
GH_TOKEN: ${{ github.token }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
# The open/update step above is the UPWARD half of the loop; without this
# step a stale "not green" issue outlives the fix and every base-green check
# (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited.
TITLE="🔴 main branch not green"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
--search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "")
if [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \
--comment "✅ \`main\` is main-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)."
echo "Closed issue #$EXISTING"
fi
- name: Upload report artifact
if: always()
uses: actions/upload-artifact@v7

View File

@@ -23,12 +23,11 @@ on:
- next
- historic
publish_mode:
description: "auto = publish through npm Trusted Publishing (OIDC, no token, no 2FA prompt — the default); staged = npm stage publish (owner approves with 2FA); direct = legacy token publish (emergency fallback only)"
description: "staged = npm stage publish (owner approves with 2FA after the staged boot-verify); direct = legacy immediate publish (emergency fallback only)"
required: false
default: "auto"
default: "staged"
type: choice
options:
- auto
- staged
- direct
workflow_call:
@@ -63,15 +62,11 @@ jobs:
# mid-"Creating an optimized production build" while v3.8.48 had still fit in 16min.
# This job never runs on `pull_request`, so the fork-safety clause is always true here;
# it is kept verbatim so the expression stays greppable against ci.yml.
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-build"]') || 'ubuntu-latest' }}
outputs:
version: ${{ steps.resolve.outputs.version }}
tag: ${{ steps.resolve.outputs.tag }}
skip: ${{ steps.resolve.outputs.skip }}
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }}
permissions:
actions: read # find + download the CI run's next-build artifact for this SHA
contents: write # gh release upload (attach SBOM to the GitHub Release)
id-token: write # npm provenance (GitHub Packages step)
id-token: write # npm provenance
packages: write # publish to npm.pkg.github.com
steps:
- name: Checkout
@@ -205,11 +200,8 @@ jobs:
exit 0
fi
RUN=""
# $RUNNER_TEMP, never /tmp: on the .113 pool /tmp is a 12 GB tmpfs (RAM). Parking
# this 1.3 GB artefact there took 2732 min of the 76-min publish job — the
# same bytes upload from disk in 2 min. RUNNER_TEMP is per-runner and on disk.
for candidate in $CANDIDATES; do
if gh run download "$candidate" --repo "$REPO" --name next-build --dir "$RUNNER_TEMP/next-build" 2>/dev/null; then
if gh run download "$candidate" --repo "$REPO" --name next-build --dir /tmp/next-build 2>/dev/null; then
RUN="$candidate"
break
fi
@@ -219,8 +211,8 @@ jobs:
echo "::notice::none of the candidate runs still carries next-build (1-day retention) — falling back to a full build"
exit 0
fi
tar -xzf "$RUNNER_TEMP/next-build/e2e-build.tar.gz" -C .
rm -rf "$RUNNER_TEMP/next-build"
tar -xzf /tmp/next-build/e2e-build.tar.gz -C .
rm -rf /tmp/next-build
if [ -f .build/next/standalone/server.js ]; then
echo "✅ standalone tree restored from CI run $RUN — build:cli will skip next build"
else
@@ -273,20 +265,11 @@ jobs:
if-no-files-found: error
- name: Attach SBOM to GitHub Release
# Not only on the `release` event: the v3.8.50 package shipped through a
# workflow_dispatch (staged publish, 11 attempts) and this step was skipped, so the
# GitHub Release carried no SBOM until it was attached by hand from the run's
# `sbom-npm` artifact. Attach whenever a release for the published tag exists.
if: steps.resolve.outputs.skip != 'true' && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
if: steps.resolve.outputs.skip != 'true' && github.event_name == 'release'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ github.event_name == 'release' && github.ref_name || format('v{0}', inputs.version) }}
run: |
if ! gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "::notice::no GitHub Release for $TAG yet — SBOM stays on the sbom-npm workflow artifact"
exit 0
fi
gh release upload "$TAG" sbom-npm.cdx.json --repo "$GITHUB_REPOSITORY" --clobber
TAG: ${{ github.ref_name }}
run: gh release upload "$TAG" sbom-npm.cdx.json --clobber
# WS1.2/WS1.3 (#7065 class): the artifact that is about to be published must
# BOOT. build:cli already assembled dist/ above; this packs+installs+boots the
@@ -331,133 +314,17 @@ jobs:
fi
npm --version
# The registry upload itself moved to the `stage-npm` job below: npm REFUSES
# `--provenance` from a self-hosted runner (422 "Unsupported GitHub Actions
# runner environment"), and the heavy verification above cannot move to a
# hosted one (16 GB is not enough for build:cli's next-build fallback — see
# this job's runs-on comment). So this job proves the bytes and hands them
# over; a tiny hosted job does the upload.
- name: Pack the verified tarball for the upload job
if: steps.resolve.outputs.skip != 'true'
env:
VERSION: ${{ steps.resolve.outputs.version }}
run: |
set -euo pipefail
# --ignore-scripts: prepublishOnly would re-run build:cli-api && build:cli,
# rebuilding bytes this job has already built, validated and boot-smoked.
npm pack --ignore-scripts
TARBALL="omniroute-${VERSION}.tgz"
test -f "$TARBALL" || { echo "expected $TARBALL to exist after npm pack" >&2; ls -la ./*.tgz || true; exit 1; }
echo "packed $TARBALL ($(du -h "$TARBALL" | cut -f1))"
- name: Hand the tarball to the hosted publish job
if: steps.resolve.outputs.skip != 'true'
uses: actions/upload-artifact@v7
with:
name: npm-tarball
path: omniroute-${{ steps.resolve.outputs.version }}.tgz
retention-days: 1
if-no-files-found: error
- name: Publish to GitHub Packages
if: steps.resolve.outputs.skip != 'true'
- name: Publish to npm (staged — owner approves with 2FA)
if: steps.resolve.outputs.skip != 'true' && (github.event_name != 'workflow_dispatch' || inputs.publish_mode != 'direct')
env:
VERSION: ${{ steps.resolve.outputs.version }}
TAG: ${{ steps.resolve.outputs.tag }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
echo "Configuring for GitHub Packages..."
echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" > .npmrc
npm pkg set name="@diegosouzapw/omniroute"
npm publish --registry=https://npm.pkg.github.com --tag "$TAG" \
|| echo "⚠️ omniroute@${VERSION} might already be published on GitHub Packages."
echo "✅ Action finished for GitHub Packages"
# npm REFUSES `--provenance` from a self-hosted runner:
# 422 Unprocessable Entity - Error verifying sigstore provenance bundle:
# Unsupported GitHub Actions runner environment: "self-hosted".
# Only "github-hosted" runners are supported when publishing with provenance.
# v3.8.49 published fine because it predates USE_VPS_RUNNER being turned on
# (2026-08-02); v3.8.50 was the first release after it, so this had been latent
# for four weeks. Dropping --provenance was not an option: 3.8.49 carries a
# SLSA attestation and 3.8.50 must not regress that.
# The `publish` job cannot simply move to a hosted runner either — 16 GB is not
# enough for build:cli's next-build fallback. So it keeps proving the bytes and
# this job, which needs no memory at all, performs the upload.
stage-npm:
needs: publish
if: needs.publish.outputs.skip != 'true'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write # npm provenance — the whole reason this job is separate
steps:
- name: Download the tarball the publish job proved
uses: actions/download-artifact@v8
with:
name: npm-tarball
path: .
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: ${{ env.NPM_PUBLISH_NODE_VERSION }}
registry-url: https://registry.npmjs.org
- name: Ensure npm supports staged publishing
run: |
set -euo pipefail
CUR=$(npm --version)
if ! node -e "const [a,b]='$(npm --version)'.split('.').map(Number); process.exit(a>11||(a===11&&b>=15)?0:1)"; then
# Pinned exact version (supply-chain: never float @latest in a publish
# job); bump deliberately when a newer npm is required.
echo "npm $CUR < 11.15 — installing pinned npm 11.15.0 for staged publishing"
npm install -g --ignore-scripts npm@11.15.0
fi
npm --version
# Trusted Publishing (OIDC): npm mints a short-lived credential for THIS run from
# GitHub's id-token — no NPM_TOKEN secret, no 2FA prompt, provenance included, and
# it is the bypass npm sanctions now that tokens which skip 2FA are being retired
# (gh.io/npm-gat-bypass2fa-deprecation). Requires the package's Trusted Publisher to
# be configured on npmjs.com (owner: diegosouzapw/OmniRoute, workflow
# npm-publish.yml) and a github-hosted runner — which is why this job exists.
# Without that configuration `npm publish` fails with ENEEDAUTH: re-dispatch with
# publish_mode=staged or direct. Automatic publishing was the flow up to v3.8.48;
# v3.8.49 moved to staged (WS1.3) to keep a leaked token from publishing alone —
# OIDC gives the same guarantee without the manual approve.
- name: Publish to npm (Trusted Publishing / OIDC — automatic)
if: github.event_name != 'workflow_dispatch' || inputs.publish_mode == 'auto'
env:
VERSION: ${{ needs.publish.outputs.version }}
TAG: ${{ needs.publish.outputs.tag }}
run: |
set -euo pipefail
TARBALL="omniroute-${VERSION}.tgz"
test -f "$TARBALL" || { echo "tarball $TARBALL did not arrive from the publish job" >&2; ls -la; exit 1; }
# Deliberately NO NODE_AUTH_TOKEN in this step: npm >= 11.5 detects the GitHub
# OIDC token itself. Always pass --tag explicitly (defense in depth: an older
# VERSION can never claim `@latest`).
npm publish "$TARBALL" --provenance --access public --tag "$TAG" --ignore-scripts
echo "✅ Published omniroute@$VERSION (dist-tag=$TAG) via Trusted Publishing"
- name: Publish to npm (staged — owner approves with 2FA)
# Only on an explicit request now: Trusted Publishing below is the default.
if: github.event_name == 'workflow_dispatch' && inputs.publish_mode == 'staged'
env:
VERSION: ${{ needs.publish.outputs.version }}
TAG: ${{ needs.publish.outputs.tag }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
TARBALL="omniroute-${VERSION}.tgz"
test -f "$TARBALL" || { echo "tarball $TARBALL did not arrive from the publish job" >&2; ls -la; exit 1; }
# Always pass --tag explicitly. Defense in depth: even if VERSION is
# accidentally an older release, the historic tag will NOT claim `@latest`.
# --ignore-scripts: publishing a built tarball must never re-run
# prepublishOnly (build:cli-api && build:cli) on this small runner.
npm stage publish "$TARBALL" --provenance --access public --tag "$TAG" --ignore-scripts
npm stage publish --provenance --access public --tag "$TAG"
{
echo "## 📦 omniroute@$VERSION STAGED (not yet installable)"
echo ""
@@ -473,18 +340,31 @@ jobs:
echo "✅ Staged omniroute@$VERSION (dist-tag=$TAG) — awaiting owner 'npm stage approve'"
- name: Publish to npm (DIRECT — emergency fallback)
if: github.event_name == 'workflow_dispatch' && inputs.publish_mode == 'direct'
if: steps.resolve.outputs.skip != 'true' && github.event_name == 'workflow_dispatch' && inputs.publish_mode == 'direct'
env:
VERSION: ${{ needs.publish.outputs.version }}
TAG: ${{ needs.publish.outputs.tag }}
VERSION: ${{ steps.resolve.outputs.version }}
TAG: ${{ steps.resolve.outputs.tag }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
set -euo pipefail
TARBALL="omniroute-${VERSION}.tgz"
test -f "$TARBALL" || { echo "tarball $TARBALL did not arrive from the publish job" >&2; ls -la; exit 1; }
npm publish "$TARBALL" --provenance --access public --tag "$TAG" --ignore-scripts
npm publish --provenance --access public --tag "$TAG"
echo "✅ Published omniroute@$VERSION (dist-tag=$TAG) [DIRECT mode]"
- name: Publish to GitHub Packages
if: steps.resolve.outputs.skip != 'true'
env:
VERSION: ${{ steps.resolve.outputs.version }}
TAG: ${{ steps.resolve.outputs.tag }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
echo "Configuring for GitHub Packages..."
echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" > .npmrc
npm pkg set name="@diegosouzapw/omniroute"
npm publish --registry=https://npm.pkg.github.com --tag "$TAG" \
|| echo "⚠️ omniroute@${VERSION} might already be published on GitHub Packages."
echo "✅ Action finished for GitHub Packages"
publish-opencode-plugin:
runs-on: ubuntu-latest
permissions:

View File

@@ -4,15 +4,12 @@ on:
schedule:
- cron: "27 7 * * 1"
push:
# Scorecard only accepts the DEFAULT branch — here the active release/vX.Y.Z,
# not `main`. The job below guards on it so a push to any other branch skips.
branches: ["main", "release/**"]
branches: ["main"]
permissions: read-all
jobs:
analysis:
if: ${{ github.event_name != 'push' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
name: Scorecard analysis
runs-on: ubuntu-latest
permissions:

View File

@@ -19,12 +19,4 @@
# Keep this list SHORT and reviewed every release. Prefer fixing (rebuild on a
# patched base / bump the dep) over suppressing. Stale entries are debt.
#
# CVE-2025-68121 — Go stdlib crypto/tls (session-resumption certificate validation)
# inside the PREBUILT bogdanfinn/tls-client v1.15.1 .so that tls-client-node's
# postinstall downloads (built with go 1.24.1; fixed in 1.24.13). No upstream
# rebuild exists (v1.15.1 is still the latest release) and nothing in this repo
# can bump it. The binary is only loaded by the browser-TLS web-provider
# executors (claude-web / grok-web / lmarena / perplexity-web / notion-web),
# whose handshakes go through utls. Tracking issue: #12084. Revisit at the next
# tls-client release or base-image bump and BEFORE the v3.8.51 tag (2026-09-15).
CVE-2025-68121
# (No accepted-risk suppressions at present — ignore-unfixed covers the noise.)

View File

@@ -184,29 +184,19 @@ ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_BUILD_MEMORY_MB}"
# silently leaving no standalone bundle. Next derives the worker count from
# CIRCLE_NODE_TOTAL (workers = N-1). (#10060)
#
# Lowered 8 → 3 (7 workers → 2) in #11419, then 3 → 2 (2 workers → 1) in #7518.
# Every page-data worker inherits NODE_OPTIONS above, so the ceiling is per
# PROCESS, not per build: 7 workers on a 16 GB GitHub runner (ubuntu-24.04 /
# ubuntu-24.04-arm, 4 vCPU) exhausted the host and buildkit failed the whole
# step with `ResourceExhausted: ... cannot allocate memory`. The compile phase
# always finished ("✓ Compiled successfully in 4.2min"); the kernel killed the
# build right after "Collecting page data using N workers".
#
# #11419's first fix (8 → 3) modeled the per-worker peak as an INFERENCE
# (2560 MB, guessed from "7 workers didn't fit") and assumed the parent
# process's RSS tracked the V8 heap ceiling. Both assumptions were wrong: a
# live VPS reproduction (issue #7518, dmesg OOM-killer report) measured the
# real per-process RSS directly at ~4.5 GB, independent of the NODE_OPTIONS
# heap flag (Turbopack itself is native/Rust, outside the V8 heap) — and it
# applies to the parent process too, not just workers. 2 workers (3 processes
# × 4.5 GB = 13.5 GB) still didn't fit the 12.288 GB (75%) budget on a 16 GB
# runner, matching the still-live publish failures after #11419 merged. 1
# worker (2 processes × 4.5 GB = 9 GB) fits with headroom to spare.
# tests/unit/docker-build-memory-budget.test.ts does the arithmetic against
# the measured figure and fails if either knob is raised past what a 16 GB
# runner holds. Override for a big builder: `--build-arg
# OMNIROUTE_BUILD_WORKERS=8`.
ARG OMNIROUTE_BUILD_WORKERS=2
# Lowered 8 → 3 (7 workers → 2). Every page-data worker inherits NODE_OPTIONS
# above, so the ceiling is per PROCESS, not per build: 7 workers on a 16 GB
# GitHub runner (ubuntu-24.04 / ubuntu-24.04-arm, 4 vCPU) exhausted the host and
# buildkit failed the whole step with `ResourceExhausted: ... cannot allocate
# memory`. The compile phase always finished ("✓ Compiled successfully in
# 4.2min"); the kernel killed the build right after "Collecting page data using
# 7 workers". It was intermittent for a while and went 100% on 2026-08-22, which
# is what a threshold being crossed by ordinary codebase growth looks like.
# tests/unit/docker-build-memory-budget.test.ts does the arithmetic and fails if
# either knob is raised past what a 16 GB runner holds. 2 workers also stops
# oversubscribing the runner's 4 vCPU, which 7 did. Override for a big builder:
# `--build-arg OMNIROUTE_BUILD_WORKERS=8`.
ARG OMNIROUTE_BUILD_WORKERS=3
ENV CIRCLE_NODE_TOTAL=${OMNIROUTE_BUILD_WORKERS}
COPY . ./

View File

@@ -1,5 +1,5 @@
# ── Multi-stage Dockerfile for Native Bun Runtime (web-latest-bun) ───────────
FROM oven/bun:1.4.0-slim AS base
FROM oven/bun:1.3.14-slim AS base
WORKDIR /app
RUN apt-get update \
@@ -34,10 +34,8 @@ RUN if [ -f "node_modules/tls-client-node/scripts/postinstall.js" ]; then \
bun node_modules/tls-client-node/scripts/postinstall.js || true; \
fi
# Turbopack is supported on Bun 1.4+ (Next 16.3); override via
# --build-arg OMNIROUTE_USE_TURBOPACK=0 to force the webpack fallback.
ARG OMNIROUTE_USE_TURBOPACK=1
ENV OMNIROUTE_USE_TURBOPACK=${OMNIROUTE_USE_TURBOPACK}
# Disable Turbopack for Bun builder stage (Turbopack V8 internal worker bindings require Node)
ENV OMNIROUTE_USE_TURBOPACK=0
ARG OMNIROUTE_BASE_PATH=""
ENV OMNIROUTE_BASE_PATH=$OMNIROUTE_BASE_PATH
@@ -48,33 +46,6 @@ ENV DASHBOARD_ALLOW_EMBED=$DASHBOARD_ALLOW_EMBED
ENV NEXT_TELEMETRY_DISABLED=1
ENV NODE_ENV=production
# Cap the Next.js build heap and page-data worker pool inside the Bun image the
# same way the node Dockerfile does (#10060/#11419/#7518). Without these knobs
# Next falls back to its defaults: worker pool = os.cpus()-1 (3 on the 4-vCPU
# GitHub runner) and an 8 GB V8 heap ceiling per process. 4+ V8 processes at
# multi-GB each blow past the 16 GB runner, the cgroup OOM killer SIGKILLs a
# build worker mid-compile, and buildx fails the step with `ResourceExhausted:
# ... cannot allocate memory` — every Bun image published on main since the -bun
# targets landed (#11709, #11039).
#
# The per-process peak is a MEASURED ~4.5 GB RSS (dmesg OOM-killer report,
# #7518), independent of NODE_OPTIONS — Turbopack is native/Rust and compiles
# outside the V8 heap — and it applies to the parent process too, so 2 page-data
# workers (3 processes × 4.5 GB ≈ 13.5 GB) do not fit the 12.288 GB (75%)
# budget either. Both images therefore default to OMNIROUTE_BUILD_WORKERS=2
# (1 page-data worker): 2 processes × 4.5 GB ≈ 9 GB fits with headroom (#11663).
# The default Turbopack path keeps the compile outside the V8 heap, but the
# guards must hold for the webpack fallback (OMNIROUTE_USE_TURBOPACK=0) too, so
# they are wired exactly like the node image.
#
# NODE_OPTIONS propagates to the spawned `next build` child and its workers
# (build-next-isolated.mjs → resolveNextBuildEnv spreads process.env), so the
# ceiling is per PROCESS, not per build.
ARG OMNIROUTE_BUILD_MEMORY_MB=6144
ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_BUILD_MEMORY_MB}"
ARG OMNIROUTE_BUILD_WORKERS=2
ENV CIRCLE_NODE_TOTAL=${OMNIROUTE_BUILD_WORKERS}
# Bun native Next.js build execution
RUN bun run --quiet build

View File

@@ -1,4 +0,0 @@
- The npm publish is automatic again, through npm Trusted Publishing (OIDC): the hosted
`stage-npm` job publishes with a short-lived credential minted from GitHub's id-token —
no `NPM_TOKEN`, no 2FA prompt, provenance attached. `publish_mode=staged` (owner
approves with 2FA) and `direct` (token) remain available on `workflow_dispatch`.

View File

@@ -1 +0,0 @@
- **fix(docker):** bump the Bun image to 1.4.0, enable Turbopack on Bun, and port the node image's build memory guards so the `-bun` container builds fit the 16 GB GitHub runner instead of dying with `cannot allocate memory` ([#11719](https://github.com/diegosouzapw/OmniRoute/pull/11719)). Both images now default `OMNIROUTE_BUILD_WORKERS` to `2` (1 page-data worker) against the measured ~4.5 GB per-process RSS budget (#7518/#11663).

View File

@@ -1,4 +0,0 @@
- Excluded Next.js Node File Trace manifests (`*.nft.json`) from the published npm
tarball. They are build-time metadata and are never read while serving, but had
grown to 668.7 MB — 61% of the package — which pushed the upload past the
registry limit and made `npm publish` fail with `413 Payload Too Large`.

View File

@@ -1,5 +0,0 @@
- Fixed the Alibaba free-tier allowlist test that went red on its own once the
shipped catalog's `validUntil` (2026-08-27) passed, leaving every PR and `main`
with a failing `Unit Tests (1/8)`. The test now builds its own packs with dates
it controls, and covers the expired-pack fallback that production has actually
been serving.

View File

@@ -1,4 +0,0 @@
- Split the npm registry upload into its own GitHub-hosted job. npm refuses
`--provenance` from a self-hosted runner (`422 ... Only "github-hosted" runners
are supported`), which blocked the v3.8.50 publish; the heavy verification
cannot move to a hosted runner, so it now hands the proven tarball over instead.

View File

@@ -1 +0,0 @@
- Electron release: `electron/package-lock.json` regained the optional `electron-builder-squirrel-windows` subtree (13 entries) that `npm ci` had been refusing as out of sync — the Linux desktop leg died on it — and `electron-release.yml` gained a `build_ref` dispatch input so a release whose tag was cut with the broken lock can have its assets rebuilt from the repaired line

View File

@@ -1 +0,0 @@
- Electron release workflow: the `publish-npm` job now grants `actions: read` to the reusable `npm-publish.yml` it calls (its `publish` job requests it), which is what made GitHub refuse the whole v3.8.50 run at startup and ship the release with zero desktop assets; a `workflow_dispatch` now builds the requested tag instead of the dispatching branch and can skip the npm leg (`publish_npm=false`) when only re-attaching assets

View File

@@ -1 +0,0 @@
- npm publish workflow: the CycloneDX SBOM is attached to the GitHub Release on `workflow_dispatch` publishes too (when a release for the tag exists), not only on the `release` event — v3.8.50 shipped through a staged dispatch and its release carried no SBOM until it was attached by hand from the run's `sbom-npm` artifact

View File

@@ -1,4 +0,0 @@
- Added a unit test that fails seven days before any dated pack under `config/`
(`validUntil` and sibling keys) lapses, naming the file and key. The Alibaba
free-tier pack expired on 2026-08-27 and turned every PR red the next morning
with no commit involved; renewal now happens on someone's terms, not the clock's.

View File

@@ -1,3 +0,0 @@
- `check:workflows` now fails (under `--strict`/`--ratchet`) when any job routed to a
self-hosted runner publishes with `--provenance` — npm rejects that with `422` at the
registry, which in v3.8.50 only surfaced after the tag and Docker images were public.

View File

@@ -1,5 +0,0 @@
- `scripts/ops/runner-janitor.sh` now proves a path is idle with one `lsof`
snapshot and removes stale leftovers itself (tmpfs after 3 h — it is RAM — disk
after 24 h), kills orphan `next-build` processes, prunes checkouts of stopped
runners, and alerts on memory pressure; `--dry-run` shows exactly what it would
do. `docs/ops/RUNNER_BOX.md` reconciled to the measured box (31 GB, 10 listeners).

View File

@@ -1,5 +0,0 @@
- The `next-build` artefact (1.3 GB) is now written and read under `$RUNNER_TEMP`
(per-runner, on disk) instead of `/tmp`, which on the self-hosted pool is a
12 GB tmpfs in RAM. Landing it there took 2732 of the publish job's 76 minutes,
and the fixed `/tmp/e2e-build.tar.gz` name let E2E jobs on different runners
overwrite each other's download.

View File

@@ -1,3 +0,0 @@
- The CI `build` job now runs in two concurrency lanes — `main` and pull requests —
so a release build is never queued behind (or OOM-killed beside) PR builds on the
self-hosted pool, which holds one `next-build` comfortably and two at the edge.

View File

@@ -1 +0,0 @@
- `Coverage` job on `ci.yml`: the informational Codecov upload gets its own 5-minute ceiling and `continue-on-error`, and the job budget grows from 20 to 30 minutes (the 8-shard c8 merge alone takes ~10) — a stalled upload no longer ends the job `cancelled` and drags a fully green `main` run's conclusion down with it

View File

@@ -1,4 +0,0 @@
- Every CI job that runs a `next build` (`build`, the npm `publish`, both release-green
validations) now targets the `omni-build` runner label, which only two of the eight
self-hosted runners carry. The box holds one build comfortably and two at the edge; a
third now queues on GitHub instead of being OOM-killed by the kernel.

View File

@@ -169,7 +169,7 @@
"dedicatedGate": true
},
"zizmorFindings": {
"value": 194,
"value": 192,
"_rebaseline_2026_08_20_radar_export_workflow": "190 -> 192 (+2). Workflow novo `.github/workflows/radar-export.yml` (passo 10 do go-live do Radar: publica o export estável do catálogo como asset de release para o servidor privado baixar via RADAR_EXPORT_URL). Os +2 são unpinned-uses @vN: actions/checkout@v7 + actions/setup-node@v7 — a MESMA convenção deliberada de todos os workflows (ver _scanner_harden_workflows_2026_06_16); fixar por SHA só este violaria a convenção. O findings artipacked do checkout foi CORRIGIDO com `persist-credentials: false` (o job publica via GH_TOKEN em `gh release`, não usa a credencial do checkout). Nenhuma classe nova de template-injection / cache-poisoning / dangerous-triggers. Medido local com zizmor 1.25.2 via `node scripts/check/check-workflows.mjs --ratchet` = 191; +1 do delta conhecido do runner (ver _rebaseline_2026_07_28_ci_runner_delta: o runner enxerga 1 unpinned-uses @vN a mais que o devbox no mesmo commit; a baseline segue o runner) => 192.",
"_rebaseline_2026_07_20_aliasresolver_hook_split_7808": "175 -> 176 (+1). Companion to PR #7808 (CodeQL js/incomplete-url-substring-sanitization fix in bin/aliasResolver.mjs). The +1 is NOT caused by this PR's code changes (bin/* is not a workflow file) — it is a pre-existing drift that surfaced because the ratchet gate runs on this PR's CI: the zizmor scanner version on the GitHub runner gained a new rule (or extended an existing one) since the v3.8.49 baseline was seeded on 2026-07-17. Breakdown: the new finding is an unpinned-uses @vN class item on one of the existing workflows (same deliberate convention as _scanner_harden_workflows_2026_06_16 — @vN is intentional, SHA-pinning only this one would violate the convention). No new template-injection/artipacked/cache-poisoning/dangerous-triggers classes introduced. Measured by the Quality Gates (Extended) job on run 29713001401 = 176, baseline was 175. Note: by the time this landed on release/v3.8.49, the baseline was already at 176 via _rebaseline_2026_07_17_combo_recovery_hints — this entry is kept as historical record; no further bump applied.",
"_rebaseline_2026_07_17_v3849_release": "169 -> 175 (+6). Cycle workflow drift (v3.8.48/v3.8.49): npm-publish.yml (new, WS1.3 #7092), electron-release.yml, nightly-compat.yml, nightly-release-green.yml, CI restructures (#7501 full-history base fetch, #7355 main-green, #7202 merge-queue gates, Trunk/Codecov). Breakdown vs v3.8.47: +3 unpinned-uses (@vN convention, deliberate per _scanner_harden_workflows_2026_06_16), +2 cache-poisoning (artifact upload/cache in the OWN electron-release/npm-publish RELEASE workflows -- operator-controlled, not fork-PR exploitable), +1 excessive-permissions (nightly-compat.yml permissions:issues). No new template-injection/artipacked/dangerous-triggers. Measured with zizmor 1.25.2 via `node scripts/check/check-workflows.mjs --ratchet` = 175 on da3a0be69.",
@@ -180,8 +180,7 @@
"_rebaseline_2026_06_23_v3834_release": "152 -> 155 (+3). The 3 new unpinned-uses are in .github/workflows/nightly-release-green.yml (added by #4622 this cycle): actions/checkout@v7, actions/setup-node@v6, actions/upload-artifact@v4 — the SAME deliberate @vN convention as ci.yml's own checkout@v7/setup-node@v6 and every other workflow (see _scanner_harden_workflows_2026_06_16 + _zizmor_rebaseline_2026_06_20_ci_build_artifact_reuse). SHA-pinning only this workflow would violate the convention. The workflow-lint ratchet does NOT run on PR->release fast-gates, so it surfaced only on the release PR; measured locally via `npm run check:workflows -- --ratchet` = 155. No new template-injection/artipacked/cache-poisoning.",
"_rebaseline_2026_07_13_v3847_release_preflight": "159 -> 169 (+10). Findings from cycle-merged workflow changes: #6716 (PR gate restructure), #6781 (unit fast-path shard 2->4), #6788 (TIA tsx loader split), #6881 (electron-updater latest.yml manifests in release assets) — same deliberate @vN unpinned-uses convention as prior rebaselines; no new template-injection/artipacked/cache-poisoning classes. Measured via `npm run check:workflows -- --ratchet` = 169 on the v3.8.47 release pre-flight.",
"_rebaseline_2026_07_28_v3849_release_preflight": "176 -> 189 (+13). Pre-flight de fechamento da v3.8.49 (934 commits no ciclo). Deriva de workflow: 1 workflow novo (build-rinseaid-image.yml) mais os bumps de action do Dependabot ao longo do ciclo — todos da MESMA classe unpinned-uses @vN, convenção deliberada do repo (ver _scanner_harden_workflows_2026_06_16); fixar por SHA só estes violaria a convenção. Nenhuma classe nova de template-injection / artipacked / cache-poisoning / dangerous-triggers. Nesta mesma passada foram CORRIGIDAS 3 diretivas shellcheck malformadas (SC1125: `# shellcheck disable=SC2086 — texto`, em que o travessão invalida o par key=value) em ci.yml e nightly-release-green.yml. Medido com zizmor 1.25.2 via `npm run check:workflows -- --ratchet` = 189.",
"_rebaseline_2026_07_28_ci_runner_delta": "189 -> 190 (+1). Medido 189 no devbox e 190 no runner do GitHub no MESMO commit (run 30396592013, job Quality Gates (Extended)) — mesma classe já registrada em _rebaseline_2026_07_20_aliasresolver_hook_split_7808: a versão do zizmor no runner enxerga uma finding a mais que a local, sempre da classe unpinned-uses @vN. O valor do runner é o que o gate compara, então a baseline segue o runner.",
"_rebaseline_2026_08_28_npm_publish_hosted_stage_job": "192 -> 194 (+2). Job novo `stage-npm` em .github/workflows/npm-publish.yml: o npm RECUSA `--provenance` vindo de runner self-hosted (422 \"Unsupported GitHub Actions runner environment\"), e o job `publish` nao pode migrar para runner hospedado porque 16 GB nao bastam para o fallback next-build do build:cli (documentado no proprio runs-on). A separacao foi a unica saida que preserva a atestacao SLSA que a 3.8.49 ja tem. Os +2 sao da MESMA convencao deliberada de todos os workflows (ver _scanner_harden_workflows_2026_06_16): unpinned-uses @vN em actions/download-artifact@v8 + actions/setup-node@v7, mais o cache-poisoning que o proprio setup-node@v7 ja gera nos outros 2 jobs deste MESMO arquivo (linhas 85 e 463) e que ja esta na baseline. Fixar por SHA so este job violaria a convencao. Nenhuma classe nova: zero template-injection / artipacked / dangerous-triggers / excessive-permissions — o job declara apenas contents:read + id-token:write, que e o minimo para a proveniencia. Medido pelo job Quality Gates (Extended) no run 33162... da PR #11877 = 194."
"_rebaseline_2026_07_28_ci_runner_delta": "189 -> 190 (+1). Medido 189 no devbox e 190 no runner do GitHub no MESMO commit (run 30396592013, job Quality Gates (Extended)) — mesma classe já registrada em _rebaseline_2026_07_20_aliasresolver_hook_split_7808: a versão do zizmor no runner enxerga uma finding a mais que a local, sempre da classe unpinned-uses @vN. O valor do runner é o que o gate compara, então a baseline segue o runner."
},
"vulnCount": {
"value": 22,

View File

@@ -226,22 +226,16 @@ Three build args control what the `builder` stage costs. They are build-time onl
| --------------------------- | ------- | ----------------------------------------------------------------------------------- |
| `OMNIROUTE_USE_TURBOPACK` | `1` | `0` builds with webpack instead. Lower peak memory, slower. |
| `OMNIROUTE_BUILD_MEMORY_MB` | `6144` | V8 heap ceiling (`--max-old-space-size`) for the spawned `next build`. |
| `OMNIROUTE_BUILD_WORKERS` | `2` | Feeds `CIRCLE_NODE_TOTAL`; Next derives `workers = N - 1` for page-data collection. |
| `OMNIROUTE_BUILD_WORKERS` | `3` | Feeds `CIRCLE_NODE_TOTAL`; Next derives `workers = N - 1` for page-data collection. |
`OMNIROUTE_BUILD_WORKERS` is the one to raise on a big builder and the one to
suspect when a constrained build dies **after** `✓ Compiled successfully`. Each
page-data worker is its own process, and so is the parent `next build` itself;
a live VPS reproduction (issue #7518) measured each process's peak RSS at
~4.5 GB independent of the `NODE_OPTIONS` heap flag (Turbopack compiles in
native/Rust memory outside the V8 heap). The default of `2` (→ 1 worker, 2
processes total) is sized for the 16 GB / 4 vCPU GitHub-hosted runners the
publish pipeline uses. At `8` (→ 7 workers) that runner ran out of memory and
buildkit failed the step with `ResourceExhausted: ... cannot allocate memory`;
`3` (→ 2 workers) still didn't fit once the per-process RSS was measured
directly instead of inferred. `tests/unit/docker-build-memory-budget.test.ts`
does the arithmetic against the measured figure and fails if either knob
outgrows the runner. Both images (node and Bun) share these defaults; the Bun
image's are set in `Dockerfile.bun` (Turbopack on Bun 1.4+, `#11719`).
page-data worker is its own process and inherits `NODE_OPTIONS`, so the heap
ceiling is per process, not per build: the default of `3` (→ 2 workers) is sized
for the 16 GB / 4 vCPU GitHub-hosted runners the publish pipeline uses. At `8`
(→ 7 workers) that runner ran out of memory and buildkit failed the step with
`ResourceExhausted: ... cannot allocate memory`. `tests/unit/docker-build-memory-budget.test.ts`
does the arithmetic and fails if either knob outgrows the runner.
Turbopack compiles in native Rust memory that lives **outside** the V8 heap, so
`OMNIROUTE_BUILD_MEMORY_MB` does not bound it. On a host with a memory ceiling the

View File

@@ -1,12 +1,12 @@
---
title: "Release Checklist"
version: 3.8.51
lastUpdated: 2026-08-28
version: 3.8.40
lastUpdated: 2026-06-28
---
# Release Checklist
> **Last updated:** 2026-08-28 — v3.8.51
> **Last updated:** 2026-06-28 — v3.8.40
> Streamlined release flow that leverages Claude Code skills for automation.
>
> **Keep the queue/branch green between releases:** see [RELEASE_GREEN.md](./RELEASE_GREEN.md)
@@ -37,21 +37,7 @@ npm run test:e2e # optional but recommended
/capture-release-evidences-cc
```
## npm Trusted Publishing (default since v3.8.51) — staged on request, direct as fallback
`npm-publish.yml` publishes through **npm Trusted Publishing (OIDC)** by default: the
`stage-npm` job (github-hosted) exchanges GitHub's id-token for a short-lived npm
credential for that run — no long-lived npm token in the repository secrets, no 2FA prompt, provenance attached.
That is the bypass npm sanctions now that tokens which skip 2FA are being retired;
it restores the fully automatic flow the project had up to v3.8.48 while keeping the
WS1.3 guarantee (a leaked token cannot publish alone — there is no token).
**One-time setup (owner):** npmjs.com → package `omniroute` → Settings → *Trusted
Publisher* → GitHub: owner `diegosouzapw`, repo `OmniRoute`, workflow `npm-publish.yml`
(environment: none). Until that exists, the automatic step fails with `ENEEDAUTH`:
re-dispatch with `publish_mode=staged` (below) or `direct`.
### Staged publishing (on request — `publish_mode=staged`)
## npm Staged Publishing (default since v3.8.49 — WS1.3/D2)
The npm-publish workflow no longer publishes directly: it boots the packed tarball
(`check:pack-boot`) and then runs `npm stage publish` — the exact bytes are parked on

View File

@@ -4,66 +4,32 @@ title: Self-Hosted Runner Box Operations
# Self-Hosted Runner Box Operations (.113 pool)
The self-hosted pool (`self-hosted, omni-release` on all eight runners; `omni-build` on two) runs on the **.113** box.
Measured 2026-08-28 (v3.8.50 postmortem, Parte III):
The self-hosted pool (`self-hosted, omni-release` labels) runs on the 16 GB box at
`192.168.0.113`. Two failure modes recurred on release days and were, until v3.8.49,
manual discipline; the **janitor script codifies them** (WS3.3 of the quality plan):
| resource | value | what it means for scheduling |
| --------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| RAM / CPU | **31 GB / 32 cores** (was 16 GB when this doc was first written) | one `next-build` peaks at **~14 GB** → 2 concurrent heavy builds saturate the box, 3 take it down (2026-08-28 06:42Z: load 56, two jobs lost) |
| swap | 15 GB | it swapped its way through the v3.8.50 publish; pressure shows in `/proc/pressure/memory` |
| `/tmp` | **12 GB tmpfs = RAM** | anything parked there is memory; leftovers are swept after 3 h |
| disk | 188 GB | `_work` checkouts of 8 runners reach ~70 GB with no cap |
| runners | **10 listeners**: 8 OmniRoute + OmniHeuris + OmniMind | all share the memory above |
1. **Orphaned temp/work dirs** filling the disk → disk-full SQLite errors mid-job.
2. **>4 concurrent runners** → OOM-killed jobs (8-wide killed jobs twice on the
v3.8.47 release day; 4-wide is the proven ceiling).
## Install the janitor (one-time, on the box)
```bash
scp scripts/ops/runner-janitor.sh root@192.168.0.113:/opt/omniroute-ops/runner-janitor.sh
ssh root@192.168.0.113 'chmod +x /opt/omniroute-ops/runner-janitor.sh; apt-get install -y lsof'
# cron (root): every 30 min, log to /var/log/runner-janitor.log
*/30 * * * * MAX_ACTIVE_RUNNERS=8 /opt/omniroute-ops/runner-janitor.sh >> /var/log/runner-janitor.log 2>&1
sudo mkdir -p /opt/omniroute-ops
sudo cp scripts/ops/runner-janitor.sh /opt/omniroute-ops/
sudo chmod +x /opt/omniroute-ops/runner-janitor.sh
( sudo crontab -l 2>/dev/null; echo '*/30 * * * * /opt/omniroute-ops/runner-janitor.sh >> /var/log/runner-janitor.log 2>&1' ) | sudo crontab -
```
`lsof` is required: the janitor proves a path is idle with one snapshot of open
files before removing it, and without the tool it removes nothing and says so
(exit 1). Try any change with `--dry-run` first — it prints exactly what it would
do and touches nothing.
What it does every run: sweeps our own leftovers (`runner-*`, `omniroute-*`,
`next-build*`, `e2e-build.tar.gz`) after **3 h on tmpfs** and 24 h on disk
`_work/_temp`; kills a `next-build` older than 75 min (no job runs that long — on
2026-08-27 one ran 70 min after GitHub had declared its job lost); prunes 48 h-old
checkouts of runners whose unit is **stopped**; alerts on disk ≥ 85 %, memory PSI
`full/avg60` ≥ 10 %, and more listeners than `MAX_ACTIVE_RUNNERS` (with an
omniroute/other breakdown). Exit 1 = attention needed; read the log.
## Runner units: KillMode
The runner's default `KillMode=process` leaves `Runner.Worker → npm → next-build`
alive when a unit is stopped or restarted — an orphan build keeps eating RAM and
CPU with no job attached. Every OmniRoute unit carries a drop-in
(`/etc/systemd/system/actions.runner.diegosouzapw-OmniRoute.<name>.service.d/10-killmode.conf`)
with `KillMode=mixed`: SIGTERM to the listener first, SIGKILL to the whole cgroup at
`TimeoutStop`. It takes effect on the unit's next restart — restart **one runner at
a time, only when idle**, with the idle check and the restart in the same command.
What it does every 30min: sweeps runner temp leftovers older than 24h, alerts at
≥85% root-disk usage, and alerts when more than the runner ceiling (default 4, tunable
via the script's own environment) of `Runner.Listener` processes are up. Alerts land in `/var/log/runner-janitor.log`
with a non-zero exit (grep for `⚠`).
## Operating rules
- **Heavy-build ceiling: 2 at a time — enforced by label.** Every job that runs a
`next build` (`ci.yml` `build`, `npm-publish.yml` `publish`, both `nightly-release-green`
validations) targets `[self-hosted, omni-build]`, and only **two** runners carry that
label (`omniroute-113-5`, `omniroute-113-6`, added through the runners API — no
re-registration). The other six keep `omni-release` and take nothing heavy; GitHub
queues a third build instead of the kernel killing one. Pair with the `heavy-build-*`
concurrency lanes in `ci.yml`. To add capacity, label another runner — never raise
the count past what 31 GB holds (one next-build ≈ 1416 GB).
- **Never clean `/tmp` or `_work` by hand while any runner is busy.** A
check-then-delete with a gap between the two is how a live Build job lost its
`_work` on 2026-08-27. The janitor does the check and the removal in one step;
let it.
- Stopping a runner mid-job cancels the job (observed live): `systemctl stop` only
when its listener has no `Runner.Worker` child — and do it in one command.
- Workflows must not park artefacts in `/tmp` (it is RAM). Download to
`$RUNNER_TEMP` (on disk, per runner) — the 1.3 GB `next-build` artefact took 2732
minutes to land on the tmpfs and 2 minutes to upload from disk.
- **Ceiling: 4 runners** on the 16 GB box. Runners 58 stay STOPPED except for
explicit off-peak experiments — never during a release window.
- Stopping a runner mid-job cancels the job (observed live): `systemctl stop`
only when its runner is idle (`Runner.Listener` without a `Runner.Worker` child).
- The `.15` VPS is homologation-only — never runs CI runners.

View File

@@ -297,45 +297,6 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/@electron/windows-sign": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@electron/windows-sign/-/windows-sign-1.2.2.tgz",
"integrity": "sha512-dfZeox66AvdPtb2lD8OsIIQh12Tp0GNCRUDfBHIKGpbmopZto2/A8nSpYYLoedPIHpqkeblZ/k8OV0Gy7PYuyQ==",
"dev": true,
"license": "BSD-2-Clause",
"optional": true,
"peer": true,
"dependencies": {
"cross-dirname": "^0.1.0",
"debug": "^4.3.4",
"fs-extra": "^11.1.1",
"minimist": "^1.2.8",
"postject": "^1.0.0-alpha.6"
},
"bin": {
"electron-windows-sign": "bin/electron-windows-sign.js"
},
"engines": {
"node": ">=14.14"
}
},
"node_modules/@electron/windows-sign/node_modules/fs-extra": {
"version": "11.4.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz",
"integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
},
"engines": {
"node": ">=14.14"
}
},
"node_modules/@isaacs/fs-minipass": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
@@ -1130,15 +1091,6 @@
"dev": true,
"license": "MIT"
},
"node_modules/cross-dirname": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/cross-dirname/-/cross-dirname-0.1.0.tgz",
"integrity": "sha512-+R08/oI0nl3vfPcqftZRpytksBXDzOUveBq/NBVx0sUp1axwzPQrKinNx5yd5sxPu8j1wIy8AfnVQ+5eFdha6Q==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true
},
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -1459,19 +1411,6 @@
"node": ">=14.0.0"
}
},
"node_modules/electron-builder-squirrel-windows": {
"version": "26.15.3",
"resolved": "https://registry.npmjs.org/electron-builder-squirrel-windows/-/electron-builder-squirrel-windows-26.15.3.tgz",
"integrity": "sha512-Jc19XPV9y9+2bAdZPkXuVNGNIEFBq9poHC61l8Kv6FdK7DRG3+Ic0rerC0DXOaeHNz8yW0fg/JnF8GQROOF5MA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"app-builder-lib": "26.15.3",
"builder-util": "26.15.3",
"electron-winstaller": "5.4.0"
}
},
"node_modules/electron-publish": {
"version": "26.15.3",
"resolved": "https://registry.npmjs.org/electron-publish/-/electron-publish-26.15.3.tgz",
@@ -1506,66 +1445,6 @@
"tiny-typed-emitter": "^2.1.0"
}
},
"node_modules/electron-winstaller": {
"version": "5.4.0",
"resolved": "https://registry.npmjs.org/electron-winstaller/-/electron-winstaller-5.4.0.tgz",
"integrity": "sha512-bO3y10YikuUwUuDUQRM4KfwNkKhnpVO7IPdbsrejwN9/AABJzzTQ4GeHwyzNSrVO+tEH3/Np255a3sVZpZDjvg==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@electron/asar": "^3.2.1",
"debug": "^4.1.1",
"fs-extra": "^7.0.1",
"lodash": "^4.17.21",
"temp": "^0.9.0"
},
"engines": {
"node": ">=8.0.0"
},
"optionalDependencies": {
"@electron/windows-sign": "^1.1.2"
}
},
"node_modules/electron-winstaller/node_modules/fs-extra": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz",
"integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"graceful-fs": "^4.1.2",
"jsonfile": "^4.0.0",
"universalify": "^0.1.0"
},
"engines": {
"node": ">=6 <7 || >=8"
}
},
"node_modules/electron-winstaller/node_modules/jsonfile": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
"integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==",
"dev": true,
"license": "MIT",
"peer": true,
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
},
"node_modules/electron-winstaller/node_modules/universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 4.0.0"
}
},
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -2480,20 +2359,6 @@
"node": ">= 18"
}
},
"node_modules/mkdirp": {
"version": "0.5.6",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
"integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"minimist": "^1.2.6"
},
"bin": {
"mkdirp": "bin/cmd.js"
}
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -2757,36 +2622,6 @@
"node": ">=18"
}
},
"node_modules/postject": {
"version": "1.0.0-alpha.6",
"resolved": "https://registry.npmjs.org/postject/-/postject-1.0.0-alpha.6.tgz",
"integrity": "sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"dependencies": {
"commander": "^9.4.0"
},
"bin": {
"postject": "dist/cli.js"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/postject/node_modules/commander": {
"version": "9.5.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz",
"integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": "^12.20.0 || >=14"
}
},
"node_modules/proc-log": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz",
@@ -2981,21 +2816,6 @@
"node": ">= 4"
}
},
"node_modules/rimraf": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
"integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
"deprecated": "Rimraf versions prior to v4 are no longer supported",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/roarr": {
"version": "2.15.4",
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
@@ -3251,21 +3071,6 @@
"node": ">=18"
}
},
"node_modules/temp": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/temp/-/temp-0.9.4.tgz",
"integrity": "sha512-yYrrsWnrXMcdsnu/7YMYAofM1ktpL5By7vZhf15CrXijWWrEYZks5AXBudalfSWJLlnen/QUJUB5aoB0kqZUGA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"mkdirp": "^0.5.1",
"rimraf": "~2.6.2"
},
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/temp-file": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/temp-file/-/temp-file-3.4.0.tgz",

View File

@@ -52,8 +52,7 @@
"!**/*.test.js",
"!**/*.test.mjs",
"!**/*.spec.ts",
"!**/*.spec.tsx",
"!**/*.nft.json"
"!**/*.spec.tsx"
],
"workspaces": [
"open-sse",

View File

@@ -131,15 +131,9 @@ function runNextBuild() {
}
export function resolveNextBuildBundlerFlag(baseEnv = process.env) {
// Turbopack is the default; OMNIROUTE_USE_TURBOPACK=0 is the documented escape hatch
// to webpack (Windows, native-binding trouble, RAM-constrained machines — #6409, and
// docs/reference/ENVIRONMENT.md). The choice is env-only ON PURPOSE: the variable is
// the operator's control and CI sets it explicitly, so sniffing the runtime here would
// silently override an operator who asked for Turbopack. Bun 1.4+ supports Turbopack's
// V8 worker bindings (#11471), so the historical `process.versions.bun` → `--webpack`
// hardcode is gone; the `OMNIROUTE_USE_TURBOPACK=0` fallback remains for Bun < 1.4
// images built with the webpack path.
if (baseEnv.OMNIROUTE_USE_TURBOPACK === "0") {
// Turbopack is the default on Node.js; on Bun or when explicitly disabled (=0),
// use Webpack (--webpack) to avoid Turbopack V8 internal worker API mismatches.
if (process.versions.bun || baseEnv.OMNIROUTE_USE_TURBOPACK === "0") {
return "--webpack";
}
return "--turbopack";

View File

@@ -42,7 +42,6 @@ import { execFileSync, spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { findProvenanceOnSelfHosted, formatProvenanceFinding } from "./lib/provenanceRunner.mjs";
const ROOT = process.cwd();
const WORKFLOWS_DIR = path.join(ROOT, ".github", "workflows");
@@ -276,23 +275,6 @@ export function runZizmor(workflowsDir) {
// Main
// ---------------------------------------------------------------------------
/**
* Hard rule (not a lint count): `--provenance` inside a job that runs on a
* self-hosted runner. npm answers 422 at the registry, and in v3.8.50 that
* answer only came after the tag, the GitHub Release and the Docker images were
* already out. Blocks under --strict AND --ratchet (the CI mode); plain mode
* reports it like everything else.
* @param {string[]} files absolute workflow paths
*/
export function runProvenanceRunnerCheck(files) {
const findings = [];
for (const file of files) {
const text = fs.readFileSync(file, "utf8");
findings.push(...findProvenanceOnSelfHosted(text, path.relative(ROOT, file)));
}
return findings;
}
function main() {
const hasActionlint = isBinaryAvailable("actionlint");
const hasZizmor = isBinaryAvailable("zizmor");
@@ -368,16 +350,6 @@ function main() {
}
}
const provenanceFindings = runProvenanceRunnerCheck(workflowFiles);
if (provenanceFindings.length > 0) {
console.error(
`[check-workflows] provenance×self-hosted: ${provenanceFindings.length} finding(s) — HARD RULE:`
);
provenanceFindings.forEach((f) => console.error(` ${formatProvenanceFinding(f)}`));
} else if (!QUIET) {
console.log("[check-workflows] provenance×self-hosted: OK (0 findings)");
}
const total = actionlintCount + zizmorCount;
process.stdout.write(`workflowFindings=${total}\n`);
process.stdout.write(`actionlintFindings=${actionlintCount}\n`);
@@ -385,15 +357,6 @@ function main() {
// Read this line with the count above: a finding total is only reproducible against the
// version that produced it. See zizmorVersion().
process.stdout.write(`zizmorVersion=${hasZizmor ? zizmorVersion() : "absent"}\n`);
process.stdout.write(`provenanceRunnerFindings=${provenanceFindings.length}\n`);
if ((STRICT || RATCHET) && provenanceFindings.length > 0) {
console.error(
`\n[check-workflows] FAIL — ${provenanceFindings.length} job(s) publish with --provenance from a self-hosted runner.\n` +
" npm rejects that with 422 at the registry. Move the upload step to a github-hosted job\n" +
" (see .github/workflows/npm-publish.yml `stage-npm` for the pattern)."
);
process.exit(1);
}
if (STRICT && total > 0) {
console.error(`\n[check-workflows] FAIL — ${total} workflow finding(s) total (--strict mode).`);

View File

@@ -1,90 +0,0 @@
/**
* scripts/check/lib/configExpiry.mjs
*
* Finds dated validity fields in JSON config packs so a test can fail BEFORE
* they lapse. Origin: config/alibaba-free-tier-allowlist.json carried
* `"validUntil": "2026-08-27"`; on 2026-08-28 the loader started (correctly)
* rejecting the pack and a test that asserted "the shipped pack loads" turned
* every PR and main red with no commit involved (#11866). A time bomb, not a
* regression — and the only kind of defect a diff review can never catch.
*
* Pure helpers; the repo-wide assertion lives in
* tests/unit/config-expiry-time-bomb.test.ts.
*/
import fs from "node:fs";
import path from "node:path";
export const EXPIRY_KEY =
/^(validUntil|valid_until|validTo|valid_to|expiresAt|expires_at|expiry|expires)$/;
const DAY_MS = 86_400_000;
/**
* Walks a parsed JSON value and returns every string-valued expiry field.
* @returns {{ file: string, keyPath: string, raw: string, expiresAt: number|null }[]}
*/
export function collectExpiryFields(value, file, keyPath = []) {
const out = [];
if (Array.isArray(value)) {
value.forEach((v, i) => out.push(...collectExpiryFields(v, file, [...keyPath, String(i)])));
return out;
}
if (!value || typeof value !== "object") return out;
for (const [key, v] of Object.entries(value)) {
const kp = [...keyPath, key];
if (EXPIRY_KEY.test(key) && typeof v === "string") {
const ms = Date.parse(v);
out.push({ file, keyPath: kp.join("."), raw: v, expiresAt: Number.isFinite(ms) ? ms : null });
} else if (v && typeof v === "object") {
out.push(...collectExpiryFields(v, file, kp));
}
}
return out;
}
/** @returns {"expired"|"expiring"|"ok"|"unparseable"} */
export function classifyExpiry(field, nowMs, warnDays = 7) {
if (field.expiresAt === null) return "unparseable";
if (field.expiresAt < nowMs) return "expired";
if (field.expiresAt < nowMs + warnDays * DAY_MS) return "expiring";
return "ok";
}
/** All *.json under dir, recursively, skipping node_modules. Sorted for stable output. */
export function walkJsonFiles(dir) {
const out = [];
const stack = [dir];
while (stack.length > 0) {
const current = stack.pop();
let entries;
try {
entries = fs.readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const e of entries) {
const full = path.join(current, e.name);
if (e.isDirectory()) {
if (e.name !== "node_modules") stack.push(full);
} else if (e.isFile() && e.name.endsWith(".json")) {
out.push(full);
}
}
}
return out.sort();
}
/**
* Scans every JSON file under `dir`; `file` in the result is relative to `dir`
* with forward slashes, so allowlists can key on it portably.
*/
export function scanConfigExpiry(dir) {
return walkJsonFiles(dir).flatMap((f) => {
let parsed;
try {
parsed = JSON.parse(fs.readFileSync(f, "utf8"));
} catch {
return []; // not this scanner's job to validate JSON
}
return collectExpiryFields(parsed, path.relative(dir, f).split(path.sep).join("/"));
});
}

View File

@@ -1,83 +0,0 @@
/**
* scripts/check/lib/provenanceRunner.mjs
*
* npm refuses `--provenance` from a self-hosted runner:
*
* 422 Unprocessable Entity - Error verifying sigstore provenance bundle:
* Unsupported GitHub Actions runner environment: "self-hosted".
* Only "github-hosted" runners are supported when publishing with provenance.
*
* v3.8.50 hit this at the very end of a 76-minute publish job — after the tag,
* the GitHub Release and the Docker images were already public — because
* `USE_VPS_RUNNER` had been turned on (2026-08-02) with no release in between to
* surface it. The combination is greppable, so it must fail in CI the moment a
* workflow introduces it, not four weeks later at the registry.
*
* Pure: takes workflow YAML text, returns the offending (job, step) pairs.
*/
import { load as yamlLoad } from "js-yaml";
const SELF_HOSTED = /\bself-hosted\b/;
const EXPRESSION = /\$\{\{/;
// Lookahead, not \b: `--provenance-file=…` is a different flag (a pre-built
// bundle) and must not match — a word boundary sits between "e" and "-".
const PROVENANCE = /(^|\s)--provenance(?=\s|=|$)/m;
/**
* Classifies a job's `runs-on` value.
* @returns {"self-hosted"|"hosted"|"unknown"}
* "unknown" = an expression with no literal `self-hosted` in it (e.g.
* `${{ matrix.os }}`); the check does not guess, it skips.
*/
export function classifyRunsOn(runsOn) {
if (runsOn == null) return "unknown";
if (typeof runsOn === "string") {
if (SELF_HOSTED.test(runsOn)) return "self-hosted";
return EXPRESSION.test(runsOn) ? "unknown" : "hosted";
}
if (Array.isArray(runsOn)) {
return runsOn.some((v) => typeof v === "string" && SELF_HOSTED.test(v))
? "self-hosted"
: "hosted";
}
if (typeof runsOn === "object") {
// { group: ..., labels: ... } form
const labels = runsOn.labels;
return classifyRunsOn(Array.isArray(labels) ? labels : labels == null ? "" : String(labels));
}
return "unknown";
}
/**
* @param {string} yamlText
* @param {string} fileName used only for reporting
* @returns {{ file: string, job: string, step: string }[]}
*/
export function findProvenanceOnSelfHosted(yamlText, fileName = "<workflow>") {
let doc;
try {
doc = yamlLoad(yamlText);
} catch {
// actionlint owns syntax; an unparseable file is not this rule's finding.
return [];
}
const jobs =
doc && typeof doc === "object" && doc.jobs && typeof doc.jobs === "object" ? doc.jobs : {};
const findings = [];
for (const [jobName, job] of Object.entries(jobs)) {
if (!job || typeof job !== "object") continue;
if (classifyRunsOn(job["runs-on"]) !== "self-hosted") continue;
const steps = Array.isArray(job.steps) ? job.steps : [];
steps.forEach((step, i) => {
if (step && typeof step.run === "string" && PROVENANCE.test(step.run)) {
findings.push({ file: fileName, job: jobName, step: step.name || `#${i + 1}` });
}
});
}
return findings;
}
/** Human-readable line per finding, used by the CLI. */
export function formatProvenanceFinding(f) {
return `${f.file}: job "${f.job}", step "${f.step}" runs \`--provenance\` on a self-hosted runner — npm rejects that (422). Move the upload to a github-hosted job.`;
}

View File

@@ -123,48 +123,6 @@ function discoverPackagedExecutable() {
throw new Error(`Packaged Electron smoke check does not support ${platform()}.`);
}
/**
* The packaged app opens SQLite lazily: `/login` (the readiness URL) never touches the
* database, so a smoke that only waits for readiness sees no `[DB]` line at all. After
* readiness the smoke requests a DB-backed endpoint and waits for evidence that the
* database opened. The primary open path does NOT print "[DB] Driver: ..." (only the
* recovery path and the sql.js fallback do), so the evidence is any `[DB]`/`[Migration]`
* startup line — and the #7592 guard below rejects the fallback's own line explicitly.
*/
export const DB_TOUCH_PATH = "/api/monitoring/health";
export const DB_OPEN_EVIDENCE_PATTERN =
/\[DB\] (Driver: |SQLite database ready|Added [^\n]* column|Changing cache_size|cache_size changed)|\[Migration\] (Applied|Pre-migration backup)/;
export async function waitForDatabaseOpen(getLogs, { timeoutMs = 15_000, pollMs = 250 } = {}) {
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
const logs = getLogs();
assertNoFatalLogs(logs);
if (DB_OPEN_EVIDENCE_PATTERN.test(logs)) return logs;
await sleep(pollMs);
}
throw new Error(
`Packaged Electron app logged no [DB]/[Migration] startup line within ${timeoutMs}ms of ` +
`touching ${DB_TOUCH_PATH} — the database never opened, so the SQLite driver cannot be verified.`
);
}
async function openDatabaseForSmoke({ logs, smokeUrl }) {
const touchUrl = new URL(DB_TOUCH_PATH, smokeUrl).toString();
try {
const response = await fetchWithTimeout(touchUrl, 5_000);
console.log(
`[electron-smoke] touched ${touchUrl} (HTTP ${response.status}) to open the database`
);
} catch (error) {
console.log(
`[electron-smoke] touching ${touchUrl} failed (${error instanceof Error ? error.message : String(error)}) — waiting for the database anyway`
);
}
await waitForDatabaseOpen(() => logs.value);
console.log("[electron-smoke] database opened");
}
async function fetchWithTimeout(url, timeoutMs) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -492,13 +450,8 @@ export function assertNativeDriverSelected(logs) {
);
}
// The primary open path prints no "[DB] Driver: ..." line at all (only the recovery path and
// the sql.js fallback do), so a database that demonstrably opened WITHOUT the fallback's own
// line is the native driver — that is exactly what #7592 guards.
if (DB_OPEN_EVIDENCE_PATTERN.test(logs)) return;
throw new Error(
"Packaged Electron app logs show no database activity at all — cannot confirm which SQLite " +
"Packaged Electron app logs contain no '[DB] Driver: ...' line — cannot confirm which SQLite " +
"driver loaded."
);
}
@@ -553,14 +506,7 @@ async function waitForReady({ logs, smokeUrl, timeoutMs, settleMs, exitState })
* by the single-launch path and the cold-restart (two-launch) path so both
* exercise identical spawn/readiness/shutdown behavior.
*/
async function launchAndCollectLogs({
appExecutable,
smokeUrl,
dataDir,
timeoutMs,
settleMs,
streamLogs,
}) {
async function launchAndCollectLogs({ appExecutable, smokeUrl, dataDir, timeoutMs, settleMs, streamLogs }) {
const smokeEnv = buildSmokeEnv({ dataDir });
await assertPortIsFree(smokeUrl);
await ensureSmokeEnvDirs(smokeEnv, dataDir);
@@ -592,8 +538,6 @@ async function launchAndCollectLogs({
try {
await waitForReady({ logs, smokeUrl, timeoutMs, settleMs, exitState });
// Outside waitForReady on purpose: a missing database is a verdict, not a readiness retry.
await openDatabaseForSmoke({ logs, smokeUrl });
return logs.value;
} catch (error) {
if (!streamLogs) {
@@ -624,14 +568,7 @@ async function main() {
!process.env.ELECTRON_SMOKE_DATA_DIR && process.env.ELECTRON_SMOKE_KEEP_DATA !== "1";
try {
await launchAndCollectLogs({
appExecutable,
smokeUrl,
dataDir,
timeoutMs,
settleMs,
streamLogs,
});
await launchAndCollectLogs({ appExecutable, smokeUrl, dataDir, timeoutMs, settleMs, streamLogs });
if (!coldRestart) return;

View File

@@ -1,172 +1,53 @@
#!/usr/bin/env bash
# runner-janitor — self-hosted runner box hygiene for the .113 pool.
# runner-janitor — self-hosted runner box hygiene (WS3.3, v3.8.49 quality plan).
#
# Runs from cron every 30 min (see docs/ops/RUNNER_BOX.md). It ACTS on what it
# can prove is safe and ALERTS on what needs an operator decision. Reads of
# "is this in use?" and the removal happen in the same command, never in two
# passes: a check-then-delete with a gap is how a live Build job lost its _work
# on 2026-08-27.
# The .113 runner box has recurring failure modes that until now were manual
# discipline: orphaned tmpfs/work dirs filling the disk, and >4 concurrent
# runners OOM-killing jobs (16 GB box; incidents on the v3.8.47 release day).
# Install via cron on the box (see docs/ops/RUNNER_BOX.md):
# */30 * * * * /opt/omniroute-ops/runner-janitor.sh >> /var/log/runner-janitor.log 2>&1
#
# Measured box (2026-08-28): 31 GB RAM, 32 cores, 15 GB swap, /tmp = 12 GB
# tmpfs (RAM!), 188 GB disk. A single `next-build` peaks at ~14 GB, so two
# concurrent heavy builds saturate the box and three take it down (06:42Z that
# day: load 56, two jobs lost). The v3.8.50 postmortem (Parte III) has the numbers.
#
# What it does, in order:
# 1) sweep stale artefacts our tooling leaves behind — tmpfs bases after 3 h
# (they hold RAM), disk _work/_temp bases after 24 h; only names we create,
# only when no process has them open
# 2) kill zombie builds: a `next-build` older than ZOMBIE_BUILD_MAX_MIN has no
# job attached (a real Build step measures ~26 min). On 2026-08-27 one ran
# 70 minutes after GitHub had already declared its job lost, eating 3.6 GB
# and a full core set. KillMode=mixed on the units covers systemctl
# stop/restart; this covers the lost-connection path.
# 3) prune 48 h-old checkouts under _work of runners whose unit is INACTIVE
# (stopped runners cannot be mid-job; active ones are never touched)
# 4) alert: root disk >= DISK_ALERT_PCT, memory PSI full/avg60 >= threshold,
# Runner.Listener count above the ceiling (with a per-project breakdown —
# the box also hosts OmniHeuris and OmniMind runners)
#
# Usage: runner-janitor.sh [--dry-run] [--help]
# Exit codes: 0 healthy · 1 attention needed (printed to stdout for the log).
set -euo pipefail
DRY_RUN=0
for arg in "$@"; do
case "$arg" in
--dry-run) DRY_RUN=1 ;;
-h|--help)
sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'
exit 0 ;;
*) echo "unknown argument: $arg" >&2; exit 2 ;;
esac
done
MAX_ACTIVE_RUNNERS="${MAX_ACTIVE_RUNNERS:-8}"
MAX_ACTIVE_RUNNERS="${MAX_ACTIVE_RUNNERS:-4}"
DISK_ALERT_PCT="${DISK_ALERT_PCT:-85}"
TMPFS_MAX_AGE_HOURS="${TMPFS_MAX_AGE_HOURS:-3}"
WORK_TEMP_MAX_AGE_HOURS="${WORK_TEMP_MAX_AGE_HOURS:-24}"
WORK_CHECKOUT_MAX_AGE_HOURS="${WORK_CHECKOUT_MAX_AGE_HOURS:-48}"
ZOMBIE_BUILD_MAX_MIN="${ZOMBIE_BUILD_MAX_MIN:-75}"
ZOMBIE_BUILD_COMM="${ZOMBIE_BUILD_COMM:-next-build}"
PSI_FULL_AVG60_ALERT="${PSI_FULL_AVG60_ALERT:-10}"
# Overridable so the unit test can point everything at a fixture tree.
JANITOR_TMP_BASES="${JANITOR_TMP_BASES-/tmp}"
JANITOR_WORK_TEMP_BASES="${JANITOR_WORK_TEMP_BASES-/opt/actions-runner*/_work/_temp /home/*/actions-runner*/_work/_temp}"
JANITOR_RUNNER_DIRS="${JANITOR_RUNNER_DIRS-/opt/actions-runner*}"
JANITOR_PSI_FILE="${JANITOR_PSI_FILE:-/proc/pressure/memory}"
JANITOR_DF_PATH="${JANITOR_DF_PATH:-/}"
WORK_DIR_MAX_AGE_HOURS="${WORK_DIR_MAX_AGE_HOURS:-24}"
STATUS=0
say() { echo "[janitor] $*"; }
# "Is anything using this?" — ONE snapshot of every open path on the box
# (lsof -Fn), then a prefix match per candidate. `lsof +D <dir>` walks the whole
# tree instead and took minutes on a 5 GB leftover — unusable from cron. An
# absent lsof means "cannot prove idle": the sweep keeps the path and says so.
LSOF_BIN="${JANITOR_LSOF:-lsof}"
have_busy_tools() { command -v "$LSOF_BIN" >/dev/null 2>&1; }
SNAP=""
cleanup() { [ -n "$SNAP" ] && rm -f -- "$SNAP"; }
trap cleanup EXIT
# One lsof for the whole run (~13 s / 83k lines on the box), kept ONLY for the
# bases we sweep — 460 candidates grepping a re-printed 83k-line string was the
# slow part, not lsof itself.
snapshot_open_paths() {
have_busy_tools || return 0
SNAP=$(mktemp) || return 0
local prefixes="" b
for b in $JANITOR_TMP_BASES $JANITOR_WORK_TEMP_BASES; do [ -d "$b" ] && prefixes="$prefixes"$'\n'"$b/"; done
# -F n: one "n<path>" line per open file; -w: no warnings
"$LSOF_BIN" -w -Fn 2>/dev/null | sed -n 's/^n//p' | grep -F -f <(printf '%s' "$prefixes" | sed '/^$/d') > "$SNAP" 2>/dev/null || true
}
is_busy() {
local p="$1"
[ -n "$SNAP" ] && [ -s "$SNAP" ] || return 1
# exact path, or anything beneath it when it is a directory
grep -qxF -- "$p" "$SNAP" && return 0
[ -d "$p" ] && grep -qF -- "$p/" "$SNAP"
}
echo "[janitor] $(date -u +%FT%TZ) start"
# sweep <base> <max-age-minutes>: only names our tooling creates, never through
# a symlinked base, never across a filesystem, and remove+check in one step.
sweep() {
local base="$1" max_min="$2" p
[ -d "$base" ] || return 0
[ -L "$base" ] && { say "skip symlinked base: $base"; return 0; }
while IFS= read -r -d '' p; do
if ! have_busy_tools; then say "cannot prove idle (lsof missing — apt install lsof), kept: $p"; STATUS=1; continue; fi
if is_busy "$p"; then say "busy, kept: $p"; continue; fi
if [ "$DRY_RUN" -eq 1 ]; then say "would remove ($(( max_min / 60 ))h+): $p"; else rm -rf -- "$p" && say "removed ($(( max_min / 60 ))h+): $p"; fi
done < <(find -P "$base" -xdev -mindepth 1 -maxdepth 1 \
\( -name 'runner-*' -o -name 'omniroute-*' -o -name 'next-build*' -o -name 'e2e-build.tar.gz' \) \
! -type l -mmin "+$max_min" -print0 2>/dev/null || true)
}
say "$(date -u +%FT%TZ) start${DRY_RUN:+ (dry-run=$DRY_RUN)} busy-tools=$(have_busy_tools && echo ok || echo MISSING)"
# 1) stale artefacts — tmpfs is RAM, so it gets the short fuse
snapshot_open_paths
for base in $JANITOR_TMP_BASES; do sweep "$base" $(( TMPFS_MAX_AGE_HOURS * 60 )); done
for base in $JANITOR_WORK_TEMP_BASES; do sweep "$base" $(( WORK_TEMP_MAX_AGE_HOURS * 60 )); done
say "stale temp sweep done"
# 2) zombie builds
ZOMBIES=0
while read -r pid etimes comm; do
[ -n "${pid:-}" ] || continue
if [ "$etimes" -gt $(( ZOMBIE_BUILD_MAX_MIN * 60 )) ]; then
say "⚠ zombie build pid=$pid comm=$comm age=$(( etimes / 60 ))min > ${ZOMBIE_BUILD_MAX_MIN}min — no job runs this long"
if [ "$DRY_RUN" -eq 1 ]; then say "[dry-run] would: kill -TERM $pid (then -KILL)"; else
kill -TERM "$pid" 2>/dev/null || true; sleep 10
kill -0 "$pid" 2>/dev/null && { kill -KILL "$pid" 2>/dev/null || true; say " needed SIGKILL"; }
fi
ZOMBIES=$(( ZOMBIES + 1 )); STATUS=1
fi
done < <(ps -eo pid=,etimes=,comm= 2>/dev/null | awk -v c="$ZOMBIE_BUILD_COMM" '$3 ~ ("^" c) {print $1, $2, $3}' || true)
say "zombie builds: $ZOMBIES"
# 3) old checkouts of STOPPED runners
for d in $JANITOR_RUNNER_DIRS; do
[ -d "$d" ] && [ -f "$d/.runner" ] || continue
agent=$(grep -o '"agentName": *"[^"]*"' "$d/.runner" 2>/dev/null | sed 's/.*"\([^"]*\)"$/\1/')
[ -n "$agent" ] || continue
unit=$(systemctl list-units --plain --no-legend "actions.runner.*.${agent}.service" 2>/dev/null | awk 'NR==1{print $1}')
[ -n "$unit" ] || continue
if systemctl is-active --quiet "$unit"; then continue; fi
while IFS= read -r -d '' co; do
if [ "$DRY_RUN" -eq 1 ]; then say "would prune checkout of stopped runner $agent: $co"; else rm -rf -- "$co" && say "pruned checkout of stopped runner $agent: $co"; fi
done < <(find -P "$d/_work" -xdev -mindepth 2 -maxdepth 2 -type d -mmin "+$(( WORK_CHECKOUT_MAX_AGE_HOURS * 60 ))" -print0 2>/dev/null || true)
# 1) Sweep stale runner temp/work leftovers (>24h — no legitimate job runs that long).
# Hardened for a root cron on world-writable paths: never follow a symlinked base
# (a compromised runner could plant one), -P + -xdev so the sweep cannot traverse
# out of the filesystem, and patterns narrowed to names OUR tooling creates
# (no generic tmp* — unrelated system temp files are out of scope).
for base in /tmp /home/*/actions-runner*/_work/_temp; do
[ -d "$base" ] || continue
[ -L "$base" ] && { echo "[janitor] skip symlinked base: $base"; continue; }
find -P "$base" -xdev -maxdepth 1 \( -name 'runner-*' -o -name 'omniroute-*' \) \
! -type l -mmin +$((WORK_DIR_MAX_AGE_HOURS * 60)) -exec rm -rf {} + 2>/dev/null || true
done
echo "[janitor] stale temp sweep done"
# 4a) disk
USAGE=$(df --output=pcent "$JANITOR_DF_PATH" 2>/dev/null | tail -1 | tr -dc '0-9')
if [ "${USAGE:-0}" -ge "$DISK_ALERT_PCT" ]; then
say "⚠ ROOT DISK ${USAGE}% >= ${DISK_ALERT_PCT}% — clean before the next heavy run"; STATUS=1
else
say "disk ${USAGE:-?}% OK"
fi
# 4b) memory pressure (PSI) — the box swapped its way through the v3.8.50 publish
if [ -r "$JANITOR_PSI_FILE" ]; then
FULL60=$(awk '/^full/ {for(i=1;i<=NF;i++) if ($i ~ /^avg60=/) {sub("avg60=","",$i); print $i}}' "$JANITOR_PSI_FILE" 2>/dev/null || echo "")
if [ -n "$FULL60" ] && awk -v v="$FULL60" -v t="$PSI_FULL_AVG60_ALERT" 'BEGIN{exit !(v+0 >= t+0)}'; then
say "⚠ MEMORY PRESSURE psi full/avg60=${FULL60}% >= ${PSI_FULL_AVG60_ALERT}% — too many heavy jobs at once"; STATUS=1
else
say "memory psi full/avg60=${FULL60:-n/a}% OK"
fi
fi
# 4c) concurrency ceiling — alert with a breakdown; the fix is fewer/labelled
# runners (an operator decision), not killing listeners from cron.
ACTIVE=$(pgrep -fc "Runner.Listener" || true)
OMNI=$(pgrep -fc "actions-runner-omniroute[^ ]*/bin[^ ]*/Runner.Listener" || true)
if [ "${ACTIVE:-0}" -gt "$MAX_ACTIVE_RUNNERS" ]; then
say "${ACTIVE} Runner.Listener processes (omniroute=${OMNI:-0}, other=$(( ${ACTIVE:-0} - ${OMNI:-0} ))) > ceiling ${MAX_ACTIVE_RUNNERS} — stop idle extras: systemctl stop <unit> only when it has no Runner.Worker child"
# 2) Disk pressure — alert loudly before SQLITE_FULL kills jobs mid-run.
USAGE=$(df --output=pcent / | tail -1 | tr -dc '0-9')
if [ "$USAGE" -ge "$DISK_ALERT_PCT" ]; then
echo "[janitor] ⚠ ROOT DISK ${USAGE}% >= ${DISK_ALERT_PCT}% — clean before the next heavy run"
STATUS=1
else
say "runners active: ${ACTIVE:-0}/${MAX_ACTIVE_RUNNERS} (omniroute=${OMNI:-0}) OK"
echo "[janitor] disk ${USAGE}% OK"
fi
say "done status=$STATUS"
# 3) Concurrency ceiling — 8-wide OOMed the 16 GB box twice on release day;
# 4 is the proven ceiling. This CODIFIES the rule that was manual discipline.
ACTIVE=$(pgrep -fc "Runner.Listener" || true)
if [ "${ACTIVE:-0}" -gt "$MAX_ACTIVE_RUNNERS" ]; then
echo "[janitor] ⚠ ${ACTIVE} Runner.Listener processes > ceiling ${MAX_ACTIVE_RUNNERS} — stop the extra runners (systemctl stop actions.runner.<name>)"
STATUS=1
else
echo "[janitor] runners active: ${ACTIVE:-0}/${MAX_ACTIVE_RUNNERS} OK"
fi
echo "[janitor] done status=$STATUS"
exit "$STATUS"

View File

@@ -92,7 +92,7 @@ describe("Protocol clients E2E", () => {
method: "PATCH",
body: JSON.stringify({ a2aEnabled: true }),
});
expect([200, 401, 403]).toContain(response.status);
expect([200, 401]).toContain(response.status);
});
it(
@@ -134,7 +134,7 @@ describe("Protocol clients E2E", () => {
}
const auditRes = await apiFetch("/api/mcp/audit?limit=50&tool=omniroute_get_health");
expect([200, 401, 403]).toContain(auditRes.status);
expect([200, 401]).toContain(auditRes.status);
if (auditRes.status === 200) {
expect(auditRes.ok).toBe(true);
const auditJson = (await auditRes.json()) as any;

View File

@@ -7,9 +7,6 @@
*/
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
ALIBABA_FREE_TIER_TEXT_CAPABLE_MODELS,
ALIBABA_NO_FREE_TIER_TEXT_MODELS,
@@ -30,90 +27,18 @@ test("built-in allowlist includes operator free models and excludes paid blockli
assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.7-max"), false);
});
/**
* The shipped `config/alibaba-free-tier-allowlist.json` carries a `validUntil`,
* so asserting against it made this test a time bomb: it went red on its own on
* 2026-08-28, the day after the pack expired, and stayed red on every PR and on
* `main` (#11866). Nothing had changed — the clock moved.
*
* Production was never affected: an expired pack falls back to the embedded
* list by design. So the contract worth pinning is the BEHAVIOR on both sides of
* the expiry, with packs this test owns and dates it controls — never the
* freshness of the catalog that ships in the repo.
*/
function withAllowlistPack(
pack: Record<string, unknown>,
assertions: () => void
): void {
const dir = mkdtempSync(join(tmpdir(), "alibaba-allowlist-"));
const packPath = join(dir, "allowlist.json");
writeFileSync(packPath, JSON.stringify(pack), "utf8");
test("allowlist JSON pack overrides embedded lists when valid", () => {
const previousPath = process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH;
const packPath = `${process.cwd()}/config/alibaba-free-tier-allowlist.json`;
process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = packPath;
resetAlibabaFreeTierAllowlistCache();
try {
assertions();
} finally {
if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath;
else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH;
resetAlibabaFreeTierAllowlistCache();
rmSync(dir, { recursive: true, force: true });
}
}
test("allowlist JSON pack overrides embedded lists while it is still valid", () => {
withAllowlistPack(
{
asOf: "2026-07-28",
validUntil: "2999-01-01",
capable: ["pack-only-capable-model", "qwen3.6-plus"],
noFreeTier: ["pack-only-paid-model"],
},
() => {
const pack = loadAlibabaFreeTierAllowlistPack();
assert.ok(pack, "a pack inside its validity window must load");
assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!));
assert.ok(pack!.capable.includes("qwen3.6-plus"));
// Positive anchor: the pack must actually REPLACE the embedded list, not
// merely load. `pack-only-capable-model` exists nowhere else.
assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), true);
assert.equal(isAlibabaBuiltinNoFreeTierTextModel("pack-only-paid-model"), true);
}
);
});
const pack = loadAlibabaFreeTierAllowlistPack();
assert.ok(pack);
assert.ok(isAlibabaFreeTierAllowlistPackValid(pack!));
assert.ok(pack!.capable.includes("qwen3.6-plus"));
test("an expired allowlist pack is ignored and the embedded list serves instead", () => {
// This is the path production has actually been on since 2026-08-27, and it
// had no coverage at all — which is why the expiry surfaced as a red test
// rather than as a deliberate, understood fallback.
withAllowlistPack(
{
asOf: "2026-07-28",
validUntil: "2026-08-27",
capable: ["pack-only-capable-model"],
noFreeTier: ["pack-only-paid-model"],
},
() => {
assert.equal(loadAlibabaFreeTierAllowlistPack(), null, "expired pack must not load");
assert.equal(isAlibabaBuiltinFreeTierTextModel("pack-only-capable-model"), false);
// The embedded list must be what answers once the pack is rejected.
assert.equal(isAlibabaBuiltinFreeTierTextModel("qwen3.6-plus"), true);
assert.equal(isAlibabaBuiltinNoFreeTierTextModel("qwen3.7-max"), true);
}
);
});
test("isAlibabaFreeTierAllowlistPackValid compares against the instant it is given", () => {
const pack = { asOf: "2026-07-28", validUntil: "2026-08-27", capable: ["x"], noFreeTier: [] };
assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-26")), true);
assert.equal(isAlibabaFreeTierAllowlistPackValid(pack, Date.parse("2026-08-28")), false);
// No expiry declared means the pack never goes stale on its own.
assert.equal(
isAlibabaFreeTierAllowlistPackValid(
{ asOf: "2026-07-28", capable: ["x"], noFreeTier: [] },
Date.parse("2999-01-01")
),
true
);
if (previousPath) process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH = previousPath;
else delete process.env.ALIBABA_FREE_TIER_ALLOWLIST_PATH;
resetAlibabaFreeTierAllowlistCache();
});

View File

@@ -1,19 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { resolveNextBuildBundlerFlag } from "../../../scripts/build/build-next-isolated.mjs";
test("resolveNextBuildBundlerFlag returns --turbopack by default", () => {
const flag = resolveNextBuildBundlerFlag({});
assert.equal(flag, "--turbopack");
});
test("resolveNextBuildBundlerFlag returns --webpack when OMNIROUTE_USE_TURBOPACK is '0'", () => {
const flag = resolveNextBuildBundlerFlag({ OMNIROUTE_USE_TURBOPACK: "0" });
assert.equal(flag, "--webpack");
});
test("resolveNextBuildBundlerFlag returns --turbopack when OMNIROUTE_USE_TURBOPACK is '1'", () => {
const flag = resolveNextBuildBundlerFlag({ OMNIROUTE_USE_TURBOPACK: "1" });
assert.equal(flag, "--turbopack");
});

View File

@@ -88,37 +88,13 @@ test("createSyncDriverFactory prefers better-sqlite3 when running under Node", (
}
});
// `OMNIROUTE_USE_TURBOPACK` is the operator's only control over the bundler:
// Turbopack is the default and `0` is the documented escape hatch (webpack), taken
// for Windows / native-binding trouble / RAM-constrained machines — see
// docs/reference/ENVIRONMENT.md and #6409. Nothing sniffs the runtime, so pin that:
// a hidden override would silently ignore an explicit `=1` from an operator who set
// it on purpose (CI does, in build.yml / ci.yml / quality.yml). Bun 1.4+ supports
// Turbopack's V8 worker bindings (#11471), so the historical bun-only webpack
// forced path is gone from the Bun image as well.
test("resolveNextBuildBundlerFlag is decided by OMNIROUTE_USE_TURBOPACK alone, not by the runtime", async () => {
const buildIsolated = await import("../../scripts/build/build-next-isolated.mjs");
test("resolveNextBuildBundlerFlag automatically disables Turbopack and uses Webpack under Bun", async () => {
const originalBun = process.versions.bun;
try {
for (const bun of [undefined, "1.1.20", "1.3.14"]) {
if (bun === undefined) {
delete (process.versions as Record<string, string | undefined>).bun;
} else {
(process.versions as Record<string, string>).bun = bun;
}
const where = `bun=${bun ?? "absent"}`;
assert.equal(buildIsolated.resolveNextBuildBundlerFlag({}), "--turbopack", where);
assert.equal(
buildIsolated.resolveNextBuildBundlerFlag({ OMNIROUTE_USE_TURBOPACK: "1" }),
"--turbopack",
where
);
assert.equal(
buildIsolated.resolveNextBuildBundlerFlag({ OMNIROUTE_USE_TURBOPACK: "0" }),
"--webpack",
where
);
}
(process.versions as Record<string, string>).bun = "1.1.20";
const buildIsolated = await import("../../scripts/build/build-next-isolated.mjs");
assert.equal(buildIsolated.resolveNextBuildBundlerFlag({}), "--webpack");
assert.equal(buildIsolated.resolveNextBuildBundlerFlag({ OMNIROUTE_USE_TURBOPACK: "1" }), "--webpack");
} finally {
if (originalBun === undefined) {
delete (process.versions as Record<string, string | undefined>).bun;

View File

@@ -1,145 +0,0 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync, readdirSync } from "node:fs";
import { join } from "node:path";
import {
classifyRunsOn,
findProvenanceOnSelfHosted,
} from "../../scripts/check/lib/provenanceRunner.mjs";
/**
* v3.8.50, 10th publish attempt, 76 minutes in — after the tag, the GitHub
* Release and the Docker images were already public:
*
* 422 Unprocessable Entity - Error verifying sigstore provenance bundle:
* Unsupported GitHub Actions runner environment: "self-hosted".
*
* `USE_VPS_RUNNER` had routed the publish job to the .113 pool on 2026-08-02;
* no release happened between 07-30 and 08-28, so nothing surfaced it. The
* pairing is pure text, so it must fail the workflow lint on the PR that
* introduces it.
*/
const ROOT = join(import.meta.dirname, "../..");
const WORKFLOWS = join(ROOT, ".github/workflows");
// The exact runs-on expression npm-publish.yml used when it broke.
const VPS_EXPR =
"${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('[\"self-hosted\",\"omni-release\"]') || 'ubuntu-latest' }}";
function workflow(runsOn: string, run: string, extra = ""): string {
return [
"name: t",
"on: push",
"jobs:",
" publish:",
` runs-on: ${runsOn}`,
extra,
" steps:",
" - name: upload",
` run: ${run}`,
"",
].join("\n");
}
test("classifyRunsOn: literal, array, object-with-labels and the fromJSON expression are self-hosted", () => {
assert.equal(classifyRunsOn("self-hosted"), "self-hosted");
assert.equal(classifyRunsOn(["self-hosted", "omni-release"]), "self-hosted");
assert.equal(classifyRunsOn({ group: "Default", labels: ["self-hosted"] }), "self-hosted");
assert.equal(classifyRunsOn(VPS_EXPR), "self-hosted");
});
test("classifyRunsOn: hosted labels are hosted, opaque expressions are unknown (never guessed)", () => {
assert.equal(classifyRunsOn("ubuntu-latest"), "hosted");
assert.equal(classifyRunsOn(["ubuntu-latest"]), "hosted");
assert.equal(classifyRunsOn("${{ matrix.os }}"), "unknown");
assert.equal(classifyRunsOn(undefined), "unknown");
});
test("flags --provenance inside a job routed to the self-hosted pool", () => {
const found = findProvenanceOnSelfHosted(
workflow(
JSON.stringify(VPS_EXPR),
'npm stage publish --provenance --access public --tag "$TAG"'
),
"npm-publish.yml"
);
assert.deepEqual(found, [{ file: "npm-publish.yml", job: "publish", step: "upload" }]);
});
test("also catches the literal label and the --provenance-file form", () => {
assert.equal(
findProvenanceOnSelfHosted(workflow("self-hosted", "npm publish --provenance")).length,
1
);
assert.equal(
findProvenanceOnSelfHosted(
workflow("[self-hosted, omni-release]", "npm publish --provenance-file=./p.json")
).length,
0,
"--provenance-file is a different flag (a pre-built bundle) and is not what the registry rejects"
);
assert.equal(
findProvenanceOnSelfHosted(workflow("self-hosted", "npm publish --provenance=true")).length,
1
);
});
test("does not flag hosted jobs, unknown runners, or self-hosted jobs without the flag", () => {
assert.deepEqual(
findProvenanceOnSelfHosted(workflow("ubuntu-latest", "npm publish --provenance")),
[]
);
assert.deepEqual(
findProvenanceOnSelfHosted(workflow("${{ matrix.os }}", "npm publish --provenance")),
[]
);
assert.deepEqual(
findProvenanceOnSelfHosted(workflow("self-hosted", "npm publish --access public")),
[]
);
// The word only in a step NAME or a comment is not a finding.
assert.deepEqual(
findProvenanceOnSelfHosted(
[
"name: t",
"on: push",
"jobs:",
" j:",
" runs-on: self-hosted",
" steps:",
" - name: provenance note",
" run: echo hi # --provenance later",
"",
].join("\n")
),
[],
"a comment after the command is still part of the run string — accept that the regex is conservative"
);
});
test("reusable-workflow jobs (uses:) and unparseable YAML are not this rule's findings", () => {
const reusable = [
"name: t",
"on: push",
"jobs:",
" j:",
" uses: ./.github/workflows/x.yml",
"",
].join("\n");
assert.deepEqual(findProvenanceOnSelfHosted(reusable), []);
assert.deepEqual(findProvenanceOnSelfHosted("jobs: [unclosed"), []);
});
test("regression guard: no workflow in this repo publishes with --provenance from a self-hosted runner", () => {
const files = readdirSync(WORKFLOWS).filter((f) => /\.ya?ml$/.test(f));
assert.ok(files.length > 10, "expected the real workflow set");
const findings = files.flatMap((f) =>
findProvenanceOnSelfHosted(readFileSync(join(WORKFLOWS, f), "utf8"), f)
);
assert.deepEqual(
findings,
[],
`npm rejects provenance from self-hosted runners (422) — move the upload to a github-hosted job: ${JSON.stringify(findings)}`
);
});

View File

@@ -1,135 +0,0 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
classifyExpiry,
collectExpiryFields,
scanConfigExpiry,
} from "../../scripts/check/lib/configExpiry.mjs";
/**
* Time bombs: config packs with a `validUntil` (or sibling key) that lapse with
* no commit involved. The Alibaba free-tier pack expired on 2026-08-27 and from
* the 28th every PR and main carried a red Unit Tests shard (#11866). Nothing a
* diff review could have caught.
*
* This suite fails SEVEN DAYS BEFORE any pack under config/ lapses, naming the
* file and key, so renewal happens on someone's terms instead of the clock's.
*/
const ROOT = join(import.meta.dirname, "../..");
const CONFIG_DIR = join(ROOT, "config");
const DAY = 86_400_000;
const WARN_DAYS = 7;
/**
* Packs known to be expired/expiring, each pinned to the issue that owns the
* renewal decision. An entry whose pack is no longer expiring FAILS below as a
* stale allowlist entry — remove it when the pack is renewed.
*/
const ALLOWLIST: Record<string, string> = {
"alibaba-free-tier-allowlist.json":
"#11866 — validUntil 2026-08-27 has passed; the loader already falls back to the embedded list, and renewing the curated free-tier pack is an operator data decision, not a test fix",
};
const NOW = Date.UTC(2026, 7, 28); // 2026-08-28, fixed: this suite must not itself depend on the clock
const day = (offset: number) => new Date(NOW + offset * DAY).toISOString().slice(0, 10);
test("collectExpiryFields: finds nested and array-nested expiry keys, ignores non-string values", () => {
const fields = collectExpiryFields(
{
validUntil: day(3),
nested: { expiresAt: day(30), other: "x" },
list: [{ expiry: day(-1) }, { expires: 12345 }],
expires_at: "not a date",
},
"pack.json"
);
assert.deepEqual(
fields.map((f) => [f.keyPath, f.expiresAt === null ? null : "date"]),
[
["validUntil", "date"],
["nested.expiresAt", "date"],
["list.0.expiry", "date"],
["expires_at", null],
]
);
});
test("classifyExpiry: expired / expiring inside the warning window / ok / unparseable", () => {
const f = (raw: string) => ({
file: "p",
keyPath: "validUntil",
raw,
expiresAt: Number.isFinite(Date.parse(raw)) ? Date.parse(raw) : null,
});
assert.equal(classifyExpiry(f(day(-1)), NOW, WARN_DAYS), "expired");
assert.equal(
classifyExpiry(f(day(0)), NOW, WARN_DAYS),
"expiring",
"lapsing today is already too late to be 'ok'"
);
assert.equal(classifyExpiry(f(day(6)), NOW, WARN_DAYS), "expiring");
assert.equal(classifyExpiry(f(day(8)), NOW, WARN_DAYS), "ok");
assert.equal(classifyExpiry(f("never"), NOW, WARN_DAYS), "unparseable");
});
test("scanConfigExpiry: walks a config tree, skips node_modules and invalid JSON, keys files portably", () => {
const dir = mkdtempSync(join(tmpdir(), "cfg-expiry-"));
try {
mkdirSync(join(dir, "sub"), { recursive: true });
mkdirSync(join(dir, "node_modules", "dep"), { recursive: true });
writeFileSync(join(dir, "a.json"), JSON.stringify({ validUntil: day(3) }));
writeFileSync(join(dir, "sub", "b.json"), JSON.stringify({ deep: { expiresAt: day(40) } }));
writeFileSync(
join(dir, "node_modules", "dep", "c.json"),
JSON.stringify({ validUntil: day(-5) })
);
writeFileSync(join(dir, "broken.json"), "{ not json");
writeFileSync(join(dir, "notes.txt"), JSON.stringify({ validUntil: day(-5) }));
const found = scanConfigExpiry(dir).map((f) => `${f.file}:${f.keyPath}`);
assert.deepEqual(found, ["a.json:validUntil", "sub/b.json:deep.expiresAt"]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test(`repo: no pack under config/ lapses within ${WARN_DAYS} days unless its renewal is tracked`, (t) => {
const fields = scanConfigExpiry(CONFIG_DIR);
// Positive anchor: the scanner must be seeing SOMETHING, or a renamed key
// would silently turn this whole suite into a no-op.
assert.ok(
fields.length >= 1,
"expected at least one dated pack under config/ (the Alibaba allowlist) — if the key was renamed, extend EXPIRY_KEY"
);
const failures: string[] = [];
const seenAllowlisted = new Set<string>();
for (const f of fields) {
const status = classifyExpiry(f, Date.now(), WARN_DAYS);
const tracked = ALLOWLIST[f.file];
if (status === "unparseable") {
t.diagnostic(`${f.file} ${f.keyPath}="${f.raw}" is not a date — not monitored`);
continue;
}
if (status === "ok") continue;
if (tracked) {
seenAllowlisted.add(f.file);
t.diagnostic(`${f.file} ${f.keyPath}=${f.raw} is ${status} — tracked: ${tracked}`);
continue;
}
failures.push(
`${f.file}${f.keyPath}=${f.raw} is ${status}: renew the pack (or track it in ALLOWLIST with its issue)`
);
}
for (const file of Object.keys(ALLOWLIST)) {
if (!seenAllowlisted.has(file)) {
failures.push(
`stale ALLOWLIST entry: ${file} is no longer expired/expiring — remove it (${ALLOWLIST[file]})`
);
}
}
assert.deepEqual(failures, [], failures.join("\n"));
});

View File

@@ -4,109 +4,76 @@ import { readFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
// The Docker publish workflow builds on GitHub-hosted runners (ubuntu-24.04 and
// ubuntu-24.04-arm): 4 vCPU, 16 GB RAM. Every Next page-data worker AND the
// parent `next build` process are separate OS processes, so the budget has to
// cover all of them, not just the workers. Both images (node and Bun) run the
// same build-next-isolated.mjs pipeline on the same runners; the Bun image runs
// Turbopack by default on Bun 1.4+ (#11471) and the webpack fallback
// (OMNIROUTE_USE_TURBOPACK=0) keeps memory in V8, so the guards must hold for
// both bundlers on both images (#11709).
// ubuntu-24.04-arm): 4 vCPU, 16 GB RAM. Every Next page-data worker is its own
// process and inherits NODE_OPTIONS, so the V8 ceiling is per PROCESS: the
// build's worst case is roughly `workers × OMNIROUTE_BUILD_MEMORY_MB`.
//
// With 7 workers × 6144 MB the runner ran out and buildkit failed the step with
// `ResourceExhausted: ... cannot allocate memory`, right after "Collecting page
// data using 7 workers" — every Docker publish since 2026-08-22 23:14 UTC.
// Lowering to 2 workers (#10060 / PR #11419) was not enough: it modeled the
// per-process peak as an INFERENCE (`WORKER_PEAK_MB = 2560`, derived only from
// "7 workers didn't fit") and assumed the parent process tracked the V8 heap
// ceiling (`OMNIROUTE_BUILD_MEMORY_MB`) rather than its own RSS. The owner's
// live VPS reproduction (issue #7518, dmesg OOM-killer report, 2026-08-24)
// measured the real number directly: `next-build (v16) ... anon-rss:4522744kB`
// (~4.5 GB) per process, independent of the NODE_OPTIONS heap flag — Turbopack
// itself is native/Rust and compiles outside the V8 heap. With 2 workers that
// keeps the publish pipeline failing at "Collecting page data using 2 workers"
// (run 32907937950, 2026-08-25).
//
// This pins the budget on the MEASURED figure, applied uniformly to every
// process (parent + workers) and to both images, so raising the worker count
// has to be a deliberate change that re-does the arithmetic, not a one-line
// bump that silently reds the publish pipeline again.
// This pins the budget so raising either knob has to be a deliberate change
// that re-does the arithmetic, not a one-line bump that silently reds the
// publish pipeline again.
const RUNNER_MEMORY_MB = 16 * 1024;
// Leave room for buildkit, the snapshotter and page cache.
const HEADROOM_FRACTION = 0.75;
// Measured (not inferred) peak RSS for a single Next/Turbopack build process —
// parent or page-data worker alike — from the dmesg OOM-killer report above.
// If a future build OOMs again, re-measure via dmesg before raising this
// number — do not weaken the budget with another guess.
const MEASURED_PROCESS_RSS_MB = 4500;
// Planning figure for one page-data worker's peak RSS. It is an INFERENCE, not
// a measurement: 7 workers did not fit in 16 GB alongside the parent, which
// puts the per-worker peak somewhere north of ~1.8 GB. 2.5 GB is that bound
// rounded up, so the budget below stays conservative. If a future build OOMs
// again with a worker count this test accepts, raise this number — do not
// weaken the budget.
const WORKER_PEAK_MB = 2560;
const DOCKERFILES = [
{ label: "Dockerfile", raw: readFileSync(fileURLToPath(new URL("../../Dockerfile", import.meta.url)), "utf8") },
{ label: "Dockerfile.bun", raw: readFileSync(fileURLToPath(new URL("../../Dockerfile.bun", import.meta.url)), "utf8") },
];
const dockerfile = readFileSync(
fileURLToPath(new URL("../../Dockerfile", import.meta.url)),
"utf8"
);
function readArgDefault(name: string, label: string): number {
const raw = DOCKERFILES.find((entry) => entry.label === label)!.raw;
const match = raw.match(new RegExp(`^ARG ${name}=(\\d+)$`, "m"));
assert.ok(match, `${label} no longer declares ARG ${name}`);
function readArgDefault(name: string): number {
const match = dockerfile.match(new RegExp(`^ARG ${name}=(\\d+)$`, "m"));
assert.ok(match, `Dockerfile no longer declares ARG ${name}`);
return Number(match![1]);
}
for (const { label } of DOCKERFILES) {
test(`the ${label} build's worker pool is derived from OMNIROUTE_BUILD_WORKERS`, () => {
// assert.ok(boolean), not assert.match — a failing assert.match dumps the
// whole Dockerfile into the report.
const raw = DOCKERFILES.find((entry) => entry.label === label)!.raw;
assert.ok(
/^ENV CIRCLE_NODE_TOTAL=\$\{OMNIROUTE_BUILD_WORKERS\}$/m.test(raw),
`${label}: CIRCLE_NODE_TOTAL must stay wired to the build arg so a big builder can raise it`
);
assert.ok(
/^ENV NODE_OPTIONS="--max-old-space-size=\$\{OMNIROUTE_BUILD_MEMORY_MB\}"$/m.test(raw),
`${label}: the build heap ceiling must stay wired to OMNIROUTE_BUILD_MEMORY_MB`
);
});
test("the Docker build's worker pool is derived from OMNIROUTE_BUILD_WORKERS", () => {
// assert.ok(boolean), not assert.match — a failing assert.match dumps the
// whole Dockerfile into the report.
assert.ok(
/^ENV CIRCLE_NODE_TOTAL=\$\{OMNIROUTE_BUILD_WORKERS\}$/m.test(dockerfile),
"CIRCLE_NODE_TOTAL must stay wired to the build arg so a big builder can raise it"
);
assert.ok(
/^ENV NODE_OPTIONS="--max-old-space-size=\$\{OMNIROUTE_BUILD_MEMORY_MB\}"$/m.test(dockerfile),
"the build heap ceiling must stay wired to OMNIROUTE_BUILD_MEMORY_MB"
);
});
test(`worker count × measured per-process RSS fits a 16 GB GitHub runner (${label})`, () => {
const workerPool = readArgDefault("OMNIROUTE_BUILD_WORKERS", label);
test("worker count × per-process heap fits a 16 GB GitHub runner", () => {
const workerPool = readArgDefault("OMNIROUTE_BUILD_WORKERS");
const heapMb = readArgDefault("OMNIROUTE_BUILD_MEMORY_MB");
// Next derives `workers = CIRCLE_NODE_TOTAL - 1`.
const workers = workerPool - 1;
assert.ok(workers >= 1, `${label}: CIRCLE_NODE_TOTAL=${workerPool} leaves no build workers`);
// Next derives `workers = CIRCLE_NODE_TOTAL - 1`.
const workers = workerPool - 1;
assert.ok(workers >= 1, `CIRCLE_NODE_TOTAL=${workerPool} leaves no build workers`);
// Every process — the parent `next build` process AND each page-data
// worker — is budgeted at the measured per-process RSS floor (see the file
// banner comment). The V8 heap ceiling (OMNIROUTE_BUILD_MEMORY_MB) bounds
// JS allocations but not Turbopack's native/Rust memory, so it cannot stand
// in for the parent process's real RSS.
const processes = workers + 1;
const worstCaseMb = processes * MEASURED_PROCESS_RSS_MB;
const budgetMb = RUNNER_MEMORY_MB * HEADROOM_FRACTION;
assert.ok(
worstCaseMb <= budgetMb,
`${label}: ${processes} processes (1 parent + ${workers} workers) × ${MEASURED_PROCESS_RSS_MB} MB ` +
`measured RSS = ${worstCaseMb} MB exceeds the ${budgetMb} MB budget on a ` +
`${RUNNER_MEMORY_MB} MB runner — the Docker publish step dies with "ResourceExhausted: ` +
`cannot allocate memory" during page-data collection`
);
});
// The parent `next build` process is the one that genuinely needs the raised
// ceiling (the webpack/turbopack production pass, #4076); the workers are
// budgeted at their inferred peak instead.
const worstCaseMb = heapMb + workers * WORKER_PEAK_MB;
const budgetMb = RUNNER_MEMORY_MB * HEADROOM_FRACTION;
assert.ok(
worstCaseMb <= budgetMb,
`parent ${heapMb} MB + ${workers} workers × ${WORKER_PEAK_MB} MB = ${worstCaseMb} MB ` +
`exceeds the ${budgetMb} MB budget on a ${RUNNER_MEMORY_MB} MB runner — the Docker ` +
`publish step dies with "ResourceExhausted: cannot allocate memory" during page-data ` +
`collection`
);
});
test(`both images default to OMNIROUTE_BUILD_WORKERS=2 (1 page-data worker) (${label})`, () => {
// At OMNIROUTE_BUILD_WORKERS=2 → CIRCLE_NODE_TOTAL=2 → Next derives 1
// page-data worker, so 2 processes (parent + worker) × ~4.5 GB ≈ 9 GB fit
// the 12.288 GB (75%) budget on a 16 GB runner with headroom. At =3 → 2
// workers → 3 processes × 4.5 GB ≈ 13.5 GB, which exceeds it (measured
// per-process RSS, #7518). Raising this default must re-do the budget
// arithmetic and stay green on the test above (#11663).
assert.equal(
readArgDefault("OMNIROUTE_BUILD_WORKERS", label),
2,
`${label}: OMNIROUTE_BUILD_WORKERS must stay 2 (1 page-data worker)`
);
});
test(`the worker pool does not oversubscribe the runner's 4 vCPU (${label})`, () => {
const workers = readArgDefault("OMNIROUTE_BUILD_WORKERS", label) - 1;
assert.ok(workers <= 4, `${workers} workers oversubscribe a 4 vCPU runner`);
});
}
test("the worker pool does not oversubscribe the runner's 4 vCPU", () => {
const workers = readArgDefault("OMNIROUTE_BUILD_WORKERS") - 1;
assert.ok(workers <= 4, `${workers} workers oversubscribe a 4 vCPU runner`);
});

View File

@@ -7,8 +7,6 @@ import {
FATAL_LOG_PATTERNS,
LINUX_EXECUTABLE_NAMES,
stopApp,
waitForDatabaseOpen,
DB_TOUCH_PATH,
} from "../../scripts/dev/smoke-electron-packaged.mjs";
test("electron smoke discovers the default Linux executable name", () => {
@@ -96,46 +94,6 @@ test("electron smoke flags a cold-restart fallback to the sql.js WASM driver", (
test("electron smoke flags startup logs missing any driver selection line", () => {
assert.throws(
() => assertNativeDriverSelected("[electron] [server] listening on 20128"),
/no database activity/
);
});
test("electron smoke waits for database-open evidence after touching a DB-backed endpoint", async () => {
assert.equal(DB_TOUCH_PATH, "/api/monitoring/health");
let logs = "[electron] [Server] [STARTUP] ready\n";
setTimeout(() => {
logs += "[electron] [Server] [DB] Added usage_history.combo_strategy column\n";
}, 60);
const seen = await waitForDatabaseOpen(() => logs, { timeoutMs: 2_000, pollMs: 20 });
assert.match(seen, /\[DB\] Added/);
});
test("electron smoke fails clearly when the database never opens", async () => {
await assert.rejects(
() =>
waitForDatabaseOpen(() => "[electron] [Server] [STARTUP] ready\n", {
timeoutMs: 120,
pollMs: 20,
}),
/logged no \[DB\]\/\[Migration\] startup line within 120ms/
);
});
test("electron smoke driver guard: native line, DB evidence and sql.js fallback", () => {
assert.doesNotThrow(() =>
assertNativeDriverSelected("[DB] Driver: better-sqlite3 | file: /tmp/x/storage.sqlite\n")
);
assert.doesNotThrow(() =>
assertNativeDriverSelected(
"[electron] [Server] [DB] Added call_logs.session_tag column\n[electron] [Server] [Migration] Applied: 046_database_settings\n"
)
);
assert.throws(
() => assertNativeDriverSelected("[DB] Driver: sql.js | file: /tmp/x/storage.sqlite\n"),
/sql\.js \(WASM\) driver/
);
assert.throws(
() => assertNativeDriverSelected("[STARTUP] nothing here\n"),
/no database activity/
/no '\[DB\] Driver: \.\.\.' line/
);
});

View File

@@ -1,72 +0,0 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync, readdirSync } from "node:fs";
import type { Dirent } from "node:fs";
import { join } from "node:path";
/**
* v3.8.50 was refused by the registry with `413 Payload Too Large` on
* `POST /-/stage/package/omniroute`: the tarball had reached 288.7 MB packed
* (1.1 GB unpacked), against 174.5 MB for the 3.8.49 that published fine.
*
* 668.7 MB of that — 61% of the whole package — was 842 `*.nft.json` files.
* Those are Next.js Node File Trace manifests: build-time metadata used to
* COMPUTE the standalone bundle, never read while serving. They had doubled
* since 3.8.49 (325.0 MB across 748 files), which is what tipped the payload
* over the limit.
*
* The guard is the `files[]` negation, so a future entry that re-widens the
* glob (or a rewrite of the array) cannot silently put them back.
*/
const pkg = JSON.parse(readFileSync(join(import.meta.dirname, "../../package.json"), "utf8")) as {
files?: string[];
};
test("package.json files[] excludes Next's .nft.json trace manifests", () => {
const files = pkg.files ?? [];
assert.ok(files.length > 0, "package.json must declare files[]");
assert.ok(
files.includes("!**/*.nft.json"),
"files[] must negate **/*.nft.json — they are build metadata and were 61% of the 3.8.50 payload"
);
});
test("the negation sits after the positive dist/ entry it has to override", () => {
// npm applies files[] in order: a negation listed BEFORE the directory that
// pulls the files in is a no-op. Positive anchor, so this test cannot pass
// just because both strings happen to be present somewhere.
const files = pkg.files ?? [];
const dist = files.indexOf("dist/");
const negation = files.indexOf("!**/*.nft.json");
assert.notEqual(dist, -1, "dist/ must still be published");
assert.ok(negation > dist, "the .nft.json negation must come after dist/");
});
test("no source module reads a .nft.json at runtime", () => {
// If this ever stops holding, the exclusion above becomes a runtime break
// rather than a size win — which is exactly the assumption worth pinning.
const roots = ["src", "open-sse", "bin"];
const hits: string[] = [];
for (const root of roots) {
const dir = join(import.meta.dirname, "../..", root);
const stack = [dir];
while (stack.length > 0) {
const current = stack.pop() as string;
let entries: Dirent[];
try {
entries = readdirSync(current, { withFileTypes: true });
} catch {
continue;
}
for (const entry of entries) {
const full = join(current, entry.name);
if (entry.isDirectory()) {
if (entry.name !== "node_modules") stack.push(full);
} else if (/\.(ts|tsx|mjs|js)$/.test(entry.name)) {
if (readFileSync(full, "utf8").includes(".nft.json")) hits.push(full);
}
}
}
}
assert.deepEqual(hits, [], `nothing may depend on .nft.json at runtime: ${hits.join(", ")}`);
});

View File

@@ -1,196 +0,0 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
statSync,
utimesSync,
writeFileSync,
} from "node:fs";
import os from "node:os";
import path from "node:path";
/**
* scripts/ops/runner-janitor.sh runs from cron on the .113 runner box. This
* suite pins its safety contract against a fixture tree — never the real /tmp:
* every base, the runner dirs, the PSI file and the df path are redirected, the
* zombie pattern is set to a name no process has, and the ceilings are lifted
* so the outcome does not depend on the box the test happens to run on.
*/
const ROOT = path.resolve(import.meta.dirname, "..", "..");
const SCRIPT = path.join(ROOT, "scripts", "ops", "runner-janitor.sh");
const HOUR = 3_600_000;
// The sweep needs lsof to PROVE a path is idle (one snapshot of open paths). Hosted CI
// images ship both; a bare devbox may not. Each branch below asserts what must
// hold in that environment — without the tools the contract is "delete nothing,
// say why", which is exactly the behaviour worth pinning.
const HAVE_BUSY_TOOLS =
spawnSync("bash", ["-c", "command -v lsof"], { stdio: "ignore" }).status === 0;
function fixture() {
const base = mkdtempSync(path.join(os.tmpdir(), "janitor-fixture-"));
const old = new Date(Date.now() - 5 * HOUR);
const mk = (name: string, dir: boolean, when: Date | null) => {
const p = path.join(base, name);
if (dir) {
mkdirSync(p);
writeFileSync(path.join(p, "x"), "x");
} else writeFileSync(p, "x");
if (when) utimesSync(p, when, when);
return p;
};
return {
base,
staleTar: mk("e2e-build.tar.gz", false, old), // fixed-name artefact ci.yml/npm-publish leave behind
staleBuild: mk("next-build-abc", true, old),
staleUpgrade: mk("omniroute-install-upgrade-xyz", true, old),
fresh: mk("omniroute-batch-api-fresh", true, null), // in use right now
unrelated: mk("somebody-elses.log", false, old), // not ours — never touched
};
}
function run(args: string[], base: string, extraEnv: Record<string, string> = {}) {
return spawnSync("bash", [SCRIPT, ...args], {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
env: {
...process.env,
JANITOR_TMP_BASES: base,
JANITOR_WORK_TEMP_BASES: "",
JANITOR_RUNNER_DIRS: path.join(base, "no-runners-here-*"),
JANITOR_PSI_FILE: path.join(base, "no-psi"),
JANITOR_DF_PATH: base,
ZOMBIE_BUILD_COMM: "janitor-test-no-such-process",
MAX_ACTIVE_RUNNERS: "9999",
DISK_ALERT_PCT: "101",
...extraEnv,
},
});
}
describe("runner-janitor.sh", () => {
it("is executable bash with strict mode and prints usage on --help", () => {
assert.ok(existsSync(SCRIPT));
assert.ok(statSync(SCRIPT).mode & 0o111, "must be chmod +x (cron runs it directly)");
const body = readFileSync(SCRIPT, "utf8");
assert.ok(body.startsWith("#!/usr/bin/env bash"));
assert.ok(body.includes("set -euo pipefail"));
const help = run(["--help"], os.tmpdir());
assert.equal(help.status, 0, help.stderr);
assert.match(help.stdout, /--dry-run/);
});
it("without lsof it cannot prove idle, so it deletes nothing and says why (exit 1)", () => {
const f = fixture();
try {
const r = run([], f.base, { JANITOR_LSOF: "/nonexistent/lsof" });
assert.equal(r.status, 1, "a janitor that cannot do its job must show up in the cron log");
assert.match(r.stdout, /busy-tools=MISSING/);
assert.match(
r.stdout,
/cannot prove idle \(lsof missing — apt install lsof\), kept: .*e2e-build\.tar\.gz/
);
for (const p of [f.staleTar, f.staleBuild, f.staleUpgrade, f.fresh, f.unrelated]) {
assert.ok(existsSync(p), `must not delete ${p} when idleness cannot be proven`);
}
} finally {
rmSync(f.base, { recursive: true, force: true });
}
});
it("--dry-run names what it WOULD remove and removes nothing", (t) => {
if (!HAVE_BUSY_TOOLS) return t.skip("lsof absent on this box — sweep branch covered in CI");
const f = fixture();
try {
const r = run(["--dry-run"], f.base);
assert.equal(r.status, 0, r.stderr + r.stdout);
assert.match(r.stdout, /busy-tools=ok/);
for (const p of [f.staleTar, f.staleBuild, f.staleUpgrade]) {
assert.match(
r.stdout,
new RegExp(`would remove \\(3h\\+\\): ${p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`)
);
assert.doesNotMatch(
r.stdout,
new RegExp(`removed \\(3h\\+\\): ${p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`),
"dry-run must never claim it removed something"
);
assert.ok(existsSync(p), `dry-run must not delete ${p}`);
}
assert.doesNotMatch(
r.stdout,
/omniroute-batch-api-fresh/,
"a fresh dir is never a candidate"
);
assert.doesNotMatch(r.stdout, /somebody-elses\.log/, "only names our tooling creates");
assert.match(r.stdout, /zombie builds: 0/);
assert.match(r.stdout, /done status=0/);
} finally {
rmSync(f.base, { recursive: true, force: true });
}
});
it("for real: sweeps the three stale artefacts, keeps the fresh one and the stranger", (t) => {
if (!HAVE_BUSY_TOOLS) return t.skip("lsof absent on this box — sweep branch covered in CI");
const f = fixture();
try {
const r = run([], f.base);
assert.equal(r.status, 0, r.stderr + r.stdout);
assert.ok(!existsSync(f.staleTar), "stale e2e-build.tar.gz must go (it is RAM on tmpfs)");
assert.ok(!existsSync(f.staleBuild), "stale next-build dir must go");
assert.ok(!existsSync(f.staleUpgrade), "stale install-upgrade dir must go");
assert.ok(existsSync(f.fresh), "a fresh dir must survive");
assert.ok(existsSync(f.unrelated), "files we did not create must survive even when old");
} finally {
rmSync(f.base, { recursive: true, force: true });
}
});
it("tmpfs fuse is shorter than the disk fuse (RAM vs disk), both overridable", () => {
const f = fixture();
try {
// With a 6h tmpfs fuse the 5h-old artefacts are NOT stale yet.
const r = run(["--dry-run"], f.base, { TMPFS_MAX_AGE_HOURS: "6" });
assert.doesNotMatch(
r.stdout,
/would remove|removed \(|cannot prove idle/,
"nothing is stale under a 6h fuse, so no candidate is even examined"
);
const body = readFileSync(SCRIPT, "utf8");
assert.match(body, /TMPFS_MAX_AGE_HOURS:-3\}/, "tmpfs default must stay short — it is RAM");
assert.match(body, /WORK_TEMP_MAX_AGE_HOURS:-24\}/);
} finally {
rmSync(f.base, { recursive: true, force: true });
}
});
it("alerts (exit 1) on disk and memory pressure thresholds without touching files", () => {
const f = fixture();
try {
writeFileSync(
path.join(f.base, "psi"),
"some avg10=0.00 avg60=0.00 avg300=0.00 total=1\nfull avg10=0.00 avg60=23.50 avg300=9.00 total=1\n"
);
const r = run(["--dry-run"], f.base, {
JANITOR_PSI_FILE: path.join(f.base, "psi"),
DISK_ALERT_PCT: "0",
});
assert.equal(r.status, 1, "attention needed must be exit 1 for the cron log");
assert.match(r.stdout, /MEMORY PRESSURE psi full\/avg60=23\.50%/);
assert.ok(existsSync(f.fresh) && existsSync(f.unrelated));
assert.match(r.stdout, /ROOT DISK \d+% >= 0%/);
assert.ok(existsSync(f.staleTar), "alerting never deletes");
} finally {
rmSync(f.base, { recursive: true, force: true });
}
});
it("rejects unknown arguments instead of silently running", () => {
const r = run(["--yolo"], os.tmpdir());
assert.equal(r.status, 2);
});
});