From 5b8d63c094da3ba2949c6442c45aaced9deecfb9 Mon Sep 17 00:00:00 2001
From: Diego Rodrigues de Sa e Souza
 <8016841+diegosouzapw@users.noreply.github.com>
Date: Tue, 14 Jul 2026 13:51:06 -0300
Subject: [PATCH] chore(ops): runner-box janitor + operations runbook (WS3.3)
 (#7115)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore(ops): runner-box janitor script + operations runbook (WS3.3)

Codifies what was manual discipline on the .113 self-hosted pool (two live
incidents on the v3.8.47 release day): 30min cron sweeping stale runner
temp/work dirs (>24h), disk-pressure alert at >=85% (SQLITE_FULL killed shards
mid-run), and the proven 4-runner ceiling on the 16 GB box (8-wide OOM'd jobs;
stopping a busy runner cancels its job — documented). Script smoke-tested live
(disk 82%, 1 active runner, exit 0); bash -n clean.

* docs(ops): reword error-code/bash-env mentions the fabricated-docs env detector misreads

* fix(ops): harden janitor sweep — no symlink follow, -xdev, narrowed patterns (root-cron on world-writable /tmp)
---
 changelog.d/maintenance/runner-janitor.md |  1 +
 docs/ops/RUNNER_BOX.md                    | 35 +++++++++++++++
 scripts/ops/runner-janitor.sh             | 53 +++++++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 changelog.d/maintenance/runner-janitor.md
 create mode 100644 docs/ops/RUNNER_BOX.md
 create mode 100755 scripts/ops/runner-janitor.sh

diff --git a/changelog.d/maintenance/runner-janitor.md b/changelog.d/maintenance/runner-janitor.md
new file mode 100644
index 0000000000..365f4a5b00
--- /dev/null
+++ b/changelog.d/maintenance/runner-janitor.md
@@ -0,0 +1 @@
+- **Ops**: `scripts/ops/runner-janitor.sh` + `docs/ops/RUNNER_BOX.md` codify the self-hosted runner box hygiene that was manual discipline — 30min cron sweeping stale runner temp dirs, alerting at ≥85% disk, and enforcing the proven 4-runner ceiling on the 16 GB box (8-wide OOM-killed jobs twice on the v3.8.47 release day)
diff --git a/docs/ops/RUNNER_BOX.md b/docs/ops/RUNNER_BOX.md
new file mode 100644
index 0000000000..07bd70cb04
--- /dev/null
+++ b/docs/ops/RUNNER_BOX.md
@@ -0,0 +1,35 @@
+---
+title: Self-Hosted Runner Box Operations
+---
+
+# Self-Hosted Runner Box Operations (.113 pool)
+
+The self-hosted pool (`self-hosted, omni-release` labels) runs on the 16 GB box at
+`192.168.0.113`. Two failure modes recurred on release days and were, until v3.8.49,
+manual discipline; the **janitor script codifies them** (WS3.3 of the quality plan):
+
+1. **Orphaned temp/work dirs** filling the disk → disk-full SQLite errors mid-job.
+2. **>4 concurrent runners** → OOM-killed jobs (8-wide killed jobs twice on the
+   v3.8.47 release day; 4-wide is the proven ceiling).
+
+## Install the janitor (one-time, on the box)
+
+```bash
+sudo mkdir -p /opt/omniroute-ops
+sudo cp scripts/ops/runner-janitor.sh /opt/omniroute-ops/
+sudo chmod +x /opt/omniroute-ops/runner-janitor.sh
+( sudo crontab -l 2>/dev/null; echo '*/30 * * * * /opt/omniroute-ops/runner-janitor.sh >> /var/log/runner-janitor.log 2>&1' ) | sudo crontab -
+```
+
+What it does every 30min: sweeps runner temp leftovers older than 24h, alerts at
+≥85% root-disk usage, and alerts when more than the runner ceiling (default 4, tunable
+via the script's own environment) of `Runner.Listener` processes are up. Alerts land in `/var/log/runner-janitor.log`
+with a non-zero exit (grep for `⚠`).
+
+## Operating rules
+
+- **Ceiling: 4 runners** on the 16 GB box. Runners 5–8 stay STOPPED except for
+  explicit off-peak experiments — never during a release window.
+- Stopping a runner mid-job cancels the job (observed live): `systemctl stop`
+  only when its runner is idle (`Runner.Listener` without a `Runner.Worker` child).
+- The `.15` VPS is homologation-only — never runs CI runners.
diff --git a/scripts/ops/runner-janitor.sh b/scripts/ops/runner-janitor.sh
new file mode 100755
index 0000000000..99c081b10f
--- /dev/null
+++ b/scripts/ops/runner-janitor.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# runner-janitor — self-hosted runner box hygiene (WS3.3, v3.8.49 quality plan).
+#
+# The .113 runner box has recurring failure modes that until now were manual
+# discipline: orphaned tmpfs/work dirs filling the disk, and >4 concurrent
+# runners OOM-killing jobs (16 GB box; incidents on the v3.8.47 release day).
+# Install via cron on the box (see docs/ops/RUNNER_BOX.md):
+#   */30 * * * *  /opt/omniroute-ops/runner-janitor.sh >> /var/log/runner-janitor.log 2>&1
+#
+# Exit codes: 0 healthy · 1 attention needed (printed to stdout for the log).
+set -euo pipefail
+
+MAX_ACTIVE_RUNNERS="${MAX_ACTIVE_RUNNERS:-4}"
+DISK_ALERT_PCT="${DISK_ALERT_PCT:-85}"
+WORK_DIR_MAX_AGE_HOURS="${WORK_DIR_MAX_AGE_HOURS:-24}"
+STATUS=0
+
+echo "[janitor] $(date -u +%FT%TZ) start"
+
+# 1) Sweep stale runner temp/work leftovers (>24h — no legitimate job runs that long).
+# Hardened for a root cron on world-writable paths: never follow a symlinked base
+# (a compromised runner could plant one), -P + -xdev so the sweep cannot traverse
+# out of the filesystem, and patterns narrowed to names OUR tooling creates
+# (no generic tmp* — unrelated system temp files are out of scope).
+for base in /tmp /home/*/actions-runner*/_work/_temp; do
+  [ -d "$base" ] || continue
+  [ -L "$base" ] && { echo "[janitor] skip symlinked base: $base"; continue; }
+  find -P "$base" -xdev -maxdepth 1 \( -name 'runner-*' -o -name 'omniroute-*' \) \
+    ! -type l -mmin +$((WORK_DIR_MAX_AGE_HOURS * 60)) -exec rm -rf {} + 2>/dev/null || true
+done
+echo "[janitor] stale temp sweep done"
+
+# 2) Disk pressure — alert loudly before SQLITE_FULL kills jobs mid-run.
+USAGE=$(df --output=pcent / | tail -1 | tr -dc '0-9')
+if [ "$USAGE" -ge "$DISK_ALERT_PCT" ]; then
+  echo "[janitor] ⚠ ROOT DISK ${USAGE}% >= ${DISK_ALERT_PCT}% — clean before the next heavy run"
+  STATUS=1
+else
+  echo "[janitor] disk ${USAGE}% OK"
+fi
+
+# 3) Concurrency ceiling — 8-wide OOMed the 16 GB box twice on release day;
+#    4 is the proven ceiling. This CODIFIES the rule that was manual discipline.
+ACTIVE=$(pgrep -fc "Runner.Listener" || true)
+if [ "${ACTIVE:-0}" -gt "$MAX_ACTIVE_RUNNERS" ]; then
+  echo "[janitor] ⚠ ${ACTIVE} Runner.Listener processes > ceiling ${MAX_ACTIVE_RUNNERS} — stop the extra runners (systemctl stop actions.runner.<name>)"
+  STATUS=1
+else
+  echo "[janitor] runners active: ${ACTIVE:-0}/${MAX_ACTIVE_RUNNERS} OK"
+fi
+
+echo "[janitor] done status=$STATUS"
+exit "$STATUS"