mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-09 08:22:11 +03:00
e71be03398
* chore(ops): make the runner janitor act on what it can prove, not advise The .113 janitor already knew the rules and had been shouting them into a log nobody reads: on 2026-08-28 12:00Z it reported "10 listeners > ceiling 8" and "disk 85%" — for hours — while 6.7 GB of dead-run leftovers sat on the 12 GB tmpfs (RAM) because its patterns matched neither e2e-build.tar.gz nor next-build/, its 24 h fuse is a day too long for memory, and its _work/_temp base (/home/*/actions-runner*) does not exist on this box (runners live under /opt). Measured while draining the v3.8.50 npm publish (postmortem, Parte III). What changes: - idle is PROVEN before removal, with ONE lsof snapshot filtered to the swept bases (lsof +D per path walked whole trees and took minutes; 460 candidates grepping a re-printed 83k-line string was the other half). 20 s on the box. Without lsof the janitor removes nothing and says why (exit 1). - tmpfs leftovers go after 3 h, disk _work/_temp after 24 h; both overridable. Patterns gain next-build* and e2e-build.tar.gz; /opt/actions-runner* is swept. - zombie builds: a next-build older than 75 min has no job (a real Build step is ~26 min). On 2026-08-27 one ran 70 min after GitHub had declared its job lost, holding 3.6 GB. KillMode=mixed on the units covers systemctl stop/restart; this covers the lost-connection path. - prunes 48 h-old checkouts under _work of runners whose unit is STOPPED — an active runner is never touched. - alerts on memory PSI (full/avg60) and reports the listener ceiling with an omniroute/other breakdown (the box also hosts OmniHeuris and OmniMind). Enforcing the ceiling stays an operator decision (label split), not cron's. - --dry-run prints exactly what it would do and touches nothing; unknown arguments are rejected. Dry-run on the real box: 460 stale omniroute-* test fixtures (930 MB of RAM) it would reclaim, 0 busy, 0 false "removed" lines, 20 s. The unit suite drives the script against a fixture tree with every base redirected; the sweep branch runs where lsof exists (hosted CI images) and the without-lsof contract everywhere. docs/ops/RUNNER_BOX.md reconciled to the measured box: 31 GB (it said 16), ten listeners, the 14 GB next-build ceiling, the KillMode drop-in, and the rule that nothing is cleaned by hand while a runner is busy. * docs(ops): restore the frontmatter fumadocs requires on RUNNER_BOX.md Rewriting the page whole dropped its `title:` frontmatter, and docs/ is compiled into the Next build by fumadocs-mdx — so Build, Fast Production Build and dast-smoke all died with "[MDX] invalid frontmatter in docs/ops/RUNNER_BOX.md". Same block as before, verbatim.
173 lines
8.5 KiB
Bash
Executable File
173 lines
8.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# runner-janitor — self-hosted runner box hygiene for the .113 pool.
|
|
#
|
|
# Runs from cron every 30 min (see docs/ops/RUNNER_BOX.md). It ACTS on what it
|
|
# can prove is safe and ALERTS on what needs an operator decision. Reads of
|
|
# "is this in use?" and the removal happen in the same command, never in two
|
|
# passes: a check-then-delete with a gap is how a live Build job lost its _work
|
|
# on 2026-08-27.
|
|
#
|
|
# Measured box (2026-08-28): 31 GB RAM, 32 cores, 15 GB swap, /tmp = 12 GB
|
|
# tmpfs (RAM!), 188 GB disk. A single `next-build` peaks at ~14 GB, so two
|
|
# concurrent heavy builds saturate the box and three take it down (06:42Z that
|
|
# day: load 56, two jobs lost). The v3.8.50 postmortem (Parte III) has the numbers.
|
|
#
|
|
# What it does, in order:
|
|
# 1) sweep stale artefacts our tooling leaves behind — tmpfs bases after 3 h
|
|
# (they hold RAM), disk _work/_temp bases after 24 h; only names we create,
|
|
# only when no process has them open
|
|
# 2) kill zombie builds: a `next-build` older than ZOMBIE_BUILD_MAX_MIN has no
|
|
# job attached (a real Build step measures ~26 min). On 2026-08-27 one ran
|
|
# 70 minutes after GitHub had already declared its job lost, eating 3.6 GB
|
|
# and a full core set. KillMode=mixed on the units covers systemctl
|
|
# stop/restart; this covers the lost-connection path.
|
|
# 3) prune 48 h-old checkouts under _work of runners whose unit is INACTIVE
|
|
# (stopped runners cannot be mid-job; active ones are never touched)
|
|
# 4) alert: root disk >= DISK_ALERT_PCT, memory PSI full/avg60 >= threshold,
|
|
# Runner.Listener count above the ceiling (with a per-project breakdown —
|
|
# the box also hosts OmniHeuris and OmniMind runners)
|
|
#
|
|
# Usage: runner-janitor.sh [--dry-run] [--help]
|
|
# Exit codes: 0 healthy · 1 attention needed (printed to stdout for the log).
|
|
set -euo pipefail
|
|
|
|
DRY_RUN=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--dry-run) DRY_RUN=1 ;;
|
|
-h|--help)
|
|
sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'
|
|
exit 0 ;;
|
|
*) echo "unknown argument: $arg" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
MAX_ACTIVE_RUNNERS="${MAX_ACTIVE_RUNNERS:-8}"
|
|
DISK_ALERT_PCT="${DISK_ALERT_PCT:-85}"
|
|
TMPFS_MAX_AGE_HOURS="${TMPFS_MAX_AGE_HOURS:-3}"
|
|
WORK_TEMP_MAX_AGE_HOURS="${WORK_TEMP_MAX_AGE_HOURS:-24}"
|
|
WORK_CHECKOUT_MAX_AGE_HOURS="${WORK_CHECKOUT_MAX_AGE_HOURS:-48}"
|
|
ZOMBIE_BUILD_MAX_MIN="${ZOMBIE_BUILD_MAX_MIN:-75}"
|
|
ZOMBIE_BUILD_COMM="${ZOMBIE_BUILD_COMM:-next-build}"
|
|
PSI_FULL_AVG60_ALERT="${PSI_FULL_AVG60_ALERT:-10}"
|
|
# Overridable so the unit test can point everything at a fixture tree.
|
|
JANITOR_TMP_BASES="${JANITOR_TMP_BASES-/tmp}"
|
|
JANITOR_WORK_TEMP_BASES="${JANITOR_WORK_TEMP_BASES-/opt/actions-runner*/_work/_temp /home/*/actions-runner*/_work/_temp}"
|
|
JANITOR_RUNNER_DIRS="${JANITOR_RUNNER_DIRS-/opt/actions-runner*}"
|
|
JANITOR_PSI_FILE="${JANITOR_PSI_FILE:-/proc/pressure/memory}"
|
|
JANITOR_DF_PATH="${JANITOR_DF_PATH:-/}"
|
|
|
|
STATUS=0
|
|
say() { echo "[janitor] $*"; }
|
|
|
|
# "Is anything using this?" — ONE snapshot of every open path on the box
|
|
# (lsof -Fn), then a prefix match per candidate. `lsof +D <dir>` walks the whole
|
|
# tree instead and took minutes on a 5 GB leftover — unusable from cron. An
|
|
# absent lsof means "cannot prove idle": the sweep keeps the path and says so.
|
|
LSOF_BIN="${JANITOR_LSOF:-lsof}"
|
|
have_busy_tools() { command -v "$LSOF_BIN" >/dev/null 2>&1; }
|
|
SNAP=""
|
|
cleanup() { [ -n "$SNAP" ] && rm -f -- "$SNAP"; }
|
|
trap cleanup EXIT
|
|
# One lsof for the whole run (~13 s / 83k lines on the box), kept ONLY for the
|
|
# bases we sweep — 460 candidates grepping a re-printed 83k-line string was the
|
|
# slow part, not lsof itself.
|
|
snapshot_open_paths() {
|
|
have_busy_tools || return 0
|
|
SNAP=$(mktemp) || return 0
|
|
local prefixes="" b
|
|
for b in $JANITOR_TMP_BASES $JANITOR_WORK_TEMP_BASES; do [ -d "$b" ] && prefixes="$prefixes"$'\n'"$b/"; done
|
|
# -F n: one "n<path>" line per open file; -w: no warnings
|
|
"$LSOF_BIN" -w -Fn 2>/dev/null | sed -n 's/^n//p' | grep -F -f <(printf '%s' "$prefixes" | sed '/^$/d') > "$SNAP" 2>/dev/null || true
|
|
}
|
|
is_busy() {
|
|
local p="$1"
|
|
[ -n "$SNAP" ] && [ -s "$SNAP" ] || return 1
|
|
# exact path, or anything beneath it when it is a directory
|
|
grep -qxF -- "$p" "$SNAP" && return 0
|
|
[ -d "$p" ] && grep -qF -- "$p/" "$SNAP"
|
|
}
|
|
|
|
# sweep <base> <max-age-minutes>: only names our tooling creates, never through
|
|
# a symlinked base, never across a filesystem, and remove+check in one step.
|
|
sweep() {
|
|
local base="$1" max_min="$2" p
|
|
[ -d "$base" ] || return 0
|
|
[ -L "$base" ] && { say "skip symlinked base: $base"; return 0; }
|
|
while IFS= read -r -d '' p; do
|
|
if ! have_busy_tools; then say "cannot prove idle (lsof missing — apt install lsof), kept: $p"; STATUS=1; continue; fi
|
|
if is_busy "$p"; then say "busy, kept: $p"; continue; fi
|
|
if [ "$DRY_RUN" -eq 1 ]; then say "would remove ($(( max_min / 60 ))h+): $p"; else rm -rf -- "$p" && say "removed ($(( max_min / 60 ))h+): $p"; fi
|
|
done < <(find -P "$base" -xdev -mindepth 1 -maxdepth 1 \
|
|
\( -name 'runner-*' -o -name 'omniroute-*' -o -name 'next-build*' -o -name 'e2e-build.tar.gz' \) \
|
|
! -type l -mmin "+$max_min" -print0 2>/dev/null || true)
|
|
}
|
|
|
|
say "$(date -u +%FT%TZ) start${DRY_RUN:+ (dry-run=$DRY_RUN)} busy-tools=$(have_busy_tools && echo ok || echo MISSING)"
|
|
|
|
# 1) stale artefacts — tmpfs is RAM, so it gets the short fuse
|
|
snapshot_open_paths
|
|
for base in $JANITOR_TMP_BASES; do sweep "$base" $(( TMPFS_MAX_AGE_HOURS * 60 )); done
|
|
for base in $JANITOR_WORK_TEMP_BASES; do sweep "$base" $(( WORK_TEMP_MAX_AGE_HOURS * 60 )); done
|
|
say "stale temp sweep done"
|
|
|
|
# 2) zombie builds
|
|
ZOMBIES=0
|
|
while read -r pid etimes comm; do
|
|
[ -n "${pid:-}" ] || continue
|
|
if [ "$etimes" -gt $(( ZOMBIE_BUILD_MAX_MIN * 60 )) ]; then
|
|
say "⚠ zombie build pid=$pid comm=$comm age=$(( etimes / 60 ))min > ${ZOMBIE_BUILD_MAX_MIN}min — no job runs this long"
|
|
if [ "$DRY_RUN" -eq 1 ]; then say "[dry-run] would: kill -TERM $pid (then -KILL)"; else
|
|
kill -TERM "$pid" 2>/dev/null || true; sleep 10
|
|
kill -0 "$pid" 2>/dev/null && { kill -KILL "$pid" 2>/dev/null || true; say " needed SIGKILL"; }
|
|
fi
|
|
ZOMBIES=$(( ZOMBIES + 1 )); STATUS=1
|
|
fi
|
|
done < <(ps -eo pid=,etimes=,comm= 2>/dev/null | awk -v c="$ZOMBIE_BUILD_COMM" '$3 ~ ("^" c) {print $1, $2, $3}' || true)
|
|
say "zombie builds: $ZOMBIES"
|
|
|
|
# 3) old checkouts of STOPPED runners
|
|
for d in $JANITOR_RUNNER_DIRS; do
|
|
[ -d "$d" ] && [ -f "$d/.runner" ] || continue
|
|
agent=$(grep -o '"agentName": *"[^"]*"' "$d/.runner" 2>/dev/null | sed 's/.*"\([^"]*\)"$/\1/')
|
|
[ -n "$agent" ] || continue
|
|
unit=$(systemctl list-units --plain --no-legend "actions.runner.*.${agent}.service" 2>/dev/null | awk 'NR==1{print $1}')
|
|
[ -n "$unit" ] || continue
|
|
if systemctl is-active --quiet "$unit"; then continue; fi
|
|
while IFS= read -r -d '' co; do
|
|
if [ "$DRY_RUN" -eq 1 ]; then say "would prune checkout of stopped runner $agent: $co"; else rm -rf -- "$co" && say "pruned checkout of stopped runner $agent: $co"; fi
|
|
done < <(find -P "$d/_work" -xdev -mindepth 2 -maxdepth 2 -type d -mmin "+$(( WORK_CHECKOUT_MAX_AGE_HOURS * 60 ))" -print0 2>/dev/null || true)
|
|
done
|
|
|
|
# 4a) disk
|
|
USAGE=$(df --output=pcent "$JANITOR_DF_PATH" 2>/dev/null | tail -1 | tr -dc '0-9')
|
|
if [ "${USAGE:-0}" -ge "$DISK_ALERT_PCT" ]; then
|
|
say "⚠ ROOT DISK ${USAGE}% >= ${DISK_ALERT_PCT}% — clean before the next heavy run"; STATUS=1
|
|
else
|
|
say "disk ${USAGE:-?}% OK"
|
|
fi
|
|
|
|
# 4b) memory pressure (PSI) — the box swapped its way through the v3.8.50 publish
|
|
if [ -r "$JANITOR_PSI_FILE" ]; then
|
|
FULL60=$(awk '/^full/ {for(i=1;i<=NF;i++) if ($i ~ /^avg60=/) {sub("avg60=","",$i); print $i}}' "$JANITOR_PSI_FILE" 2>/dev/null || echo "")
|
|
if [ -n "$FULL60" ] && awk -v v="$FULL60" -v t="$PSI_FULL_AVG60_ALERT" 'BEGIN{exit !(v+0 >= t+0)}'; then
|
|
say "⚠ MEMORY PRESSURE psi full/avg60=${FULL60}% >= ${PSI_FULL_AVG60_ALERT}% — too many heavy jobs at once"; STATUS=1
|
|
else
|
|
say "memory psi full/avg60=${FULL60:-n/a}% OK"
|
|
fi
|
|
fi
|
|
|
|
# 4c) concurrency ceiling — alert with a breakdown; the fix is fewer/labelled
|
|
# runners (an operator decision), not killing listeners from cron.
|
|
ACTIVE=$(pgrep -fc "Runner.Listener" || true)
|
|
OMNI=$(pgrep -fc "actions-runner-omniroute[^ ]*/bin[^ ]*/Runner.Listener" || true)
|
|
if [ "${ACTIVE:-0}" -gt "$MAX_ACTIVE_RUNNERS" ]; then
|
|
say "⚠ ${ACTIVE} Runner.Listener processes (omniroute=${OMNI:-0}, other=$(( ${ACTIVE:-0} - ${OMNI:-0} ))) > ceiling ${MAX_ACTIVE_RUNNERS} — stop idle extras: systemctl stop <unit> only when it has no Runner.Worker child"
|
|
STATUS=1
|
|
else
|
|
say "runners active: ${ACTIVE:-0}/${MAX_ACTIVE_RUNNERS} (omniroute=${OMNI:-0}) OK"
|
|
fi
|
|
|
|
say "done status=$STATUS"
|
|
exit "$STATUS"
|