diff --git a/Dockerfile b/Dockerfile index 848d25c109..a35f57e280 100644 --- a/Dockerfile +++ b/Dockerfile @@ -181,10 +181,23 @@ ENV NODE_OPTIONS="--max-old-space-size=${OMNIROUTE_BUILD_MEMORY_MB}" # workers for page-data collection (31 on a 32-core builder); on memory-tight # hosts 31 workers + webpack's multi-GB heap blow past RAM and a worker dies # with SIGSEGV at teardown ("worker exited with code: null and signal: SIGSEGV"), -# silently leaving no standalone bundle. Next derives the default worker count -# from CIRCLE_NODE_TOTAL (workers = N-1), so N=8 → 7 workers: fast enough while -# fitting comfortably in RAM on any host. (#10060) -ENV CIRCLE_NODE_TOTAL=8 +# silently leaving no standalone bundle. Next derives the worker count from +# CIRCLE_NODE_TOTAL (workers = N-1). (#10060) +# +# Lowered 8 → 3 (7 workers → 2). Every page-data worker inherits NODE_OPTIONS +# above, so the ceiling is per PROCESS, not per build: 7 workers on a 16 GB +# GitHub runner (ubuntu-24.04 / ubuntu-24.04-arm, 4 vCPU) exhausted the host and +# buildkit failed the whole step with `ResourceExhausted: ... cannot allocate +# memory`. The compile phase always finished ("✓ Compiled successfully in +# 4.2min"); the kernel killed the build right after "Collecting page data using +# 7 workers". It was intermittent for a while and went 100% on 2026-08-22, which +# is what a threshold being crossed by ordinary codebase growth looks like. +# tests/unit/docker-build-memory-budget.test.ts does the arithmetic and fails if +# either knob is raised past what a 16 GB runner holds. 2 workers also stops +# oversubscribing the runner's 4 vCPU, which 7 did. Override for a big builder: +# `--build-arg OMNIROUTE_BUILD_WORKERS=8`. +ARG OMNIROUTE_BUILD_WORKERS=3 +ENV CIRCLE_NODE_TOTAL=${OMNIROUTE_BUILD_WORKERS} COPY . ./ RUN --mount=type=cache,id=s/92ca8a61-c1ba-421f-a389-d48ac7258c2d-next-cache,target=/app/.build/next/cache \ diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index b24e7b7f61..5c650df5da 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -219,13 +219,23 @@ docker build --target runner-cli -t omniroute:cli . ### Build-time resources -Two build args control what the `builder` stage costs. They are build-time only — +Three build args control what the `builder` stage costs. They are build-time only — `OMNIROUTE_MEMORY_MB` (below) is a separate, runtime knob. -| Build arg | Default | Effect | -| --------------------------- | ------- | ---------------------------------------------------------------------- | -| `OMNIROUTE_USE_TURBOPACK` | `1` | `0` builds with webpack instead. Lower peak memory, slower. | -| `OMNIROUTE_BUILD_MEMORY_MB` | `4096` | V8 heap ceiling (`--max-old-space-size`) for the spawned `next build`. | +| Build arg | Default | Effect | +| --------------------------- | ------- | ----------------------------------------------------------------------------------- | +| `OMNIROUTE_USE_TURBOPACK` | `1` | `0` builds with webpack instead. Lower peak memory, slower. | +| `OMNIROUTE_BUILD_MEMORY_MB` | `6144` | V8 heap ceiling (`--max-old-space-size`) for the spawned `next build`. | +| `OMNIROUTE_BUILD_WORKERS` | `3` | Feeds `CIRCLE_NODE_TOTAL`; Next derives `workers = N - 1` for page-data collection. | + +`OMNIROUTE_BUILD_WORKERS` is the one to raise on a big builder and the one to +suspect when a constrained build dies **after** `✓ Compiled successfully`. Each +page-data worker is its own process and inherits `NODE_OPTIONS`, so the heap +ceiling is per process, not per build: the default of `3` (→ 2 workers) is sized +for the 16 GB / 4 vCPU GitHub-hosted runners the publish pipeline uses. At `8` +(→ 7 workers) that runner ran out of memory and buildkit failed the step with +`ResourceExhausted: ... cannot allocate memory`. `tests/unit/docker-build-memory-budget.test.ts` +does the arithmetic and fails if either knob outgrows the runner. Turbopack compiles in native Rust memory that lives **outside** the V8 heap, so `OMNIROUTE_BUILD_MEMORY_MB` does not bound it. On a host with a memory ceiling the @@ -268,12 +278,12 @@ The 1 GiB Docker default is a dashboard/light-chat floor, not a production siz Size **cgroup `--memory` above the heap** — native buffers, SQLite, and compression intermediates sit outside V8. -| Workload | `OMNIROUTE_MEMORY_MB` | Container / cgroup | Notes | -| --- | --- | --- | --- | -| Dashboard, one light chat | `1024` (image default) | ≥2 GiB | | -| One coding agent (Claude/Codex/Grok) | `8192` | ≥10 GiB | Typical single-session `/v1/responses` | -| Two concurrent long `/v1/responses` | `10240`–`12288` | ≥12–16 GiB | Measured V8 abort at ~12 GiB heap | -| Three+ concurrent long contexts | do not on one process | serialize / more RAM | Default heavyweight admission is 1 in-flight; raising it without RAM reintroduces the abort | +| Workload | `OMNIROUTE_MEMORY_MB` | Container / cgroup | Notes | +| ------------------------------------ | ---------------------- | -------------------- | ------------------------------------------------------------------------------------------- | +| Dashboard, one light chat | `1024` (image default) | ≥2 GiB | | +| One coding agent (Claude/Codex/Grok) | `8192` | ≥10 GiB | Typical single-session `/v1/responses` | +| Two concurrent long `/v1/responses` | `10240`–`12288` | ≥12–16 GiB | Measured V8 abort at ~12 GiB heap | +| Three+ concurrent long contexts | do not on one process | serialize / more RAM | Default heavyweight admission is 1 in-flight; raising it without RAM reintroduces the abort | `omniroute serve` on bare metal calibrates ~35% of RAM (clamped `[512, 4096]`) when `OMNIROUTE_MEMORY_MB` is **unset**. Docker always sets `1024`, so that calibration never runs in the official image. @@ -287,19 +297,19 @@ docker run -d --name omniroute --restart unless-stopped --stop-timeout 40 \ Beyond the defaults documented in [ENVIRONMENT.md](../reference/ENVIRONMENT.md), the following variables matter most when running under Docker: -| Variable | Purpose | Default | -| ----------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------ | -| `OMNIROUTE_WS_BRIDGE_SECRET` | Shared secret for the WebSocket bridge. **Required in production** — set to a strong random string. | unset (must be provided) | -| `REDIS_URL` | Connection string for the rate limiter / cache backend | `redis://redis:6379` | -| `REDIS_PORT` | Host-side port for the bundled Redis container | `6379` | -| `REDIS_BIND_HOST` | Host interface the bundled Redis port is published on (loopback unless you add AUTH) | `127.0.0.1` | -| `AUTO_UPDATE_HOST_REPO_DIR` | Host path mounted into `cli` profile at `/workspace/omniroute` for self-update workflows | `.` (current directory) | +| Variable | Purpose | Default | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `OMNIROUTE_WS_BRIDGE_SECRET` | Shared secret for the WebSocket bridge. **Required in production** — set to a strong random string. | unset (must be provided) | +| `REDIS_URL` | Connection string for the rate limiter / cache backend | `redis://redis:6379` | +| `REDIS_PORT` | Host-side port for the bundled Redis container | `6379` | +| `REDIS_BIND_HOST` | Host interface the bundled Redis port is published on (loopback unless you add AUTH) | `127.0.0.1` | +| `AUTO_UPDATE_HOST_REPO_DIR` | Host path mounted into `cli` profile at `/workspace/omniroute` for self-update workflows | `.` (current directory) | | `OMNIROUTE_MEMORY_MB` | Runtime Node heap ceiling for the Docker standalone server; overrides the image default above. Coding agents: `8192`+ (see [runtime RAM](#runtime-ram-for-coding-agents)). | `1024` | -| `DASHBOARD_PORT` / `API_PORT` | Override exposed ports for dashboard (20128) and API (20129) | `20128` / `20129` | -| `OMNIROUTE_BASE_PATH` | URL subpath when the app is published behind a reverse proxy (e.g. `/omniroute`) | _(empty = root)_ | -| `NEXT_PUBLIC_BASE_URL` | Public browser origin including the subpath (e.g. `https://host/omniroute`) | unset | -| `PROD_DASHBOARD_PORT` | Host-side dashboard port for `docker-compose.prod.yml` | `20130` | -| `CLIPROXYAPI_PORT` | Host-side port for the `cliproxyapi` sidecar | `8317` | +| `DASHBOARD_PORT` / `API_PORT` | Override exposed ports for dashboard (20128) and API (20129) | `20128` / `20129` | +| `OMNIROUTE_BASE_PATH` | URL subpath when the app is published behind a reverse proxy (e.g. `/omniroute`) | _(empty = root)_ | +| `NEXT_PUBLIC_BASE_URL` | Public browser origin including the subpath (e.g. `https://host/omniroute`) | unset | +| `PROD_DASHBOARD_PORT` | Host-side dashboard port for `docker-compose.prod.yml` | `20130` | +| `CLIPROXYAPI_PORT` | Host-side port for the `cliproxyapi` sidecar | `8317` | ## Reverse Proxy on a Subpath (Traefik / nginx) @@ -361,11 +371,11 @@ intervals. For orchestrators (Kubernetes, Nomad, etc.): -| Probe | Prefer | Avoid | -| --- | --- | --- | -| Liveness | HTTP `GET /livez`, or TCP on the main port (`PORT`, default `20128`) | `/api/monitoring/health` as liveness | -| Readiness | HTTP `GET /healthz` | Tight timeouts that treat event-loop busy as dead | -| Deep / blackbox | `/api/monitoring/health` | — | +| Probe | Prefer | Avoid | +| --------------- | -------------------------------------------------------------------- | ------------------------------------------------- | +| Liveness | HTTP `GET /livez`, or TCP on the main port (`PORT`, default `20128`) | `/api/monitoring/health` as liveness | +| Readiness | HTTP `GET /healthz` | Tight timeouts that treat event-loop busy as dead | +| Deep / blackbox | `/api/monitoring/health` | — | `/healthz` reports process lifecycle (`ok` / `starting` / `stopping`). `/livez` is process-alive only (200 whenever the handler can run; it does not wait for @@ -431,10 +441,10 @@ Endpoint tunnel panels (Cloudflare, Tailscale, ngrok) can be shown or hidden fro ## Image Tags -| Image | Tag | Size | Description | -| ------------------------ | -------- | ------ | --------------------- | +| Image | Tag | Size | Description | +| ------------------------ | -------- | ------ | ---------------------------------------------------- | | `diegosouzapw/omniroute` | `latest` | ~250MB | Highest **published** stable SemVer (not git `main`) | -| `diegosouzapw/omniroute` | `3.8.0` | ~250MB | Pin this class of tag for GitOps | +| `diegosouzapw/omniroute` | `3.8.0` | ~250MB | Pin this class of tag for GitOps | Multi-platform manifest: `linux/amd64` + `linux/arm64` native (Apple Silicon, AWS Graviton, Raspberry Pi). Docker selects the matching architecture automatically; pass `--platform linux/amd64` if you need to force AMD64 emulation on ARM hosts. @@ -442,12 +452,12 @@ Multi-platform manifest: `linux/amd64` + `linux/arm64` native (Apple Silicon, AW OmniRoute publishes separate Docker channels for stable releases, active release-branch testing, and development builds. -| Channel | Source | Mutability | Recommended use | -| ------------------------------- | ----------------------------------- | --------------------------- | ----------------------------------------------------------------------------------------------- | -| `:` / `:-web` | Signed/versioned release | Immutable | Production deployments that pin an exact release | +| Channel | Source | Mutability | Recommended use | +| ------------------------------- | ----------------------------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `:` / `:-web` | Signed/versioned release | Immutable | Production deployments that pin an exact release | | `:latest` / `:latest-web` | Highest **published** stable SemVer | Mutable stable pointer | Follows stable releases **after** a SemVer publish job — does **not** track `main` or unreleased `release/v*` commits | -| `:next` / `:next-web` | Current default `release/v*` branch | Mutable pre-release pointer | Testing fixes that have landed on the active release branch but are not yet in a stable release | -| `:main` / `:main-web` | `main` branch | Mutable development pointer | Development and integration testing only | +| `:next` / `:next-web` | Current default `release/v*` branch | Mutable pre-release pointer | Testing fixes that have landed on the active release branch but are not yet in a stable release | +| `:main` / `:main-web` | `main` branch | Mutable development pointer | Development and integration testing only | #### Using the pre-release channel @@ -491,30 +501,30 @@ A release-branch build can never move `latest`; only an eligible stable semantic **`latest` is not a currency guarantee for git.** Merged fixes on `main` or on the active `release/v*` branch are **not** in `:latest` until a stable SemVer image is published and the publish job promotes `:latest` (same digest as that SemVer). If `latest` looks frozen while GitHub already shows the fix, pull `:next` to test the release branch or wait for the SemVer tag. -| You want | Use | -| --- | --- | -| GitOps / production that must not drift | Pin `:X.Y.Z` (or the image digest) | -| Follow published stables and accept a recreate on each release | `:latest` | -| Test unreleased `release/v*` commits | `:next` (not production) | -| Test `main` | `:main` (not production) | +| You want | Use | +| -------------------------------------------------------------- | ---------------------------------- | +| GitOps / production that must not drift | Pin `:X.Y.Z` (or the image digest) | +| Follow published stables and accept a recreate on each release | `:latest` | +| Test unreleased `release/v*` commits | `:next` (not production) | +| Test `main` | `:main` (not production) | ## Availability: default SQLite is single-replica Stock Docker / Kubernetes OmniRoute is **one Node process + one SQLite writer**. High availability is **not supported** on that topology. -| Constraint | Consequence | -| --- | --- | -| Single writer | Do **not** run multiple replicas against the same SQLite file. That corrupts the DB. | +| Constraint | Consequence | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Single writer | Do **not** run multiple replicas against the same SQLite file. That corrupts the DB. | | Recreate / restart / HEALTHCHECK kill | **Full outage** of in-flight SSE, dashboard sessions, and in-memory state. Every connected client drops. New requests during the empty-endpoint window get a reverse-proxy **`502 Bad Gateway: Unknown error`**, not OmniRoute JSON — clients cannot distinguish this from a provider failure (#11015). | -| Same event loop as `/healthz` | A busy catalog or compression tick can delay probes; a short timeout then restarts the **only** replica. | +| Same event loop as `/healthz` | A busy catalog or compression tick can delay probes; a short timeout then restarts the **only** replica. | **Probe matrix** (see also [Kubernetes probe recommendations](../ops/MONITORING_GUIDE.md#kubernetes-probe-recommendations)): -| Probe | Target | Do not use | -| --- | --- | --- | -| Liveness | TCP on `PORT` (default `20128`), or soft HTTP `/healthz` | `/api/monitoring/health` | -| Readiness | HTTP `GET /healthz` | Tight timeouts that treat event-loop busy as dead | -| Deep / humans | `/api/monitoring/health` | Automated kubelet liveness | +| Probe | Target | Do not use | +| ------------- | -------------------------------------------------------- | ------------------------------------------------- | +| Liveness | TCP on `PORT` (default `20128`), or soft HTTP `/healthz` | `/api/monitoring/health` | +| Readiness | HTTP `GET /healthz` | Tight timeouts that treat event-loop busy as dead | +| Deep / humans | `/api/monitoring/health` | Automated kubelet liveness | **Upgrades:** expect every session to drop. Drain clients if you can; there is no rolling update on default SQLite. Compose `restart: unless-stopped` plus Docker `HEALTHCHECK` will also replace the only process when the container is Unhealthy — same blast radius. @@ -555,13 +565,13 @@ One Node process is **one V8 heap**. Two overlapping ~3 MiB / ~750k-token codi To go beyond two concurrent **large** jobs **today**: -| Do | Do not | -| --- | --- | -| Run **N containers/pods**, each with its **own** `DATA_DIR` / volume | Set `replicas > 1` against one SQLite file | -| Keep each instance at 1–2 heavy in-flight and 12–16 Gi cgroup | Give one process 8× RAM and `max=8` | -| Optional: `QUOTA_STORE_DRIVER=redis` + `QUOTA_STORE_REDIS_URL` for **shared quota counters** | Treat Redis as shared SQLite — it is not | -| Duplicate provider secrets into each instance (or accept partitioned dashboards) | Expect one dashboard / one call-log across instances | -| Front with any load balancer; sticky by API key or session is enough | Require a vendor-specific size-aware middleware | +| Do | Do not | +| -------------------------------------------------------------------------------------------- | ---------------------------------------------------- | +| Run **N containers/pods**, each with its **own** `DATA_DIR` / volume | Set `replicas > 1` against one SQLite file | +| Keep each instance at 1–2 heavy in-flight and 12–16 Gi cgroup | Give one process 8× RAM and `max=8` | +| Optional: `QUOTA_STORE_DRIVER=redis` + `QUOTA_STORE_REDIS_URL` for **shared quota counters** | Treat Redis as shared SQLite — it is not | +| Duplicate provider secrets into each instance (or accept partitioned dashboards) | Expect one dashboard / one call-log across instances | +| Front with any load balancer; sticky by API key or session is enough | Require a vendor-specific size-aware middleware | Hardware: `concurrent_large ≈ N × 2` at ~8–12 Gi heap / ~12–16 Gi cgroup **per instance**. Host RAM must cover `N × cgroup`, not “one 16 Gi pod with N=8.” diff --git a/scripts/check/check-fabricated-docs.mjs b/scripts/check/check-fabricated-docs.mjs index 90efcc9384..e4fc2ac751 100644 --- a/scripts/check/check-fabricated-docs.mjs +++ b/scripts/check/check-fabricated-docs.mjs @@ -93,6 +93,14 @@ const ENV_VAR_ALLOWLIST = new Set([ "DATA_DIR", "REQUIRE_API_KEY", "OMNIROUTE_BUILD_PROFILE", // build-time only + // Docker builder-stage knobs. Both are documented in docs/guides/DOCKER_GUIDE.md + // because they are the two levers for a memory-constrained build host, but + // neither is read through process.env in this repo: OMNIROUTE_BUILD_WORKERS is + // a Dockerfile ARG that only feeds CIRCLE_NODE_TOTAL, and CIRCLE_NODE_TOTAL is + // read by Next itself (node_modules) to size the page-data worker pool. Pinned + // by tests/unit/docker-build-memory-budget.test.ts. + "OMNIROUTE_BUILD_WORKERS", + "CIRCLE_NODE_TOTAL", "OMNIROUTE_BUILD_SHA", "OMNIROUTE_URL", // used by ad-hoc tooling, validated elsewhere "OMNIROUTE_KEY", // ditto diff --git a/tests/unit/docker-build-memory-budget.test.ts b/tests/unit/docker-build-memory-budget.test.ts new file mode 100644 index 0000000000..4c33386eb3 --- /dev/null +++ b/tests/unit/docker-build-memory-budget.test.ts @@ -0,0 +1,79 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +// The Docker publish workflow builds on GitHub-hosted runners (ubuntu-24.04 and +// ubuntu-24.04-arm): 4 vCPU, 16 GB RAM. Every Next page-data worker is its own +// process and inherits NODE_OPTIONS, so the V8 ceiling is per PROCESS: the +// build's worst case is roughly `workers × OMNIROUTE_BUILD_MEMORY_MB`. +// +// With 7 workers × 6144 MB the runner ran out and buildkit failed the step with +// `ResourceExhausted: ... cannot allocate memory`, right after "Collecting page +// data using 7 workers" — every Docker publish since 2026-08-22 23:14 UTC. +// +// This pins the budget so raising either knob has to be a deliberate change +// that re-does the arithmetic, not a one-line bump that silently reds the +// publish pipeline again. + +const RUNNER_MEMORY_MB = 16 * 1024; +// Leave room for buildkit, the snapshotter and page cache. +const HEADROOM_FRACTION = 0.75; +// Planning figure for one page-data worker's peak RSS. It is an INFERENCE, not +// a measurement: 7 workers did not fit in 16 GB alongside the parent, which +// puts the per-worker peak somewhere north of ~1.8 GB. 2.5 GB is that bound +// rounded up, so the budget below stays conservative. If a future build OOMs +// again with a worker count this test accepts, raise this number — do not +// weaken the budget. +const WORKER_PEAK_MB = 2560; + +const dockerfile = readFileSync( + fileURLToPath(new URL("../../Dockerfile", import.meta.url)), + "utf8" +); + +function readArgDefault(name: string): number { + const match = dockerfile.match(new RegExp(`^ARG ${name}=(\\d+)$`, "m")); + assert.ok(match, `Dockerfile no longer declares ARG ${name}`); + return Number(match![1]); +} + +test("the Docker build's worker pool is derived from OMNIROUTE_BUILD_WORKERS", () => { + // assert.ok(boolean), not assert.match — a failing assert.match dumps the + // whole Dockerfile into the report. + assert.ok( + /^ENV CIRCLE_NODE_TOTAL=\$\{OMNIROUTE_BUILD_WORKERS\}$/m.test(dockerfile), + "CIRCLE_NODE_TOTAL must stay wired to the build arg so a big builder can raise it" + ); + assert.ok( + /^ENV NODE_OPTIONS="--max-old-space-size=\$\{OMNIROUTE_BUILD_MEMORY_MB\}"$/m.test(dockerfile), + "the build heap ceiling must stay wired to OMNIROUTE_BUILD_MEMORY_MB" + ); +}); + +test("worker count × per-process heap fits a 16 GB GitHub runner", () => { + const workerPool = readArgDefault("OMNIROUTE_BUILD_WORKERS"); + const heapMb = readArgDefault("OMNIROUTE_BUILD_MEMORY_MB"); + + // Next derives `workers = CIRCLE_NODE_TOTAL - 1`. + const workers = workerPool - 1; + assert.ok(workers >= 1, `CIRCLE_NODE_TOTAL=${workerPool} leaves no build workers`); + + // The parent `next build` process is the one that genuinely needs the raised + // ceiling (the webpack/turbopack production pass, #4076); the workers are + // budgeted at their inferred peak instead. + const worstCaseMb = heapMb + workers * WORKER_PEAK_MB; + const budgetMb = RUNNER_MEMORY_MB * HEADROOM_FRACTION; + assert.ok( + worstCaseMb <= budgetMb, + `parent ${heapMb} MB + ${workers} workers × ${WORKER_PEAK_MB} MB = ${worstCaseMb} MB ` + + `exceeds the ${budgetMb} MB budget on a ${RUNNER_MEMORY_MB} MB runner — the Docker ` + + `publish step dies with "ResourceExhausted: cannot allocate memory" during page-data ` + + `collection` + ); +}); + +test("the worker pool does not oversubscribe the runner's 4 vCPU", () => { + const workers = readArgDefault("OMNIROUTE_BUILD_WORKERS") - 1; + assert.ok(workers <= 4, `${workers} workers oversubscribe a 4 vCPU runner`); +});