From efece42fec7f9a5ae1cb87b1dd26e64d735fb91d Mon Sep 17 00:00:00 2001 From: Ravi Tharuma <25951435+RaviTharuma@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:17:37 +0200 Subject: [PATCH] docs(docker): N independent DATA_DIRs scale out large /v1/responses (#11024) (#11027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⭐5 — Documenta o multiplicador de escala suportado hoje: N processos independentes com DATA_DIR/volume próprios (não replicas>1 numa PVC SQLite única); Redis (QUOTA_STORE_DRIVER=redis) compartilha só contadores de quota. Doc-only, teste de asserção do Docker Guide + ENVIRONMENT. Fecha #11024. --- .../maintenance/11024-n-instance-scale-out.md | 1 + docs/guides/DOCKER_GUIDE.md | 47 ++++++++++++++++++- docs/reference/ENVIRONMENT.md | 2 +- .../11024-n-instance-scale-out-docs.test.ts | 26 ++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 changelog.d/maintenance/11024-n-instance-scale-out.md create mode 100644 tests/unit/11024-n-instance-scale-out-docs.test.ts diff --git a/changelog.d/maintenance/11024-n-instance-scale-out.md b/changelog.d/maintenance/11024-n-instance-scale-out.md new file mode 100644 index 0000000000..82adafe480 --- /dev/null +++ b/changelog.d/maintenance/11024-n-instance-scale-out.md @@ -0,0 +1 @@ +- **docs(docker):** document N independent `DATA_DIR`s as the supported large `/v1/responses` scale-out (one V8 heap ≠ host RAM; do not `replicas>1` on one SQLite file) ([#11024](https://github.com/diegosouzapw/OmniRoute/issues/11024)) — thanks @RaviTharuma diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index 84eddd7384..61d414de27 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -518,7 +518,52 @@ Stock Docker / Kubernetes OmniRoute is **one Node process + one SQLite writer**. **Upgrades:** expect every session to drop. Drain clients if you can; there is no rolling update on default SQLite. Compose `restart: unless-stopped` plus Docker `HEALTHCHECK` will also replace the only process when the container is Unhealthy — same blast radius. -External Postgres / multi-writer HA is **not** a documented stock path. If you need HA, keep a single replica or run a topology the project has tested and documented separately. +External Postgres / multi-writer HA is **not** a documented stock path. If you need HA, keep a single replica or run a topology the project has tested and documented separately. The Postgres/MySQL work lives in [#8075](https://github.com/diegosouzapw/OmniRoute/issues/8075). Until that ships, the only supported way to multiply **large** `/v1/responses` capacity is N independent processes (next section), not `replicas > 1` on one volume. + +## Scale-out: N independent processes + +One Node process is **one V8 heap**. Two overlapping ~3 MiB / ~750k-token coding-agent `POST /v1/responses` (RTK + Caveman) abort that heap at ~12 Gi (`FATAL ERROR: Reached heap limit`) and can OOM a 16 Gi cgroup. See [#7849](https://github.com/diegosouzapw/OmniRoute/issues/7849). Raising `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` on that process reintroduces the abort. Small chats, `/healthz`, `/v1/models`, and MCP are **not** in that cap. + +To go beyond two concurrent **large** jobs **today**: + +| Do | Do not | +| --- | --- | +| Run **N containers/pods**, each with its **own** `DATA_DIR` / volume | Set `replicas > 1` against one SQLite file | +| Keep each instance at 1–2 heavy in-flight and 12–16 Gi cgroup | Give one process 8× RAM and `max=8` | +| Optional: `QUOTA_STORE_DRIVER=redis` + `QUOTA_STORE_REDIS_URL` for **shared quota counters** | Treat Redis as shared SQLite — it is not | +| Duplicate provider secrets into each instance (or accept partitioned dashboards) | Expect one dashboard / one call-log across instances | +| Front with any load balancer; sticky by API key or session is enough | Require a vendor-specific size-aware middleware | + +Hardware: `concurrent_large ≈ N × 2` at ~8–12 Gi heap / ~12–16 Gi cgroup **per instance**. Host RAM must cover `N × cgroup`, not “one 16 Gi pod with N=8.” + +Compose sketch (two heaps, two volumes — not `deploy.replicas: 2`): + +```yaml +services: + omniroute-a: + image: diegosouzapw/omniroute:3.8.49 + environment: + DATA_DIR: /app/data + OMNIROUTE_MEMORY_MB: "12288" + QUOTA_STORE_DRIVER: redis + QUOTA_STORE_REDIS_URL: redis://redis:6379 + volumes: [omniroute-a-data:/app/data] + ports: ["20128:20128"] + omniroute-b: + image: diegosouzapw/omniroute:3.8.49 + environment: + DATA_DIR: /app/data + OMNIROUTE_MEMORY_MB: "12288" + QUOTA_STORE_DRIVER: redis + QUOTA_STORE_REDIS_URL: redis://redis:6379 + volumes: [omniroute-b-data:/app/data] + ports: ["20138:20128"] +volumes: + omniroute-a-data: + omniroute-b-data: +``` + +In-process density (compression off the HTTP isolate) is [#11023](https://github.com/diegosouzapw/OmniRoute/issues/11023). One logical cluster on shared durable state is [#8075](https://github.com/diegosouzapw/OmniRoute/issues/8075). ## Important Notes diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index dbaaea7f57..b3c12467f8 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -202,7 +202,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `MAX_BODY_SIZE_BYTES` | `10485760` (10 MB) | `src/shared/middleware/bodySizeGuard.ts` | Maximum allowed request body size. Rejects payloads exceeding this limit. | | `OMNIROUTE_CHAT_LARGE_BODY_BYTES` | `262144` (256 KB) | `src/shared/middleware/chatBodyAdmission.ts` | Actual request bodies at or above this threshold require an atomic process-local heavyweight admission lease before JSON parsing. | | `OMNIROUTE_CHAT_HARD_MAX_BODY_BYTES` | `52428800` (50 MB) | `src/shared/middleware/chatBodyAdmission.ts` | Chat-route hard cap enforced against bytes read during bounded ingestion, including requests with missing, invalid, or dishonest `Content-Length`; excess receives `413`. | -| `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` | `1` | `src/shared/middleware/chatBodyAdmission.ts` | Maximum heavyweight chat requests admitted concurrently in one process. When capacity is unavailable, OmniRoute returns retryable `503` with `Retry-After`. | +| `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` | `1` | `src/shared/middleware/chatBodyAdmission.ts` | Maximum heavyweight chat requests admitted concurrently in **one process** (one V8 heap). Overload is retryable `503` with `Retry-After`. Two overlapping ~750k-token `/v1/responses` already abort ~12 Gi heaps (#7849); do not raise this to “use the host.” Multiply capacity with **N independent `DATA_DIR`s** (#11024), not `replicas>1` on one SQLite file. | | `OMNIROUTE_CHAT_ADMISSION_HEAP_SHED_RATIO` | `0.75` | `src/shared/middleware/chatBodyAdmission.ts` | Heap-pressure shed ratio (`heapUsed / heap_size_limit`) for the structural admission gate (#10183, #10268). A second concurrent heavyweight request past `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` is only shed with the retryable `503` when the heap is ALSO at or above this ratio; on a healthy heap it is admitted instead. | | `OMNIROUTE_CHAT_ADMISSION_HEALTHY_HEADROOM` | `OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT` (default `1`) | `src/shared/middleware/chatBodyAdmission.ts` | Bounded extra capacity for the healthy-heap fast path above (#10437). Without this bound, every busy-but-healthy-heap request bypassed admission with no ceiling at all — a slow leak or a burst that never quite trips the heap-shed ratio could still pile up unlimited concurrent heavyweight work. Once this many concurrent leases are active through the healthy-heap path, further busy requests fall through to the SAME bounded-wait/shed path used under real heap pressure. `0` disables the bypass entirely. | | `OMNIROUTE_CHAT_HEAVY_MESSAGE_COUNT` | `200` | `src/shared/middleware/chatBodyAdmission.ts` | Message count that classifies a chat request as heavyweight even when its body is below the byte threshold. | diff --git a/tests/unit/11024-n-instance-scale-out-docs.test.ts b/tests/unit/11024-n-instance-scale-out-docs.test.ts new file mode 100644 index 0000000000..0189324e94 --- /dev/null +++ b/tests/unit/11024-n-instance-scale-out-docs.test.ts @@ -0,0 +1,26 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const dockerGuide = readFileSync(new URL("../../docs/guides/DOCKER_GUIDE.md", import.meta.url), "utf8"); +const envDoc = readFileSync(new URL("../../docs/reference/ENVIRONMENT.md", import.meta.url), "utf8"); + +test("DOCKER_GUIDE documents N independent DATA_DIRs as the large-job scale-out (#11024)", () => { + assert.match(dockerGuide, /## Scale-out: N independent processes/); + assert.match(dockerGuide, /DATA_DIR/); + assert.match(dockerGuide, /replicas > 1/); + assert.match(dockerGuide, /QUOTA_STORE_DRIVER=redis/); + assert.match(dockerGuide, /#8075/); + assert.doesNotMatch( + dockerGuide, + /deploy:\s*\n\s*replicas:\s*2/, + "must not show replicas: 2 as the scale-out recipe" + ); +}); + +test("ENVIRONMENT.md points CHAT_MAX_HEAVY_IN_FLIGHT at per-process V8, not host RAM (#11024)", () => { + const row = envDoc.split("\n").find((line) => line.includes("`OMNIROUTE_CHAT_MAX_HEAVY_IN_FLIGHT`")); + assert.ok(row); + assert.match(row, /one process|per process|V8/i); + assert.match(row, /DATA_DIR|#11024/); +});