From 2acafd9c9edee36ffb7507f30d4726659a2d5902 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma <25951435+RaviTharuma@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:49:51 +0200 Subject: [PATCH] feat(api): add GET /livez as a process-alive probe (#10819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged — locally validated (3/3 focused tests, changelog gate green) after resolving base-drift against #10827's event-loop-lag doc note (both landed today, same MONITORING_GUIDE.md table cell — combined the /livez recommendation with the #10303 lag caveat). Thanks! --- changelog.d/features/10316-livez-endpoint.md | 1 + docs/guides/DOCKER_GUIDE.md | 10 ++-- docs/ops/MONITORING_GUIDE.md | 13 +++-- src/app/livez/route.ts | 27 ++++++++++ tests/unit/livez-route.test.ts | 55 ++++++++++++++++++++ 5 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 changelog.d/features/10316-livez-endpoint.md create mode 100644 src/app/livez/route.ts create mode 100644 tests/unit/livez-route.test.ts diff --git a/changelog.d/features/10316-livez-endpoint.md b/changelog.d/features/10316-livez-endpoint.md new file mode 100644 index 0000000000..01409d7b48 --- /dev/null +++ b/changelog.d/features/10316-livez-endpoint.md @@ -0,0 +1 @@ +- **feat(docker):** add `GET`/`HEAD` `/livez` as a process-alive probe, distinct from `/healthz` readiness ([#10316](https://github.com/diegosouzapw/OmniRoute/issues/10316)) diff --git a/docs/guides/DOCKER_GUIDE.md b/docs/guides/DOCKER_GUIDE.md index fde782dbf9..e1b639bcf8 100644 --- a/docs/guides/DOCKER_GUIDE.md +++ b/docs/guides/DOCKER_GUIDE.md @@ -342,13 +342,15 @@ For orchestrators (Kubernetes, Nomad, etc.): | Probe | Prefer | Avoid | | --- | --- | --- | -| Liveness | TCP on the main port (`PORT`, default `20128`), or soft HTTP `/healthz` | `/api/monitoring/health` as liveness | +| Liveness | HTTP `GET /livez`, or TCP on the main port (`PORT`, default `20128`) | `/api/monitoring/health` as liveness | | Readiness | HTTP `GET /healthz` | Tight timeouts that treat event-loop busy as dead | | Deep / blackbox | `/api/monitoring/health` | — | -`/healthz` only reports process lifecycle (`ok` / `starting` / `stopping`). It still -runs on the same Node event loop as request handling, so CPU-bound catalog or -compression work can delay it — busy ≠ dead. Full probe guidance: +`/healthz` reports process lifecycle (`ok` / `starting` / `stopping`). `/livez` is +process-alive only (200 whenever the handler can run; it does not wait for +readiness). Both still run on the same Node event loop as request handling, so +CPU-bound catalog or compression work can delay them — busy ≠ dead. Prefer TCP +liveness if HTTP probes time out. Full probe guidance: [Monitoring guide — Kubernetes probe recommendations](../ops/MONITORING_GUIDE.md#kubernetes-probe-recommendations). ## Docker Compose with Caddy (HTTPS Auto-TLS) diff --git a/docs/ops/MONITORING_GUIDE.md b/docs/ops/MONITORING_GUIDE.md index da1dc7d39b..6543b95f06 100644 --- a/docs/ops/MONITORING_GUIDE.md +++ b/docs/ops/MONITORING_GUIDE.md @@ -157,13 +157,13 @@ Response: ### Kubernetes probe recommendations -OmniRoute is a **single Node process** (one event loop). Stock Docker `HEALTHCHECK` targets `/api/monitoring/health` — that is **too heavy** for kubelet liveness intervals. +OmniRoute is a **single Node process** (one event loop). Stock Docker `HEALTHCHECK` targets lightweight `/healthz`. `/api/monitoring/health` is **too heavy** for kubelet liveness intervals. | Probe | Recommended target | Notes | | --- | --- | --- | | **Startup** | HTTP `GET /healthz` with a long `failureThreshold` (or large `startPeriod`) | Cold start + SQLite migration can exceed a few seconds | -| **Readiness** | HTTP `GET /healthz` | Remove endpoints while starting/stopping; still flaps if the loop is CPU-blocked. A **200 in multiple seconds is not healthy** (#10303) — it means the event loop was starved before the 3-byte handler ran | -| **Liveness** | **TCP** on the main service port (`PORT`, default `20128`), **or** HTTP `/healthz` with soft thresholds | Do **not** kill the pod on short event-loop stalls; busy ≠ dead | +| **Readiness** | HTTP `GET /healthz` | Lifecycle `ok` / `starting` / `stopping` (200 vs 503). Still flaps if the loop is CPU-blocked. A **200 in multiple seconds is not healthy** (#10303) — it means the event loop was starved before the 3-byte handler ran | +| **Liveness** | HTTP `GET /livez`, **or TCP** on the main service port (`PORT`, default `20128`) | `/livez` is process-alive only (always 200 if the handler runs). It still shares the event loop — busy ≠ dead, and it does not detect event-loop starvation (#10303) any better than TCP does. Prefer **TCP** if HTTP probes time out under catalog/compression load; do **not** kill the pod on short event-loop stalls either way | | **Deep health** | `GET /api/monitoring/health` from an external checker | Not for kubelet `livenessProbe` / tight `readinessProbe` | Example shape (adjust thresholds to your cold-start and compression load): @@ -186,11 +186,16 @@ readinessProbe: timeoutSeconds: 2 failureThreshold: 6 livenessProbe: - tcpSocket: + httpGet: + path: /livez port: http periodSeconds: 10 timeoutSeconds: 3 failureThreshold: 6 + # Under event-loop stall HTTP /livez can still time out. TCP is the + # conservative alternative: + # tcpSocket: + # port: http ``` **Do not** point kubelet **liveness** at `/api/monitoring/health`. That path does real DB/monitoring work and will false-positive under load. diff --git a/src/app/livez/route.ts b/src/app/livez/route.ts new file mode 100644 index 0000000000..d82f2dbee9 --- /dev/null +++ b/src/app/livez/route.ts @@ -0,0 +1,27 @@ +/** + * Process-alive probe. Distinct from /healthz (lifecycle readiness). + * Does not inspect the database, catalog, or providers. Still runs on the + * main Node event loop — busy ≠ dead; prefer TCP liveness under stall. + */ +export const dynamic = "force-dynamic"; + +const LIVE_BODY = "ok\n"; + +function createLiveResponse(method: "GET" | "HEAD"): Response { + return new Response(method === "HEAD" ? null : LIVE_BODY, { + status: 200, + headers: { + "Cache-Control": "no-store", + "Content-Length": String(LIVE_BODY.length), + "Content-Type": "text/plain; charset=utf-8", + }, + }); +} + +export function GET(): Response { + return createLiveResponse("GET"); +} + +export function HEAD(): Response { + return createLiveResponse("HEAD"); +} diff --git a/tests/unit/livez-route.test.ts b/tests/unit/livez-route.test.ts new file mode 100644 index 0000000000..55aa7320f2 --- /dev/null +++ b/tests/unit/livez-route.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import fs from "node:fs"; + +import { + markServerStarting, + markServerReady, + markServerStopping, +} from "../../src/lib/serverLifecycle.ts"; + +const livez = await import("../../src/app/livez/route.ts"); +const healthz = await import("../../src/app/healthz/route.ts"); + +test("/livez is 200 while starting and stopping; /healthz stays lifecycle", async () => { + assert.equal(livez.dynamic, "force-dynamic"); + markServerStarting(); + + const startingLive = await livez.GET(); + assert.equal(startingLive.status, 200); + assert.equal(await startingLive.text(), "ok\n"); + const startingReady = await healthz.GET(); + assert.equal(startingReady.status, 503); + + markServerReady(); + const readyLive = await livez.GET(); + assert.equal(readyLive.status, 200); + assert.equal(readyLive.headers.get("Cache-Control"), "no-store"); + assert.equal(readyLive.headers.get("Content-Type"), "text/plain; charset=utf-8"); + assert.equal(await readyLive.text(), "ok\n"); + + const readyHead = await livez.HEAD(); + assert.equal(readyHead.status, 200); + assert.equal(await readyHead.text(), ""); + + markServerStopping(); + const stoppingLive = await livez.GET(); + assert.equal(stoppingLive.status, 200); + const stoppingReady = await healthz.GET(); + assert.equal(stoppingReady.status, 503); +}); + +test("/livez source does not import monitoring or sqlite helpers", () => { + const source = fs.readFileSync("src/app/livez/route.ts", "utf8"); + assert.equal(/monitoring/i.test(source), false); + assert.equal(/sqlite/i.test(source), false); + assert.equal(source.includes("getServerLifecyclePhase"), false); +}); + +test("/livez is omitted from the centralized auth proxy matcher", () => { + const proxySource = fs.readFileSync("src/proxy.ts", "utf8"); + const matcherBlock = proxySource.match(/matcher:\s*\[([\s\S]*?)\]/)?.[1]; + assert.ok(matcherBlock, "proxy matcher configuration must remain discoverable"); + assert.equal(/["']\/livez/.test(matcherBlock), false); + assert.equal(/["']\/healthz/.test(matcherBlock), false); +});