From 0ddb47228b30cda4286afc85860072f213ae2eb4 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma <25951435+RaviTharuma@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:42:26 +0200 Subject: [PATCH] fix(monitoring): expose failed connection ids on credentialHealth (#12876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validado numa worktree combinada com a onda de dashboard/monitoring desta leva sobre `release/v3.8.51`: typecheck:core limpo, check-api-typecheck OK (289), check-file-size OK após rebaseline, 130/131 nos testes focados — a falha restante é asserção de tempo de parede sob carga, verde 6/6 isolada. Um health que diz "falhou" sem dizer **qual** conexão obriga o operador a cruzar logs para achar o óbvio. Expor os ids das que falharam é o que transforma o endpoint em ferramenta de diagnóstico. --- ...monitoring-health-failed-connection-ids.md | 1 + docs/ops/MONITORING_GUIDE.md | 50 +++++++++---- docs/reference/API_REFERENCE.md | 18 ++--- src/lib/credentialHealth/cache.ts | 63 ++++++++++++++++- src/lib/monitoring/observability.ts | 52 +++++++++++++- ...onitoring-health-cached-credential.test.ts | 70 +++++++++++++++++++ tests/unit/observability-payloads.test.ts | 63 +++++++++-------- 7 files changed, 260 insertions(+), 57 deletions(-) create mode 100644 changelog.d/fixes/monitoring-health-failed-connection-ids.md diff --git a/changelog.d/fixes/monitoring-health-failed-connection-ids.md b/changelog.d/fixes/monitoring-health-failed-connection-ids.md new file mode 100644 index 0000000000..eaa7ff3cc8 --- /dev/null +++ b/changelog.d/fixes/monitoring-health-failed-connection-ids.md @@ -0,0 +1 @@ +- **fix(monitoring):** `GET /api/monitoring/health` `credentialHealth` now includes a bounded `failedConnections` list (`connectionId`, `status`, sanitized `lastError`) when the probe-cache gauge `failed>0`, plus `source: probe-cache` and a cheap `staleDbNonOkCount` for sticky SQLite `test_status` on active rows. Documents that the live gauge is not `provider_connections.test_status`. diff --git a/docs/ops/MONITORING_GUIDE.md b/docs/ops/MONITORING_GUIDE.md index 6543b95f06..6b260fbb7f 100644 --- a/docs/ops/MONITORING_GUIDE.md +++ b/docs/ops/MONITORING_GUIDE.md @@ -105,10 +105,10 @@ Per-combo: OmniRoute exposes **two** HTTP health surfaces. They are not interchangeable for orchestrators. -| Path | Purpose | Weight | Use for | -| --- | --- | --- | --- | -| `GET /healthz` | Lifecycle liveness/readiness (`ok` / `starting` / `stopping`) | Trivial (phase flag only) | Kubernetes **readiness**; soft **liveness** if you must use HTTP | -| `GET /api/monitoring/health` | Deep system + provider summary (DB, heap, catalog counts, …) | Heavy (sync DB / monitoring work) | Dashboards, blackbox deep checks, Docker’s built-in healthcheck | +| Path | Purpose | Weight | Use for | +| ---------------------------- | ------------------------------------------------------------- | --------------------------------- | ---------------------------------------------------------------- | +| `GET /healthz` | Lifecycle liveness/readiness (`ok` / `starting` / `stopping`) | Trivial (phase flag only) | Kubernetes **readiness**; soft **liveness** if you must use HTTP | +| `GET /api/monitoring/health` | Deep system + provider summary (DB, heap, catalog counts, …) | Heavy (sync DB / monitoring work) | Dashboards, blackbox deep checks, Docker’s built-in healthcheck | > **Note:** Provider health matrices, autopilot issues, quota monitors, token health, and latency detail beyond `/api/monitoring/health` are available via the **MCP tool** `observability_snapshot` or the **dashboard** pages — there are no dedicated REST routes for those. @@ -155,16 +155,41 @@ Response: } ``` +#### `credentialHealth`: probe-cache vs SQLite `test_status` + +`GET /api/monitoring/health` → `credentialHealth` is the **in-memory probe-cache +gauge**, not a live dump of `provider_connections.test_status`. After #12532 the +request path reads `getCachedCredentialHealthSummary()` only; background probes +refresh the cache off the event loop. + +| Layer | Where | What it means | +| ------------------------ | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Probe-cache gauge | `credentialHealth.total` / `healthy` / `failed` / `unknown` / `stale` | Last credential-health probe results still held in process memory. `source` is always `probe-cache`. | +| Failed connection detail | `credentialHealth.failedConnections` | Present **only when `failed > 0`**. Bounded list of cache rows with `status=error` (`connectionId`, `status`, sanitized `lastError` / `lastErrorType`). `failedOmitted` is set when the list was capped. | +| SQLite sticky status | `credentialHealth.staleDbNonOkCount` | Count of **active** (`is_active=1`) connection rows whose persisted `test_status` is a known non-ok (`error`, `expired`, `credits_exhausted`, `banned`, `deactivated`, `unavailable`). | + +The two layers can disagree on purpose: + +- Gauge `failed=0` while `staleDbNonOkCount>0` — SQLite still has a sticky + `test_status` (for example `expired` or `credits_exhausted`) that the latest + probe-cache snapshot does not count as `status=error`. +- Gauge `failed>0` while SQLite looks healthy — a recent probe failed and is + cached; the DB row has not been updated, or was later cleared. + +Do not alert solely on `provider_connections.test_status` when scraping this +endpoint. Use `failed` + `failedConnections` for live probe failures, and +`staleDbNonOkCount` when you need the persisted sticky-status count. + ### Kubernetes probe recommendations OmniRoute is a **single Node process** (one event loop). Stock Docker `HEALTHCHECK` targets lightweight `/healthz`. `/api/monitoring/health` is **too heavy** for kubelet liveness intervals. -| Probe | Recommended target | Notes | -| --- | --- | --- | -| **Startup** | HTTP `GET /healthz` with a long `failureThreshold` (or large `startPeriod`) | Cold start + SQLite migration can exceed a few seconds | -| **Readiness** | HTTP `GET /healthz` | Lifecycle `ok` / `starting` / `stopping` (200 vs 503). Still flaps if the loop is CPU-blocked. A **200 in multiple seconds is not healthy** (#10303) — it means the event loop was starved before the 3-byte handler ran | -| **Liveness** | HTTP `GET /livez`, **or TCP** on the main service port (`PORT`, default `20128`) | `/livez` is process-alive only (always 200 if the handler runs). It still shares the event loop — busy ≠ dead, and it does not detect event-loop starvation (#10303) any better than TCP does. Prefer **TCP** if HTTP probes time out under catalog/compression load; do **not** kill the pod on short event-loop stalls either way | -| **Deep health** | `GET /api/monitoring/health` from an external checker | Not for kubelet `livenessProbe` / tight `readinessProbe` | +| Probe | Recommended target | Notes | +| --------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Startup** | HTTP `GET /healthz` with a long `failureThreshold` (or large `startPeriod`) | Cold start + SQLite migration can exceed a few seconds | +| **Readiness** | HTTP `GET /healthz` | Lifecycle `ok` / `starting` / `stopping` (200 vs 503). Still flaps if the loop is CPU-blocked. A **200 in multiple seconds is not healthy** (#10303) — it means the event loop was starved before the 3-byte handler ran | +| **Liveness** | HTTP `GET /livez`, **or TCP** on the main service port (`PORT`, default `20128`) | `/livez` is process-alive only (always 200 if the handler runs). It still shares the event loop — busy ≠ dead, and it does not detect event-loop starvation (#10303) any better than TCP does. Prefer **TCP** if HTTP probes time out under catalog/compression load; do **not** kill the pod on short event-loop stalls either way | +| **Deep health** | `GET /api/monitoring/health` from an external checker | Not for kubelet `livenessProbe` / tight `readinessProbe` | Example shape (adjust thresholds to your cold-start and compression load): @@ -202,7 +227,6 @@ livenessProbe: Related: [#10052](https://github.com/diegosouzapw/OmniRoute/issues/10052) (probes while the event loop is busy), [#9685](https://github.com/diegosouzapw/OmniRoute/issues/9685) / [#10055](https://github.com/diegosouzapw/OmniRoute/pull/10055) (catalog pricing hog), [#10117](https://github.com/diegosouzapw/OmniRoute/issues/10117) (compression token-count hog). - ### Optional request-path work (memory, skills, token refresh) Memory extraction, skills injection, and OAuth token refresh share the **main Node event loop** with `/healthz`. They are dashboard-toggle features (`memoryEnabled`, `skillsEnabled`), not a worker pool. See [Environment — event-loop cost](../reference/ENVIRONMENT.md#event-loop-cost-of-memory-skills-and-token-refresh-10349). @@ -344,9 +368,7 @@ The MCP tool `observability_snapshot` returns a **complete system snapshot** for "ageMs": 109 } ], - "quotaMonitors": { - /* see above */ - }, + "quotaMonitors": {/* see above */}, "uptime": 12345, "version": "3.8.16" } diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index bb8752691c..51f1d040d1 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -881,15 +881,15 @@ ordinary inference API keys. Credential families, scopes, and curl examples: ### Monitoring -| Endpoint | Method | Description | -| ------------------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/api/sessions` | GET | Active session tracking | -| `/api/rate-limits` | GET | Per-account rate limits | -| `/api/monitoring/health` | GET | Health check + provider summary (`catalogCount`, `configuredCount`, `activeCount`, `monitoredCount`) | -| `/api/cache/stats` | GET/DELETE | Cache stats / clear | -| `/api/modality-bridge/stats` | GET | In-memory `attempts`, successes/`bridged`, failures, cache hits, `totalLatencyMs`, `latencySamples`, sample-denominated `averageLatencyMs`, and last-use time (reset on restart; management auth) | -| `/api/modality-bridge/video/runtime` | GET | Strict trusted-loopback check before management auth/probe; sanitized FFmpeg/ffprobe availability and versions (no-store) | -| `/api/modality-bridge/video/extract` | POST | Internal authenticated trusted-loopback byte broker; 50 MiB input, bounded queue/32 MiB output, `503` capacity, `499` disconnect, `504` deadline; not a public upload API | +| Endpoint | Method | Description | +| ------------------------------------ | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/api/sessions` | GET | Active session tracking | +| `/api/rate-limits` | GET | Per-account rate limits | +| `/api/monitoring/health` | GET | Health check + provider summary (`catalogCount`, `configuredCount`, `activeCount`, `monitoredCount`). Management view includes `credentialHealth`: probe-cache scalars, `failedConnections` when `failed>0`, and `staleDbNonOkCount` (SQLite sticky `test_status`, not the gauge). See [MONITORING_GUIDE.md](../ops/MONITORING_GUIDE.md#credentialhealth-probe-cache-vs-sqlite-test_status). | +| `/api/cache/stats` | GET/DELETE | Cache stats / clear | +| `/api/modality-bridge/stats` | GET | In-memory `attempts`, successes/`bridged`, failures, cache hits, `totalLatencyMs`, `latencySamples`, sample-denominated `averageLatencyMs`, and last-use time (reset on restart; management auth) | +| `/api/modality-bridge/video/runtime` | GET | Strict trusted-loopback check before management auth/probe; sanitized FFmpeg/ffprobe availability and versions (no-store) | +| `/api/modality-bridge/video/extract` | POST | Internal authenticated trusted-loopback byte broker; 50 MiB input, bounded queue/32 MiB output, `503` capacity, `499` disconnect, `504` deadline; not a public upload API | ### Backup & Export/Import diff --git a/src/lib/credentialHealth/cache.ts b/src/lib/credentialHealth/cache.ts index 85bd9431c9..dba8d5428f 100644 --- a/src/lib/credentialHealth/cache.ts +++ b/src/lib/credentialHealth/cache.ts @@ -8,6 +8,7 @@ * Tracks testStatus, lastError, lastTested per connectionId with * configurable TTL. Auto-expiry on read for stale entries. */ +import { redactSecrets } from "@/shared/utils/logRedaction"; export interface CredentialHealthStatus { connectionId: string; @@ -35,6 +36,9 @@ export interface CredentialCacheEntry { const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes const STALE_THRESHOLD_MS = 10 * 60 * 1000; // 10 minutes — considered stale const MAX_ENTRIES = 500; +/** Bound the monitoring-health failed list so huge fleets stay scrape-safe. */ +export const CREDENTIAL_HEALTH_FAILED_LIST_CAP = 32; +const FAILED_ERROR_MAX_LEN = 200; // ── State (globalThis singleton) ────────────────────────────────────────── @@ -175,12 +179,38 @@ export function getAllCredentialHealth(): Record return result; } +export interface CredentialHealthFailedConnection { + connectionId: string; + status: "error"; + lastError?: string; + lastErrorType?: string; +} + export interface CredentialHealthSummary { total: number; healthy: number; failed: number; unknown: number; stale: number; + /** + * Probe-cache rows with status=error. Present only when failed>0. + * Capped at CREDENTIAL_HEALTH_FAILED_LIST_CAP; see failedOmitted. + */ + failedConnections?: CredentialHealthFailedConnection[]; + /** Failed rows omitted because the list was capped. */ + failedOmitted?: number; +} + +/** Sanitize a cached lastError for the public monitoring health payload. */ +export function sanitizeCredentialHealthLastError(raw: string | undefined): string | undefined { + if (!raw) return undefined; + const collapsed = raw.replace(/\s+/g, " ").trim(); + if (!collapsed) return undefined; + const redacted = redactSecrets(collapsed); + if (!redacted) return undefined; + return redacted.length > FAILED_ERROR_MAX_LEN + ? `${redacted.slice(0, FAILED_ERROR_MAX_LEN)}...` + : redacted; } /** @@ -198,18 +228,41 @@ export function getCachedCredentialHealthSummary(): CredentialHealthSummary { let failed = 0; let unknown = 0; let stale = 0; + const failedEntries: Array = []; for (const entry of state.cache.values()) { total += 1; if (entry.status.status === "active") healthy += 1; - else if (entry.status.status === "error") failed += 1; - else unknown += 1; + else if (entry.status.status === "error") { + failed += 1; + const lastError = sanitizeCredentialHealthLastError(entry.status.lastError); + const lastErrorType = + typeof entry.status.lastErrorType === "string" && entry.status.lastErrorType.trim() + ? entry.status.lastErrorType.trim() + : undefined; + failedEntries.push({ + connectionId: entry.status.connectionId, + status: "error", + ...(lastError ? { lastError } : {}), + ...(lastErrorType ? { lastErrorType } : {}), + lastTestedMs: entry.status.lastTested.getTime(), + }); + } else unknown += 1; if (now - entry.status.lastTested.getTime() > STALE_THRESHOLD_MS || now > entry.expiresAt) { stale += 1; } } - return { total, healthy, failed, unknown, stale }; + const summary: CredentialHealthSummary = { total, healthy, failed, unknown, stale }; + if (failed > 0) { + failedEntries.sort((left, right) => right.lastTestedMs - left.lastTestedMs); + const omitted = Math.max(0, failedEntries.length - CREDENTIAL_HEALTH_FAILED_LIST_CAP); + summary.failedConnections = failedEntries + .slice(0, CREDENTIAL_HEALTH_FAILED_LIST_CAP) + .map(({ lastTestedMs: _lastTestedMs, ...row }) => row); + if (omitted > 0) summary.failedOmitted = omitted; + } + return summary; } /** @@ -235,6 +288,8 @@ export function __test_putCredentialHealth(entry: { status: "active" | "error" | "unknown"; lastTested: Date; expiresAt?: number; + lastError?: string; + lastErrorType?: string; }): void { const state = getCacheState(); state.cache.set(entry.connectionId, { @@ -243,6 +298,8 @@ export function __test_putCredentialHealth(entry: { provider: entry.provider, status: entry.status, lastTested: entry.lastTested, + lastError: entry.lastError, + lastErrorType: entry.lastErrorType, consecutiveFailures: 0, }, expiresAt: entry.expiresAt ?? Date.now() + DEFAULT_TTL_MS, diff --git a/src/lib/monitoring/observability.ts b/src/lib/monitoring/observability.ts index 91854bde4a..6ccac51388 100644 --- a/src/lib/monitoring/observability.ts +++ b/src/lib/monitoring/observability.ts @@ -220,6 +220,7 @@ interface BuildHealthPayloadOptions { id?: string; provider?: string; isActive?: boolean | null; + testStatus?: string | null; rateLimitedUntil?: unknown; providerSpecificData?: Readonly> | null; }>; @@ -239,6 +240,13 @@ interface BuildHealthPayloadOptions { failed: number; unknown: number; stale: number; + failedConnections?: Array<{ + connectionId: string; + status: "error"; + lastError?: string; + lastErrorType?: string; + }>; + failedOmitted?: number; }; /** Optional injected public adaptive-admission snapshot; projected, never raw-spread. */ adaptiveAdmission?: AdaptiveAdmissionPublicSnapshot | null; @@ -252,6 +260,48 @@ function limitMonitors(monitors: QuotaMonitorSnapshot[], maxItems = 8): QuotaMon return monitors.slice(0, maxItems); } +/** + * SQLite `test_status` values that stay sticky on active rows even when the + * in-memory probe-cache gauge reports failed=0 (expired / quota / banned). + */ +const STICKY_DB_NON_OK_TEST_STATUS = new Set([ + "error", + "expired", + "credits_exhausted", + "banned", + "deactivated", + "unavailable", +]); + +/** + * Count is_active=1 (or unset) rows whose persisted test_status is a known + * non-ok. This is a cheap SQLite-layer signal and is not the probe-cache + * `failed` gauge. + */ +export function countStaleDbNonOkConnections( + connections: BuildHealthPayloadOptions["connections"] +): number { + let count = 0; + for (const connection of connections) { + if (connection.isActive === false) continue; + const status = (connection.testStatus ?? "").trim().toLowerCase(); + if (STICKY_DB_NON_OK_TEST_STATUS.has(status)) count += 1; + } + return count; +} + +function projectCredentialHealth( + credentialHealth: BuildHealthPayloadOptions["credentialHealth"], + connections: BuildHealthPayloadOptions["connections"] +) { + if (!credentialHealth) return undefined; + return { + ...credentialHealth, + source: "probe-cache" as const, + staleDbNonOkCount: countStaleDbNonOkConnections(connections), + }; +} + export function buildSessionsSummary({ activeSessions, activeSessionsByKey = {}, @@ -535,7 +585,7 @@ export function buildHealthPayload({ monitors: limitMonitors(quotaMonitorMonitors), }, sessions: buildSessionsSummary({ activeSessions, activeSessionsByKey }), - credentialHealth, // may be undefined if credentialHealth module not loaded + credentialHealth: projectCredentialHealth(credentialHealth, connections), adaptiveAdmission: projectAdaptiveAdmissionSummary(adaptiveAdmission), // #11244: the STRUCTURAL gate (chatBodyAdmission.ts) next to the adaptive one — // distinct key so clients reading `adaptiveAdmission` are untouched. diff --git a/tests/unit/monitoring-health-cached-credential.test.ts b/tests/unit/monitoring-health-cached-credential.test.ts index 91a8b0e127..c78f369f8a 100644 --- a/tests/unit/monitoring-health-cached-credential.test.ts +++ b/tests/unit/monitoring-health-cached-credential.test.ts @@ -20,6 +20,8 @@ const { getCredentialHealthSummary, __test_resetCredentialHealthCache, __test_putCredentialHealth, + CREDENTIAL_HEALTH_FAILED_LIST_CAP, + sanitizeCredentialHealthLastError, } = await import("../../src/lib/credentialHealth/cache.ts"); const { GET, __test_resetMonitoringHealthPayloadCache } = @@ -85,6 +87,9 @@ test("GET /api/monitoring/health returns the stale cached summary immediately", failed: number; unknown: number; stale: number; + source?: string; + staleDbNonOkCount?: number; + failedConnections?: Array<{ connectionId: string; status: string }>; }; }; @@ -95,10 +100,75 @@ test("GET /api/monitoring/health returns the stale cached summary immediately", failed: 1, unknown: 0, stale: 1, + failedConnections: [{ connectionId: "conn-stale-get", status: "error" }], + source: "probe-cache", + staleDbNonOkCount: 0, }); assert.ok(elapsedMs < 2000, `stale summary must return immediately, took ${elapsedMs}ms`); }); +test("getCachedCredentialHealthSummary lists failed connection ids when failed>0", () => { + __test_resetCredentialHealthCache(); + __test_putCredentialHealth({ + connectionId: "conn-ok", + provider: "openai", + status: "active", + lastTested: new Date(), + }); + __test_putCredentialHealth({ + connectionId: "conn-bad", + provider: "anthropic", + status: "error", + lastTested: new Date(), + lastError: "Invalid API key sk-abcdefghijklmnopqrstuvwxyz012345", + lastErrorType: "auth", + }); + + const summary = getCachedCredentialHealthSummary(); + assert.equal(summary.failed, 1); + assert.deepEqual(summary.failedConnections, [ + { + connectionId: "conn-bad", + status: "error", + lastError: "Invalid API key sk-[REDACTED]", + lastErrorType: "auth", + }, + ]); + assert.equal(summary.failedOmitted, undefined); +}); + +test("getCachedCredentialHealthSummary caps the failed connection list", () => { + __test_resetCredentialHealthCache(); + const overflow = 8; + for (let i = 0; i < CREDENTIAL_HEALTH_FAILED_LIST_CAP + overflow; i += 1) { + __test_putCredentialHealth({ + connectionId: `conn-fail-${i}`, + provider: "openai", + status: "error", + lastTested: new Date(Date.now() + i), + lastError: `probe failed ${i}`, + }); + } + + const summary = getCachedCredentialHealthSummary(); + assert.equal(summary.failed, CREDENTIAL_HEALTH_FAILED_LIST_CAP + overflow); + assert.equal(summary.failedConnections?.length, CREDENTIAL_HEALTH_FAILED_LIST_CAP); + assert.equal(summary.failedOmitted, overflow); + assert.equal( + summary.failedConnections?.[0]?.connectionId, + `conn-fail-${CREDENTIAL_HEALTH_FAILED_LIST_CAP + overflow - 1}` + ); +}); + +test("sanitizeCredentialHealthLastError redacts secrets and truncates", () => { + assert.equal(sanitizeCredentialHealthLastError("Bearer abcdefghijklmnopqr"), "Bearer [REDACTED]"); + const long = "x".repeat(400); + const sanitized = sanitizeCredentialHealthLastError(long); + assert.ok(sanitized); + assert.ok(sanitized.length <= 203); + assert.ok(sanitized.endsWith("...")); +}); + test("monitoring health route never imports live credential probes", () => { const source = fs.readFileSync( path.join(process.cwd(), "src/app/api/monitoring/health/route.ts"), diff --git a/tests/unit/observability-payloads.test.ts b/tests/unit/observability-payloads.test.ts index 210308f12d..42d6eec4e4 100644 --- a/tests/unit/observability-payloads.test.ts +++ b/tests/unit/observability-payloads.test.ts @@ -5,9 +5,9 @@ import { buildHealthPayload, buildSessionsSummary, buildTelemetryPayload, + countStaleDbNonOkConnections, projectAdaptiveAdmissionSummary, projectChatAdmissionSummary, - projectWalMaintenanceSummary, } from "../../src/lib/monitoring/observability.ts"; test("buildSessionsSummary returns sticky counts and ordered top sessions", () => { @@ -418,22 +418,25 @@ test("buildHealthPayload projects allowlisted structural chatAdmission fields on assert.equal(projectChatAdmissionSummary(undefined), null); }); -test("buildHealthPayload projects allowlisted walMaintenance fields only", () => { - const state = { - ticks: 4, - busyStreak: 1, - busyTotal: 2, - lastBusyAt: "2026-09-06T10:00:00.000Z", - lastOkAt: "2026-09-06T11:00:00.000Z", - // Internal keys that must never leak into the public payload. - walTimer: { _idleTimeout: 1 }, - retryTimer: null, - } as unknown as import("../../src/lib/monitoring/observability.ts").WalMaintenanceSnapshot; +test("buildHealthPayload marks credentialHealth as probe-cache and counts sticky sqlite status", () => { + assert.equal( + countStaleDbNonOkConnections([ + { id: "a", isActive: true, testStatus: "expired" }, + { id: "b", isActive: true, testStatus: "credits_exhausted" }, + { id: "c", isActive: false, testStatus: "error" }, + { id: "d", isActive: true, testStatus: "active" }, + { id: "e", isActive: true, testStatus: "unknown" }, + ]), + 2 + ); const payload = buildHealthPayload({ - appVersion: "9.9.9", - settings: { setupComplete: false }, - connections: [], + appVersion: "1.2.3", + settings: { setupComplete: true }, + connections: [ + { id: "sticky-expired", provider: "openai", isActive: true, testStatus: "expired" }, + { id: "ok", provider: "anthropic", isActive: true, testStatus: "active" }, + ], circuitBreakers: [], rateLimitStatus: {}, learnedLimits: {}, @@ -450,22 +453,22 @@ test("buildHealthPayload projects allowlisted walMaintenance fields only", () => }, quotaMonitorMonitors: [], activeSessions: [], - walMaintenance: state, + credentialHealth: { + total: 1, + healthy: 1, + failed: 0, + unknown: 0, + stale: 0, + }, }); - assert.deepEqual(payload.walMaintenance, { - ticks: 4, - busyStreak: 1, - busyTotal: 2, - lastBusyAt: "2026-09-06T10:00:00.000Z", - lastOkAt: "2026-09-06T11:00:00.000Z", + assert.deepEqual(payload.credentialHealth, { + total: 1, + healthy: 1, + failed: 0, + unknown: 0, + stale: 0, + source: "probe-cache", + staleDbNonOkCount: 1, }); - - const json = JSON.stringify(payload); - assert.equal(json.includes("walTimer"), false); - assert.equal(json.includes("retryTimer"), false); - - // Absent / null state projects to null (degraded path parity). - assert.equal(projectWalMaintenanceSummary(null), null); - assert.equal(projectWalMaintenanceSummary(undefined), null); });