From 0aede2ef632335ba477c5b68f288f290cd27de60 Mon Sep 17 00:00:00 2001 From: Regis <92858615+Regis-RCR@users.noreply.github.com> Date: Mon, 16 Mar 2026 22:26:37 +0100 Subject: [PATCH] feat(health): background health check for local provider_nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local inference backends (oMLX, Ollama, LM Studio) configured as provider_nodes have no health monitoring. When a local provider is down, OmniRoute waits the full timeout before failing. This adds a background health check that polls local provider_nodes: - GET /models with 5s timeout for each local node (localhost only) - In-memory health cache (no DB migration needed) - Promise.allSettled for parallel checks (one slow node doesn't block) - Exponential backoff on failures: 30s → 60s → 120s → 300s max - Reset to 30s on first success after failure - State transition logging (healthy ↔ unhealthy) - Expose health status via GET /api/monitoring/health (localProviders) - Auto-init on first import (same pattern as tokenHealthCheck) - 401 treated as healthy (server up, auth required) - isNodeHealthy() returns true if never checked (optimistic default) --- src/app/api/monitoring/health/route.ts | 2 + src/lib/localHealthCheck.ts | 218 +++++++++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 src/lib/localHealthCheck.ts diff --git a/src/app/api/monitoring/health/route.ts b/src/app/api/monitoring/health/route.ts index 56641877f6..f15e3bf551 100644 --- a/src/app/api/monitoring/health/route.ts +++ b/src/app/api/monitoring/health/route.ts @@ -18,6 +18,7 @@ export async function GET() { const circuitBreakers = getAllCircuitBreakerStatuses(); const rateLimitStatus = getAllRateLimitStatus(); const lockouts = getAllModelLockouts(); + const { getAllHealthStatuses } = await import("@/lib/localHealthCheck"); // System info const system = { @@ -46,6 +47,7 @@ export async function GET() { timestamp: new Date().toISOString(), system, providerHealth, + localProviders: getAllHealthStatuses(), rateLimitStatus, lockouts, setupComplete: settings?.setupComplete || false, diff --git a/src/lib/localHealthCheck.ts b/src/lib/localHealthCheck.ts new file mode 100644 index 0000000000..6517e05fa2 --- /dev/null +++ b/src/lib/localHealthCheck.ts @@ -0,0 +1,218 @@ +/** + * Local Provider Health Check + * + * Background polling of local provider_nodes (localhost) to detect + * when they are up or down. Uses GET /models with a 5s timeout. + * + * Health status is stored in-memory (no DB migration needed). + * Backoff schedule: 30s → 60s → 120s → 300s max on consecutive failures. + * Resets to 30s on first success after failure. + * + * Uses Promise.allSettled so one slow/down node doesn't block others. + */ + +import { getProviderNodes } from "@/lib/localDb"; + +// ── Types ──────────────────────────────────────────────────────────────── + +export interface HealthStatus { + nodeId: string; + prefix: string; + isHealthy: boolean; + lastCheck: Date; + lastError?: string; + consecutiveFailures: number; + responseTimeMs?: number; +} + +// ── Config ─────────────────────────────────────────────────────────────── + +const BACKOFF_SCHEDULE = [30_000, 60_000, 120_000, 300_000]; +const CHECK_TIMEOUT_MS = 5_000; +const INITIAL_DELAY_MS = 15_000; // Wait for server boot before first sweep +const LOG_PREFIX = "[LocalHealthCheck]"; + +// ── State ──────────────────────────────────────────────────────────────── + +const healthCache = new Map(); +let initialized = false; +let sweepTimer: ReturnType | null = null; + +// ── Helpers ────────────────────────────────────────────────────────────── + +function isLocalhostUrl(baseUrl: string): boolean { + try { + const u = new URL(baseUrl); + // Block credentials in URL to prevent SSRF via user@host (e.g., http://localhost@evil.com) + if (u.username || u.password) return false; + // Note: URL.hostname returns "[::1]" WITH brackets for IPv6 — both forms checked. + // Verified: node -e "new URL('http://[::1]:8080').hostname" → "[::1]" + return ( + u.hostname === "localhost" || + u.hostname === "127.0.0.1" || + u.hostname === "::1" || + u.hostname === "[::1]" + ); + } catch { + return false; + } +} + +function getNextInterval(failures: number): number { + return BACKOFF_SCHEDULE[Math.min(failures, BACKOFF_SCHEDULE.length - 1)]; +} + +// ── Core ───────────────────────────────────────────────────────────────── + +async function checkNode(node: { + id: string; + prefix: string; + baseUrl: string; +}): Promise { + const url = `${node.baseUrl.replace(/\/+$/, "")}/models`; + const start = Date.now(); + const prev = healthCache.get(node.id); + + try { + const res = await fetch(url, { signal: AbortSignal.timeout(CHECK_TIMEOUT_MS) }); + // Consume/cancel response body to free resources + res.body?.cancel().catch(() => {}); + const isHealthy = res.ok || res.status === 401; // 401 = server up but auth required + return { + nodeId: node.id, + prefix: node.prefix, + isHealthy, + lastCheck: new Date(), + consecutiveFailures: isHealthy ? 0 : (prev?.consecutiveFailures ?? 0) + 1, + responseTimeMs: Date.now() - start, + lastError: isHealthy ? undefined : `HTTP ${res.status}`, + }; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Connection failed"; + return { + nodeId: node.id, + prefix: node.prefix, + isHealthy: false, + lastCheck: new Date(), + consecutiveFailures: (prev?.consecutiveFailures ?? 0) + 1, + responseTimeMs: Date.now() - start, + lastError: message, + }; + } +} + +let sweepInProgress = false; + +/** Single sweep: check all local provider_nodes in parallel. */ +export async function sweep(): Promise { + if (sweepInProgress) return; // Prevent concurrent sweeps + sweepInProgress = true; + + try { + let nodes: Array<{ id: string; prefix: string; baseUrl: string }>; + try { + const raw = await getProviderNodes(); + nodes = (Array.isArray(raw) ? raw : []).filter( + (n: Record) => + typeof n.baseUrl === "string" && isLocalhostUrl(n.baseUrl as string) + ) as Array<{ id: string; prefix: string; baseUrl: string }>; + } catch (err) { + console.error(LOG_PREFIX, "Failed to load provider_nodes:", err); + return; + } + + // Prune stale entries for deleted nodes + const currentNodeIds = new Set(nodes.map((n) => n.id)); + for (const key of healthCache.keys()) { + if (!currentNodeIds.has(key)) healthCache.delete(key); + } + + if (nodes.length === 0) return; + + const results = await Promise.allSettled(nodes.map((node) => checkNode(node))); + + for (const result of results) { + if (result.status === "fulfilled") { + const status = result.value; + const prev = healthCache.get(status.nodeId); + + // Log state transitions + if (prev && prev.isHealthy !== status.isHealthy) { + const emoji = status.isHealthy ? "✅" : "❌"; + console.log( + LOG_PREFIX, + `${emoji} ${status.prefix} is now ${status.isHealthy ? "healthy" : "unhealthy"}${status.lastError ? ` (${status.lastError})` : ""} [${status.responseTimeMs}ms]` + ); + } + + healthCache.set(status.nodeId, status); + } + } + } finally { + sweepInProgress = false; + // Schedule next sweep with backoff based on worst-case failure count + scheduleSweep(); + } +} + +function scheduleSweep(): void { + if (!initialized) return; // Don't schedule if stopped + if (sweepTimer) clearTimeout(sweepTimer); + + // Use the maximum consecutive failures across all nodes to determine interval + let maxFailures = 0; + for (const status of healthCache.values()) { + if (status.consecutiveFailures > maxFailures) { + maxFailures = status.consecutiveFailures; + } + } + + const interval = getNextInterval(maxFailures); + sweepTimer = setTimeout(sweep, interval); +} + +// ── Public API ─────────────────────────────────────────────────────────── + +/** Get health status for a specific provider_node. */ +export function getHealthStatus(nodeId: string): HealthStatus | undefined { + return healthCache.get(nodeId); +} + +/** Check if a provider_node is healthy. Returns true if never checked (optimistic). */ +export function isNodeHealthy(nodeId: string): boolean { + const status = healthCache.get(nodeId); + return status?.isHealthy ?? true; +} + +/** Get all health statuses (for monitoring API). */ +export function getAllHealthStatuses(): Record { + return Object.fromEntries(healthCache); +} + +/** Start the health check scheduler (idempotent). */ +export function initLocalHealthCheck(): void { + if (initialized) return; + initialized = true; + + console.log( + LOG_PREFIX, + `Starting local provider health check (initial delay ${INITIAL_DELAY_MS / 1000}s)` + ); + + // Delay first sweep to let the server finish booting + sweepTimer = setTimeout(() => { + sweep().catch((err) => console.error(LOG_PREFIX, "Initial sweep failed:", err)); + }, INITIAL_DELAY_MS); +} + +/** Stop the scheduler (for tests / hot-reload). */ +export function stopLocalHealthCheck(): void { + if (sweepTimer) { + clearTimeout(sweepTimer); + sweepTimer = null; + } + initialized = false; +} + +// Auto-initialize on first import (same pattern as tokenHealthCheck.ts:272) +initLocalHealthCheck();