feat(quota): recuperação proativa de conexões em cooldown (cron heal) [Fase 3 #8] (#4900)

Integrated into release/v3.8.36

.env.example

@@ -551,6 +551,16 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500
 # heuristic in instrumentation-node.ts.  Default: unset (tests skip background).
 #OMNIROUTE_ENABLE_RUNTIME_BACKGROUND_TASKS=1
 
+# Proactive connection-cooldown recovery (#8): re-validates connections whose
+# transient `rate_limited_until` window has elapsed OUTSIDE the request hot path,
+# so the first request after a cooldown does not pay the probe latency. Lazy
+# recovery in getProviderCredentials still applies regardless. Used by:
+# src/lib/quota/connectionRecovery.ts.
+# Tick cadence (ms). Default 60000, floor 5000.
+# OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS=60000
+# Disable the proactive recovery scheduler entirely (default: false).
+# OMNIROUTE_DISABLE_CONNECTION_RECOVERY=false
+
 # Background job interval for budget reset checks (ms).  Default: 600000 (10m).
 # Used by: src/lib/jobs/budgetResetJob.ts.  Floor: 10000.
 #OMNIROUTE_BUDGET_RESET_JOB_INTERVAL_MS=600000

docs/reference/ENVIRONMENT.md

@@ -386,6 +386,8 @@ detection above).
 | `OMNIROUTE_DISABLE_BACKGROUND_SERVICES`         | `false`                                             | `src/instrumentation-node.ts`                               | Disable all background services (sync, pricing, model refresh). Useful for CI/test.                                                                                    |
 | `OMNIROUTE_ENABLE_RUNTIME_BACKGROUND_TASKS`     | _(unset)_                                           | `src/lib/config/runtimeSettings.ts`                         | Force background tasks on under automated test detection. Set `1` to override the test heuristic.                                                                      |
 | `OMNIROUTE_BUDGET_RESET_JOB_INTERVAL_MS`        | `600000`                                            | `src/lib/jobs/budgetResetJob.ts`                            | Budget reset check cadence (ms). Floor `10000`.                                                                                                                        |
+| `OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS`     | `60000`                                             | `src/lib/quota/connectionRecovery.ts`                       | Proactive connection-cooldown recovery cadence (ms): re-validates connections whose transient `rate_limited_until` has elapsed, off the request hot path. Floor `5000`. |
+| `OMNIROUTE_DISABLE_CONNECTION_RECOVERY`         | `false`                                             | `src/lib/quota/connectionRecovery.ts`                       | Disable the proactive connection-cooldown recovery scheduler (lazy recovery in `getProviderCredentials` still applies).                                                |
 | `OMNIROUTE_REASONING_CACHE_CLEANUP_INTERVAL_MS` | `1800000`                                           | `src/lib/jobs/reasoningCacheCleanupJob.ts`                  | Reasoning cache cleanup cadence (ms). Floor `60000`.                                                                                                                   |
 | `OMNIROUTE_CONFIG_HOT_RELOAD_MS`                | `5000`                                              | `src/lib/config/hotReload.ts`                               | Polling interval (ms) for config hot-reload. Lower than `1000` is rejected.                                                                                            |
 | `OMNIROUTE_DISABLE_REDIS_AUTH_CACHE`            | _(enabled)_                                         | `src/lib/db/apiKeys.ts`                                     | Set `1` to bypass the Redis-backed API-key auth cache (forces DB reads).                                                                                               |

src/instrumentation-node.ts

@@ -276,6 +276,18 @@ export async function registerNodejs(): Promise<void> {
       console.warn("[STARTUP] Auto-refresh daemon failed to start (non-fatal):", msg);
     }
 
+    // Proactive connection-cooldown recovery (#8): re-validate connections whose
+    // transient `rate_limited_until` window has elapsed OUTSIDE the request hot
+    // path, so the first request after a cooldown does not pay the probe latency.
+    // Lazy/self-recovery still happens in getProviderCredentials; this front-runs it.
+    try {
+      const { initConnectionRecoveryScheduler } = await import("@/lib/quota/connectionRecovery");
+      initConnectionRecoveryScheduler();
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn("[STARTUP] Connection recovery scheduler failed to start (non-fatal):", msg);
+    }
+
     try {
       // Arena ELO sync: model intelligence from the Arena AI leaderboard, powering the
       // Free Provider Rankings page. On by default; configurable from Dashboard Feature Flags.

src/lib/quota/connectionRecovery.ts

@@ -0,0 +1,303 @@
+/**
+ * connectionRecovery.ts — Proactive recovery of provider connections whose
+ * transient cooldown has elapsed.
+ *
+ * Today the cooldown released by `markAccountUnavailable()` (testStatus
+ * 'unavailable' + a future `rateLimitedUntil`) is recovered LAZILY: a connection
+ * only becomes eligible again when the next real request reads it in
+ * `getProviderCredentials` (src/sse/services/auth.ts). The first request after
+ * the cooldown window therefore pays the latency of re-discovering a healthy
+ * connection.
+ *
+ * Modeled on gpt-load's CronChecker, this module identifies the subset of
+ * connections that are cooling down with an already-elapsed window and clears
+ * their error state OUTSIDE the request hot path, so they are restored before
+ * the next real request arrives.
+ *
+ * This file holds the PURE selection logic (`selectRecoverableConnections`,
+ * `isRecoverableCooldownConnection`) — no DB, no network, time injected — plus a
+ * thin async tick (`runConnectionRecoveryTick`) that wires the helper to the DB.
+ * The tick is NOT auto-started on import; the caller (startup bootstrap) decides
+ * when to schedule it, so importing this module in tests never spawns a timer.
+ */
+
+import { cooldownUntilMs } from "@omniroute/open-sse/services/accountFallback.ts";
+
+/**
+ * The transient-cooldown status written by `markAccountUnavailable()` for a
+ * recoverable failure. Only connections in this status are candidates for
+ * proactive recovery.
+ */
+export const RECOVERABLE_COOLDOWN_STATUS = "unavailable";
+
+/**
+ * Terminal connection statuses that must NEVER be auto-recovered — they stay
+ * unavailable until credentials/settings change or an operator resets them.
+ * Mirrors `isTerminalConnectionStatus` (src/sse/services/auth.ts) and
+ * `TERMINAL_STATUSES` (src/lib/db/providers.ts::clearStaleCrashCooldowns).
+ */
+export const TERMINAL_CONNECTION_STATUSES = new Set<string>([
+  "banned",
+  "expired",
+  "credits_exhausted",
+]);
+
+/** Minimal connection shape needed to decide recoverability. */
+export interface RecoverableConnectionInput {
+  id: string;
+  testStatus?: string | null;
+  rateLimitedUntil?: string | null;
+}
+
+function normalizeStatus(value: string | null | undefined): string {
+  return (value || "").trim().toLowerCase();
+}
+
+/**
+ * True when `rateLimitedUntil` is set and its instant is at or before `nowMs`
+ * (the cooldown window has elapsed). Tolerates ISO strings and numeric-epoch
+ * strings — the `rate_limited_until` TEXT column can hold either (#3954).
+ */
+function hasElapsedCooldown(rateLimitedUntil: string | null | undefined, nowMs: number): boolean {
+  if (!rateLimitedUntil) return false;
+  const ms = cooldownUntilMs(rateLimitedUntil);
+  return Number.isFinite(ms) && ms <= nowMs;
+}
+
+/**
+ * Decide whether a single connection is a proactive-recovery candidate:
+ *   - has a real id, AND
+ *   - testStatus === 'unavailable' (the transient cooldown status), AND
+ *   - rateLimitedUntil is set and already in the past (< nowMs), AND
+ *   - is NOT in a terminal state (banned / expired / credits_exhausted).
+ *
+ * Pure — `nowMs` is injected so callers/tests control the clock.
+ */
+export function isRecoverableCooldownConnection(
+  connection: RecoverableConnectionInput | null | undefined,
+  nowMs: number
+): boolean {
+  if (!connection || typeof connection.id !== "string" || connection.id.length === 0) {
+    return false;
+  }
+  const status = normalizeStatus(connection.testStatus);
+  if (status !== RECOVERABLE_COOLDOWN_STATUS) return false;
+  if (TERMINAL_CONNECTION_STATUSES.has(status)) return false; // defensive; 'unavailable' is never terminal
+  return hasElapsedCooldown(connection.rateLimitedUntil, nowMs);
+}
+
+/**
+ * From a list of connections, return only those whose transient cooldown has
+ * elapsed and are safe to restore. Pure, non-mutating, time injected.
+ */
+export function selectRecoverableConnections<T extends RecoverableConnectionInput>(
+  connections: readonly T[] | null | undefined,
+  nowMs: number
+): T[] {
+  if (!Array.isArray(connections)) return [];
+  return connections.filter((connection) => isRecoverableCooldownConnection(connection, nowMs));
+}
+
+/** Result of one recovery tick (handy for logging / tests of the wiring). */
+export interface ConnectionRecoveryTickResult {
+  scanned: number;
+  recovered: number;
+  recoveredIds: string[];
+}
+
+/**
+ * Run one proactive-recovery pass: load active provider connections, select the
+ * subset whose transient cooldown has elapsed, and clear their error state via
+ * `clearAccountError` so they are eligible again before the next real request.
+ *
+ * Dependencies are injected (default to the real DB / auth modules) so the tick
+ * can be unit-tested without a live database. Best-effort and never throws — a
+ * failure to recover one connection must not abort the others or the scheduler.
+ *
+ * NOTE: not auto-started on import. The startup bootstrap is responsible for
+ * scheduling it (see runConnectionRecoveryTick usage in the report).
+ */
+export async function runConnectionRecoveryTick(
+  deps: {
+    nowMs?: number;
+    loadConnections?: () => Promise<RecoverableConnectionInput[]>;
+    clearConnectionError?: (
+      connectionId: string,
+      current: RecoverableConnectionInput
+    ) => Promise<void>;
+    logger?: { info?: (msg: string) => void; warn?: (msg: string) => void };
+  } = {}
+): Promise<ConnectionRecoveryTickResult> {
+  const nowMs = deps.nowMs ?? Date.now();
+  const result: ConnectionRecoveryTickResult = { scanned: 0, recovered: 0, recoveredIds: [] };
+
+  let connections: RecoverableConnectionInput[];
+  try {
+    const load =
+      deps.loadConnections ??
+      (async () => {
+        // Lazy import keeps this module loadable (and the pure helpers testable)
+        // without a full DB/auth graph.
+        const { getProviderConnections } = await import("@/lib/db/providers");
+        const rows = (await getProviderConnections({ isActive: true })) as Array<{
+          id?: unknown;
+          testStatus?: unknown;
+          rateLimitedUntil?: unknown;
+        }>;
+        return (Array.isArray(rows) ? rows : []).map((row) => ({
+          id: typeof row.id === "string" ? row.id : "",
+          testStatus: typeof row.testStatus === "string" ? row.testStatus : null,
+          rateLimitedUntil:
+            typeof row.rateLimitedUntil === "string" ? row.rateLimitedUntil : null,
+        }));
+      });
+    connections = await load();
+  } catch (err) {
+    deps.logger?.warn?.(
+      `[ConnectionRecovery] failed to load connections: ${err instanceof Error ? err.message : String(err)}`
+    );
+    return result;
+  }
+
+  result.scanned = connections.length;
+  const recoverable = selectRecoverableConnections(connections, nowMs);
+  if (recoverable.length === 0) return result;
+
+  const clear =
+    deps.clearConnectionError ??
+    (async (connectionId: string, current: RecoverableConnectionInput) => {
+      const { clearAccountError } = await import("@/sse/services/auth");
+      await clearAccountError(connectionId, current);
+    });
+
+  for (const connection of recoverable) {
+    try {
+      await clear(connection.id, connection);
+      result.recovered += 1;
+      result.recoveredIds.push(connection.id);
+    } catch (err) {
+      deps.logger?.warn?.(
+        `[ConnectionRecovery] failed to recover ${connection.id.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+
+  if (result.recovered > 0) {
+    deps.logger?.info?.(
+      `[ConnectionRecovery] proactively restored ${result.recovered} connection(s) with elapsed cooldown`
+    );
+  }
+  return result;
+}
+
+// ── Scheduler (opt-out, low frequency) ──────────────────────────────────────
+// Mirrors src/lib/tokenHealthCheck.ts: a globalThis-guarded singleton so HMR /
+// double-import never stacks timers, an unref'd interval so it never holds the
+// process open, and a self-disable in build/test processes. NOT auto-started on
+// import — startup bootstrap calls initConnectionRecoveryScheduler().
+
+const DEFAULT_TICK_MS = 60 * 1000; // re-validate elapsed cooldowns every 60s
+const MIN_TICK_MS = 5 * 1000; // floor to avoid hot-looping if misconfigured
+const RECOVERY_LOG_PREFIX = "[ConnectionRecovery]";
+const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
+
+declare global {
+  var __omnirouteConnRecovery:
+    | { initialized: boolean; interval: ReturnType<typeof setInterval> | null }
+    | undefined;
+}
+
+function getRecoveryState() {
+  if (!globalThis.__omnirouteConnRecovery) {
+    globalThis.__omnirouteConnRecovery = { initialized: false, interval: null };
+  }
+  return globalThis.__omnirouteConnRecovery;
+}
+
+function isEnvFlagEnabled(name: string): boolean {
+  const value = typeof process !== "undefined" ? process.env[name] : undefined;
+  return !!value && TRUE_ENV_VALUES.has(value.trim().toLowerCase());
+}
+
+function isBuildProcess(): boolean {
+  return typeof process !== "undefined" && process.env.NEXT_PHASE === "phase-production-build";
+}
+
+function isAutomatedTestProcess(): boolean {
+  return (
+    typeof process !== "undefined" &&
+    (process.env.NODE_ENV === "test" ||
+      process.env.VITEST !== undefined ||
+      process.argv.some((arg) => arg.includes("test")))
+  );
+}
+
+function isRecoverySchedulerDisabled(): boolean {
+  return (
+    isEnvFlagEnabled("OMNIROUTE_DISABLE_CONNECTION_RECOVERY") ||
+    isEnvFlagEnabled("OMNIROUTE_DISABLE_BACKGROUND_SERVICES") ||
+    isBuildProcess() ||
+    isAutomatedTestProcess()
+  );
+}
+
+/**
+ * Resolve the tick interval (ms) from OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS,
+ * falling back to the 60s default and clamping to a small floor.
+ */
+export function resolveConnectionRecoveryIntervalMs(
+  rawValue: string | undefined = typeof process !== "undefined"
+    ? process.env.OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS
+    : undefined
+): number {
+  if (!rawValue) return DEFAULT_TICK_MS;
+  const parsed = Number(rawValue);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_TICK_MS;
+  return Math.max(MIN_TICK_MS, Math.floor(parsed));
+}
+
+/**
+ * Start the proactive connection-recovery scheduler (idempotent). No-op in
+ * build/test processes or when disabled via env. Each tick runs
+ * runConnectionRecoveryTick() against the real DB.
+ */
+export function initConnectionRecoveryScheduler(): void {
+  const state = getRecoveryState();
+  if (state.initialized || isRecoverySchedulerDisabled()) return;
+  state.initialized = true;
+
+  const tickMs = resolveConnectionRecoveryIntervalMs();
+  const tickLogger = {
+    info: (msg: string) => console.log(msg),
+    warn: (msg: string) => console.warn(msg),
+  };
+
+  const runTick = () => {
+    runConnectionRecoveryTick({ logger: tickLogger }).catch((err: unknown) => {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`${RECOVERY_LOG_PREFIX} tick error (non-fatal): ${msg}`);
+    });
+  };
+
+  console.log(
+    `${RECOVERY_LOG_PREFIX} Starting proactive cooldown recovery (tick every ${Math.round(tickMs / 1000)}s)`
+  );
+
+  // Delay the first tick a little so it never piles onto cold-start work.
+  const timer = setTimeout(() => {
+    runTick();
+    state.interval = setInterval(runTick, tickMs);
+    (state.interval as { unref?: () => void } | undefined)?.unref?.();
+  }, 15_000);
+  (timer as { unref?: () => void } | undefined)?.unref?.();
+}
+
+/** Stop the scheduler (tests / hot-reload). */
+export function stopConnectionRecoveryScheduler(): void {
+  const state = getRecoveryState();
+  if (state.interval) {
+    clearInterval(state.interval);
+    state.interval = null;
+  }
+  state.initialized = false;
+}

tests/unit/quota-connection-recovery.test.ts

@@ -0,0 +1,212 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  isRecoverableCooldownConnection,
+  resolveConnectionRecoveryIntervalMs,
+  runConnectionRecoveryTick,
+  selectRecoverableConnections,
+  type RecoverableConnectionInput,
+} from "../../src/lib/quota/connectionRecovery.ts";
+
+const NOW = Date.UTC(2026, 5, 23, 12, 0, 0); // fixed clock for deterministic tests
+const PAST = new Date(NOW - 60_000).toISOString(); // 60s in the past → cooldown elapsed
+const FUTURE = new Date(NOW + 60_000).toISOString(); // 60s in the future → still cooling
+
+function conn(overrides: Partial<RecoverableConnectionInput>): RecoverableConnectionInput {
+  // Use the `in` operator so an EXPLICIT null/undefined override is honored
+  // (??/|| would collapse it back to the default and hide the no-cooldown case).
+  return {
+    id: "id" in overrides ? (overrides.id as string) : "c1",
+    testStatus: "testStatus" in overrides ? overrides.testStatus : "unavailable",
+    rateLimitedUntil: "rateLimitedUntil" in overrides ? overrides.rateLimitedUntil : PAST,
+  };
+}
+
+test("isRecoverableCooldownConnection: unavailable + elapsed cooldown → recoverable", () => {
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ testStatus: "unavailable", rateLimitedUntil: PAST }), NOW),
+    true
+  );
+});
+
+test("isRecoverableCooldownConnection: cooldown still in the future → NOT recoverable", () => {
+  assert.equal(
+    isRecoverableCooldownConnection(
+      conn({ testStatus: "unavailable", rateLimitedUntil: FUTURE }),
+      NOW
+    ),
+    false
+  );
+});
+
+test("isRecoverableCooldownConnection: terminal states are never recovered", () => {
+  for (const status of ["banned", "expired", "credits_exhausted"]) {
+    assert.equal(
+      isRecoverableCooldownConnection(conn({ testStatus: status, rateLimitedUntil: PAST }), NOW),
+      false,
+      `${status} must not be recoverable`
+    );
+  }
+});
+
+test("isRecoverableCooldownConnection: terminal-status matching is case/space insensitive", () => {
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ testStatus: " Banned ", rateLimitedUntil: PAST }), NOW),
+    false
+  );
+});
+
+test("isRecoverableCooldownConnection: no rateLimitedUntil → NOT recoverable", () => {
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ testStatus: "unavailable", rateLimitedUntil: null }), NOW),
+    false
+  );
+  assert.equal(
+    isRecoverableCooldownConnection(
+      conn({ testStatus: "unavailable", rateLimitedUntil: undefined }),
+      NOW
+    ),
+    false
+  );
+});
+
+test("isRecoverableCooldownConnection: status other than 'unavailable' is left alone", () => {
+  // Only the transient-cooldown status should be proactively restored. An
+  // 'active' or null status with a stale rateLimitedUntil is not this job's
+  // concern (the lazy backoff-decay path already handles active rows).
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ testStatus: "active", rateLimitedUntil: PAST }), NOW),
+    false
+  );
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ testStatus: null, rateLimitedUntil: PAST }), NOW),
+    false
+  );
+});
+
+test("isRecoverableCooldownConnection: missing connection id → NOT recoverable", () => {
+  assert.equal(
+    isRecoverableCooldownConnection(conn({ id: "", rateLimitedUntil: PAST }), NOW),
+    false
+  );
+});
+
+test("isRecoverableCooldownConnection: numeric-epoch rateLimitedUntil string is tolerated", () => {
+  // The rate_limited_until TEXT column can hold a numeric epoch string (#3954).
+  assert.equal(
+    isRecoverableCooldownConnection(
+      conn({ testStatus: "unavailable", rateLimitedUntil: String(NOW - 1_000) }),
+      NOW
+    ),
+    true
+  );
+  assert.equal(
+    isRecoverableCooldownConnection(
+      conn({ testStatus: "unavailable", rateLimitedUntil: String(NOW + 1_000) }),
+      NOW
+    ),
+    false
+  );
+});
+
+test("selectRecoverableConnections returns only the elapsed-cooldown unavailable rows", () => {
+  const connections: RecoverableConnectionInput[] = [
+    conn({ id: "elapsed", testStatus: "unavailable", rateLimitedUntil: PAST }),
+    conn({ id: "still-cooling", testStatus: "unavailable", rateLimitedUntil: FUTURE }),
+    conn({ id: "banned", testStatus: "banned", rateLimitedUntil: PAST }),
+    conn({ id: "expired", testStatus: "expired", rateLimitedUntil: PAST }),
+    conn({ id: "credits", testStatus: "credits_exhausted", rateLimitedUntil: PAST }),
+    conn({ id: "no-cooldown", testStatus: "unavailable", rateLimitedUntil: null }),
+    conn({ id: "active", testStatus: "active", rateLimitedUntil: PAST }),
+  ];
+
+  const recoverable = selectRecoverableConnections(connections, NOW);
+  assert.deepEqual(
+    recoverable.map((c) => c.id),
+    ["elapsed"]
+  );
+});
+
+test("selectRecoverableConnections returns [] for empty / non-array input", () => {
+  assert.deepEqual(selectRecoverableConnections([], NOW), []);
+  assert.deepEqual(
+    selectRecoverableConnections(undefined as unknown as RecoverableConnectionInput[], NOW),
+    []
+  );
+});
+
+test("selectRecoverableConnections does not mutate the input array", () => {
+  const connections: RecoverableConnectionInput[] = [
+    conn({ id: "a", rateLimitedUntil: PAST }),
+    conn({ id: "b", rateLimitedUntil: FUTURE }),
+  ];
+  const before = connections.length;
+  selectRecoverableConnections(connections, NOW);
+  assert.equal(connections.length, before);
+});
+
+test("runConnectionRecoveryTick clears only the elapsed-cooldown connections (injected deps, no DB)", async () => {
+  const cleared: string[] = [];
+  const result = await runConnectionRecoveryTick({
+    nowMs: NOW,
+    loadConnections: async () => [
+      conn({ id: "elapsed", testStatus: "unavailable", rateLimitedUntil: PAST }),
+      conn({ id: "still-cooling", testStatus: "unavailable", rateLimitedUntil: FUTURE }),
+      conn({ id: "banned", testStatus: "banned", rateLimitedUntil: PAST }),
+      conn({ id: "active", testStatus: "active", rateLimitedUntil: PAST }),
+    ],
+    clearConnectionError: async (connectionId) => {
+      cleared.push(connectionId);
+    },
+  });
+
+  assert.deepEqual(cleared, ["elapsed"]);
+  assert.equal(result.scanned, 4);
+  assert.equal(result.recovered, 1);
+  assert.deepEqual(result.recoveredIds, ["elapsed"]);
+});
+
+test("runConnectionRecoveryTick isolates a per-connection clear failure (others still recovered)", async () => {
+  const cleared: string[] = [];
+  const warnings: string[] = [];
+  const result = await runConnectionRecoveryTick({
+    nowMs: NOW,
+    loadConnections: async () => [
+      conn({ id: "boom", testStatus: "unavailable", rateLimitedUntil: PAST }),
+      conn({ id: "ok", testStatus: "unavailable", rateLimitedUntil: PAST }),
+    ],
+    clearConnectionError: async (connectionId) => {
+      if (connectionId === "boom") throw new Error("db write failed");
+      cleared.push(connectionId);
+    },
+    logger: { warn: (m) => warnings.push(m) },
+  });
+
+  assert.deepEqual(cleared, ["ok"]);
+  assert.equal(result.recovered, 1);
+  assert.equal(warnings.length, 1);
+});
+
+test("runConnectionRecoveryTick returns a zero result and never throws when loading fails", async () => {
+  const result = await runConnectionRecoveryTick({
+    nowMs: NOW,
+    loadConnections: async () => {
+      throw new Error("DB unavailable");
+    },
+    clearConnectionError: async () => {
+      throw new Error("must not be called");
+    },
+  });
+  assert.deepEqual(result, { scanned: 0, recovered: 0, recoveredIds: [] });
+});
+
+test("resolveConnectionRecoveryIntervalMs defaults to 60s and clamps to a floor", () => {
+  assert.equal(resolveConnectionRecoveryIntervalMs(undefined), 60_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs(""), 60_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs("not-a-number"), 60_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs("0"), 60_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs("-5"), 60_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs("120000"), 120_000);
+  assert.equal(resolveConnectionRecoveryIntervalMs("1000"), 5_000); // clamped up to MIN_TICK_MS
+});