From 27c994ffd3a370a816df75f113c343fae83c0daf Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Wed, 24 Jun 2026 00:04:20 -0300 Subject: [PATCH] =?UTF-8?q?feat(quota):=20recupera=C3=A7=C3=A3o=20proativa?= =?UTF-8?q?=20de=20conex=C3=B5es=20em=20cooldown=20(cron=20heal)=20[Fase?= =?UTF-8?q?=203=20#8]=20(#4900)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrated into release/v3.8.36 --- .env.example | 10 + docs/reference/ENVIRONMENT.md | 2 + src/instrumentation-node.ts | 12 + src/lib/quota/connectionRecovery.ts | 303 +++++++++++++++++++ tests/unit/quota-connection-recovery.test.ts | 212 +++++++++++++ 5 files changed, 539 insertions(+) create mode 100644 src/lib/quota/connectionRecovery.ts create mode 100644 tests/unit/quota-connection-recovery.test.ts diff --git a/.env.example b/.env.example index c1a9d5ecce..393f267015 100644 --- a/.env.example +++ b/.env.example @@ -551,6 +551,16 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # heuristic in instrumentation-node.ts. Default: unset (tests skip background). #OMNIROUTE_ENABLE_RUNTIME_BACKGROUND_TASKS=1 +# Proactive connection-cooldown recovery (#8): re-validates connections whose +# transient `rate_limited_until` window has elapsed OUTSIDE the request hot path, +# so the first request after a cooldown does not pay the probe latency. Lazy +# recovery in getProviderCredentials still applies regardless. Used by: +# src/lib/quota/connectionRecovery.ts. +# Tick cadence (ms). Default 60000, floor 5000. +# OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS=60000 +# Disable the proactive recovery scheduler entirely (default: false). +# OMNIROUTE_DISABLE_CONNECTION_RECOVERY=false + # Background job interval for budget reset checks (ms). Default: 600000 (10m). # Used by: src/lib/jobs/budgetResetJob.ts. Floor: 10000. #OMNIROUTE_BUDGET_RESET_JOB_INTERVAL_MS=600000 diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index d999288096..734d33711d 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -386,6 +386,8 @@ detection above). | `OMNIROUTE_DISABLE_BACKGROUND_SERVICES` | `false` | `src/instrumentation-node.ts` | Disable all background services (sync, pricing, model refresh). Useful for CI/test. | | `OMNIROUTE_ENABLE_RUNTIME_BACKGROUND_TASKS` | _(unset)_ | `src/lib/config/runtimeSettings.ts` | Force background tasks on under automated test detection. Set `1` to override the test heuristic. | | `OMNIROUTE_BUDGET_RESET_JOB_INTERVAL_MS` | `600000` | `src/lib/jobs/budgetResetJob.ts` | Budget reset check cadence (ms). Floor `10000`. | +| `OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS` | `60000` | `src/lib/quota/connectionRecovery.ts` | Proactive connection-cooldown recovery cadence (ms): re-validates connections whose transient `rate_limited_until` has elapsed, off the request hot path. Floor `5000`. | +| `OMNIROUTE_DISABLE_CONNECTION_RECOVERY` | `false` | `src/lib/quota/connectionRecovery.ts` | Disable the proactive connection-cooldown recovery scheduler (lazy recovery in `getProviderCredentials` still applies). | | `OMNIROUTE_REASONING_CACHE_CLEANUP_INTERVAL_MS` | `1800000` | `src/lib/jobs/reasoningCacheCleanupJob.ts` | Reasoning cache cleanup cadence (ms). Floor `60000`. | | `OMNIROUTE_CONFIG_HOT_RELOAD_MS` | `5000` | `src/lib/config/hotReload.ts` | Polling interval (ms) for config hot-reload. Lower than `1000` is rejected. | | `OMNIROUTE_DISABLE_REDIS_AUTH_CACHE` | _(enabled)_ | `src/lib/db/apiKeys.ts` | Set `1` to bypass the Redis-backed API-key auth cache (forces DB reads). | diff --git a/src/instrumentation-node.ts b/src/instrumentation-node.ts index 32debc0f35..30f79f8f59 100755 --- a/src/instrumentation-node.ts +++ b/src/instrumentation-node.ts @@ -276,6 +276,18 @@ export async function registerNodejs(): Promise { console.warn("[STARTUP] Auto-refresh daemon failed to start (non-fatal):", msg); } + // Proactive connection-cooldown recovery (#8): re-validate connections whose + // transient `rate_limited_until` window has elapsed OUTSIDE the request hot + // path, so the first request after a cooldown does not pay the probe latency. + // Lazy/self-recovery still happens in getProviderCredentials; this front-runs it. + try { + const { initConnectionRecoveryScheduler } = await import("@/lib/quota/connectionRecovery"); + initConnectionRecoveryScheduler(); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[STARTUP] Connection recovery scheduler failed to start (non-fatal):", msg); + } + try { // Arena ELO sync: model intelligence from the Arena AI leaderboard, powering the // Free Provider Rankings page. On by default; configurable from Dashboard Feature Flags. diff --git a/src/lib/quota/connectionRecovery.ts b/src/lib/quota/connectionRecovery.ts new file mode 100644 index 0000000000..1e0e8d3034 --- /dev/null +++ b/src/lib/quota/connectionRecovery.ts @@ -0,0 +1,303 @@ +/** + * connectionRecovery.ts — Proactive recovery of provider connections whose + * transient cooldown has elapsed. + * + * Today the cooldown released by `markAccountUnavailable()` (testStatus + * 'unavailable' + a future `rateLimitedUntil`) is recovered LAZILY: a connection + * only becomes eligible again when the next real request reads it in + * `getProviderCredentials` (src/sse/services/auth.ts). The first request after + * the cooldown window therefore pays the latency of re-discovering a healthy + * connection. + * + * Modeled on gpt-load's CronChecker, this module identifies the subset of + * connections that are cooling down with an already-elapsed window and clears + * their error state OUTSIDE the request hot path, so they are restored before + * the next real request arrives. + * + * This file holds the PURE selection logic (`selectRecoverableConnections`, + * `isRecoverableCooldownConnection`) — no DB, no network, time injected — plus a + * thin async tick (`runConnectionRecoveryTick`) that wires the helper to the DB. + * The tick is NOT auto-started on import; the caller (startup bootstrap) decides + * when to schedule it, so importing this module in tests never spawns a timer. + */ + +import { cooldownUntilMs } from "@omniroute/open-sse/services/accountFallback.ts"; + +/** + * The transient-cooldown status written by `markAccountUnavailable()` for a + * recoverable failure. Only connections in this status are candidates for + * proactive recovery. + */ +export const RECOVERABLE_COOLDOWN_STATUS = "unavailable"; + +/** + * Terminal connection statuses that must NEVER be auto-recovered — they stay + * unavailable until credentials/settings change or an operator resets them. + * Mirrors `isTerminalConnectionStatus` (src/sse/services/auth.ts) and + * `TERMINAL_STATUSES` (src/lib/db/providers.ts::clearStaleCrashCooldowns). + */ +export const TERMINAL_CONNECTION_STATUSES = new Set([ + "banned", + "expired", + "credits_exhausted", +]); + +/** Minimal connection shape needed to decide recoverability. */ +export interface RecoverableConnectionInput { + id: string; + testStatus?: string | null; + rateLimitedUntil?: string | null; +} + +function normalizeStatus(value: string | null | undefined): string { + return (value || "").trim().toLowerCase(); +} + +/** + * True when `rateLimitedUntil` is set and its instant is at or before `nowMs` + * (the cooldown window has elapsed). Tolerates ISO strings and numeric-epoch + * strings — the `rate_limited_until` TEXT column can hold either (#3954). + */ +function hasElapsedCooldown(rateLimitedUntil: string | null | undefined, nowMs: number): boolean { + if (!rateLimitedUntil) return false; + const ms = cooldownUntilMs(rateLimitedUntil); + return Number.isFinite(ms) && ms <= nowMs; +} + +/** + * Decide whether a single connection is a proactive-recovery candidate: + * - has a real id, AND + * - testStatus === 'unavailable' (the transient cooldown status), AND + * - rateLimitedUntil is set and already in the past (< nowMs), AND + * - is NOT in a terminal state (banned / expired / credits_exhausted). + * + * Pure — `nowMs` is injected so callers/tests control the clock. + */ +export function isRecoverableCooldownConnection( + connection: RecoverableConnectionInput | null | undefined, + nowMs: number +): boolean { + if (!connection || typeof connection.id !== "string" || connection.id.length === 0) { + return false; + } + const status = normalizeStatus(connection.testStatus); + if (status !== RECOVERABLE_COOLDOWN_STATUS) return false; + if (TERMINAL_CONNECTION_STATUSES.has(status)) return false; // defensive; 'unavailable' is never terminal + return hasElapsedCooldown(connection.rateLimitedUntil, nowMs); +} + +/** + * From a list of connections, return only those whose transient cooldown has + * elapsed and are safe to restore. Pure, non-mutating, time injected. + */ +export function selectRecoverableConnections( + connections: readonly T[] | null | undefined, + nowMs: number +): T[] { + if (!Array.isArray(connections)) return []; + return connections.filter((connection) => isRecoverableCooldownConnection(connection, nowMs)); +} + +/** Result of one recovery tick (handy for logging / tests of the wiring). */ +export interface ConnectionRecoveryTickResult { + scanned: number; + recovered: number; + recoveredIds: string[]; +} + +/** + * Run one proactive-recovery pass: load active provider connections, select the + * subset whose transient cooldown has elapsed, and clear their error state via + * `clearAccountError` so they are eligible again before the next real request. + * + * Dependencies are injected (default to the real DB / auth modules) so the tick + * can be unit-tested without a live database. Best-effort and never throws — a + * failure to recover one connection must not abort the others or the scheduler. + * + * NOTE: not auto-started on import. The startup bootstrap is responsible for + * scheduling it (see runConnectionRecoveryTick usage in the report). + */ +export async function runConnectionRecoveryTick( + deps: { + nowMs?: number; + loadConnections?: () => Promise; + clearConnectionError?: ( + connectionId: string, + current: RecoverableConnectionInput + ) => Promise; + logger?: { info?: (msg: string) => void; warn?: (msg: string) => void }; + } = {} +): Promise { + const nowMs = deps.nowMs ?? Date.now(); + const result: ConnectionRecoveryTickResult = { scanned: 0, recovered: 0, recoveredIds: [] }; + + let connections: RecoverableConnectionInput[]; + try { + const load = + deps.loadConnections ?? + (async () => { + // Lazy import keeps this module loadable (and the pure helpers testable) + // without a full DB/auth graph. + const { getProviderConnections } = await import("@/lib/db/providers"); + const rows = (await getProviderConnections({ isActive: true })) as Array<{ + id?: unknown; + testStatus?: unknown; + rateLimitedUntil?: unknown; + }>; + return (Array.isArray(rows) ? rows : []).map((row) => ({ + id: typeof row.id === "string" ? row.id : "", + testStatus: typeof row.testStatus === "string" ? row.testStatus : null, + rateLimitedUntil: + typeof row.rateLimitedUntil === "string" ? row.rateLimitedUntil : null, + })); + }); + connections = await load(); + } catch (err) { + deps.logger?.warn?.( + `[ConnectionRecovery] failed to load connections: ${err instanceof Error ? err.message : String(err)}` + ); + return result; + } + + result.scanned = connections.length; + const recoverable = selectRecoverableConnections(connections, nowMs); + if (recoverable.length === 0) return result; + + const clear = + deps.clearConnectionError ?? + (async (connectionId: string, current: RecoverableConnectionInput) => { + const { clearAccountError } = await import("@/sse/services/auth"); + await clearAccountError(connectionId, current); + }); + + for (const connection of recoverable) { + try { + await clear(connection.id, connection); + result.recovered += 1; + result.recoveredIds.push(connection.id); + } catch (err) { + deps.logger?.warn?.( + `[ConnectionRecovery] failed to recover ${connection.id.slice(0, 8)}: ${err instanceof Error ? err.message : String(err)}` + ); + } + } + + if (result.recovered > 0) { + deps.logger?.info?.( + `[ConnectionRecovery] proactively restored ${result.recovered} connection(s) with elapsed cooldown` + ); + } + return result; +} + +// ── Scheduler (opt-out, low frequency) ────────────────────────────────────── +// Mirrors src/lib/tokenHealthCheck.ts: a globalThis-guarded singleton so HMR / +// double-import never stacks timers, an unref'd interval so it never holds the +// process open, and a self-disable in build/test processes. NOT auto-started on +// import — startup bootstrap calls initConnectionRecoveryScheduler(). + +const DEFAULT_TICK_MS = 60 * 1000; // re-validate elapsed cooldowns every 60s +const MIN_TICK_MS = 5 * 1000; // floor to avoid hot-looping if misconfigured +const RECOVERY_LOG_PREFIX = "[ConnectionRecovery]"; +const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]); + +declare global { + var __omnirouteConnRecovery: + | { initialized: boolean; interval: ReturnType | null } + | undefined; +} + +function getRecoveryState() { + if (!globalThis.__omnirouteConnRecovery) { + globalThis.__omnirouteConnRecovery = { initialized: false, interval: null }; + } + return globalThis.__omnirouteConnRecovery; +} + +function isEnvFlagEnabled(name: string): boolean { + const value = typeof process !== "undefined" ? process.env[name] : undefined; + return !!value && TRUE_ENV_VALUES.has(value.trim().toLowerCase()); +} + +function isBuildProcess(): boolean { + return typeof process !== "undefined" && process.env.NEXT_PHASE === "phase-production-build"; +} + +function isAutomatedTestProcess(): boolean { + return ( + typeof process !== "undefined" && + (process.env.NODE_ENV === "test" || + process.env.VITEST !== undefined || + process.argv.some((arg) => arg.includes("test"))) + ); +} + +function isRecoverySchedulerDisabled(): boolean { + return ( + isEnvFlagEnabled("OMNIROUTE_DISABLE_CONNECTION_RECOVERY") || + isEnvFlagEnabled("OMNIROUTE_DISABLE_BACKGROUND_SERVICES") || + isBuildProcess() || + isAutomatedTestProcess() + ); +} + +/** + * Resolve the tick interval (ms) from OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS, + * falling back to the 60s default and clamping to a small floor. + */ +export function resolveConnectionRecoveryIntervalMs( + rawValue: string | undefined = typeof process !== "undefined" + ? process.env.OMNIROUTE_CONNECTION_RECOVERY_INTERVAL_MS + : undefined +): number { + if (!rawValue) return DEFAULT_TICK_MS; + const parsed = Number(rawValue); + if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_TICK_MS; + return Math.max(MIN_TICK_MS, Math.floor(parsed)); +} + +/** + * Start the proactive connection-recovery scheduler (idempotent). No-op in + * build/test processes or when disabled via env. Each tick runs + * runConnectionRecoveryTick() against the real DB. + */ +export function initConnectionRecoveryScheduler(): void { + const state = getRecoveryState(); + if (state.initialized || isRecoverySchedulerDisabled()) return; + state.initialized = true; + + const tickMs = resolveConnectionRecoveryIntervalMs(); + const tickLogger = { + info: (msg: string) => console.log(msg), + warn: (msg: string) => console.warn(msg), + }; + + const runTick = () => { + runConnectionRecoveryTick({ logger: tickLogger }).catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + console.warn(`${RECOVERY_LOG_PREFIX} tick error (non-fatal): ${msg}`); + }); + }; + + console.log( + `${RECOVERY_LOG_PREFIX} Starting proactive cooldown recovery (tick every ${Math.round(tickMs / 1000)}s)` + ); + + // Delay the first tick a little so it never piles onto cold-start work. + const timer = setTimeout(() => { + runTick(); + state.interval = setInterval(runTick, tickMs); + (state.interval as { unref?: () => void } | undefined)?.unref?.(); + }, 15_000); + (timer as { unref?: () => void } | undefined)?.unref?.(); +} + +/** Stop the scheduler (tests / hot-reload). */ +export function stopConnectionRecoveryScheduler(): void { + const state = getRecoveryState(); + if (state.interval) { + clearInterval(state.interval); + state.interval = null; + } + state.initialized = false; +} diff --git a/tests/unit/quota-connection-recovery.test.ts b/tests/unit/quota-connection-recovery.test.ts new file mode 100644 index 0000000000..24120ab1ac --- /dev/null +++ b/tests/unit/quota-connection-recovery.test.ts @@ -0,0 +1,212 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + isRecoverableCooldownConnection, + resolveConnectionRecoveryIntervalMs, + runConnectionRecoveryTick, + selectRecoverableConnections, + type RecoverableConnectionInput, +} from "../../src/lib/quota/connectionRecovery.ts"; + +const NOW = Date.UTC(2026, 5, 23, 12, 0, 0); // fixed clock for deterministic tests +const PAST = new Date(NOW - 60_000).toISOString(); // 60s in the past → cooldown elapsed +const FUTURE = new Date(NOW + 60_000).toISOString(); // 60s in the future → still cooling + +function conn(overrides: Partial): RecoverableConnectionInput { + // Use the `in` operator so an EXPLICIT null/undefined override is honored + // (??/|| would collapse it back to the default and hide the no-cooldown case). + return { + id: "id" in overrides ? (overrides.id as string) : "c1", + testStatus: "testStatus" in overrides ? overrides.testStatus : "unavailable", + rateLimitedUntil: "rateLimitedUntil" in overrides ? overrides.rateLimitedUntil : PAST, + }; +} + +test("isRecoverableCooldownConnection: unavailable + elapsed cooldown → recoverable", () => { + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: "unavailable", rateLimitedUntil: PAST }), NOW), + true + ); +}); + +test("isRecoverableCooldownConnection: cooldown still in the future → NOT recoverable", () => { + assert.equal( + isRecoverableCooldownConnection( + conn({ testStatus: "unavailable", rateLimitedUntil: FUTURE }), + NOW + ), + false + ); +}); + +test("isRecoverableCooldownConnection: terminal states are never recovered", () => { + for (const status of ["banned", "expired", "credits_exhausted"]) { + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: status, rateLimitedUntil: PAST }), NOW), + false, + `${status} must not be recoverable` + ); + } +}); + +test("isRecoverableCooldownConnection: terminal-status matching is case/space insensitive", () => { + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: " Banned ", rateLimitedUntil: PAST }), NOW), + false + ); +}); + +test("isRecoverableCooldownConnection: no rateLimitedUntil → NOT recoverable", () => { + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: "unavailable", rateLimitedUntil: null }), NOW), + false + ); + assert.equal( + isRecoverableCooldownConnection( + conn({ testStatus: "unavailable", rateLimitedUntil: undefined }), + NOW + ), + false + ); +}); + +test("isRecoverableCooldownConnection: status other than 'unavailable' is left alone", () => { + // Only the transient-cooldown status should be proactively restored. An + // 'active' or null status with a stale rateLimitedUntil is not this job's + // concern (the lazy backoff-decay path already handles active rows). + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: "active", rateLimitedUntil: PAST }), NOW), + false + ); + assert.equal( + isRecoverableCooldownConnection(conn({ testStatus: null, rateLimitedUntil: PAST }), NOW), + false + ); +}); + +test("isRecoverableCooldownConnection: missing connection id → NOT recoverable", () => { + assert.equal( + isRecoverableCooldownConnection(conn({ id: "", rateLimitedUntil: PAST }), NOW), + false + ); +}); + +test("isRecoverableCooldownConnection: numeric-epoch rateLimitedUntil string is tolerated", () => { + // The rate_limited_until TEXT column can hold a numeric epoch string (#3954). + assert.equal( + isRecoverableCooldownConnection( + conn({ testStatus: "unavailable", rateLimitedUntil: String(NOW - 1_000) }), + NOW + ), + true + ); + assert.equal( + isRecoverableCooldownConnection( + conn({ testStatus: "unavailable", rateLimitedUntil: String(NOW + 1_000) }), + NOW + ), + false + ); +}); + +test("selectRecoverableConnections returns only the elapsed-cooldown unavailable rows", () => { + const connections: RecoverableConnectionInput[] = [ + conn({ id: "elapsed", testStatus: "unavailable", rateLimitedUntil: PAST }), + conn({ id: "still-cooling", testStatus: "unavailable", rateLimitedUntil: FUTURE }), + conn({ id: "banned", testStatus: "banned", rateLimitedUntil: PAST }), + conn({ id: "expired", testStatus: "expired", rateLimitedUntil: PAST }), + conn({ id: "credits", testStatus: "credits_exhausted", rateLimitedUntil: PAST }), + conn({ id: "no-cooldown", testStatus: "unavailable", rateLimitedUntil: null }), + conn({ id: "active", testStatus: "active", rateLimitedUntil: PAST }), + ]; + + const recoverable = selectRecoverableConnections(connections, NOW); + assert.deepEqual( + recoverable.map((c) => c.id), + ["elapsed"] + ); +}); + +test("selectRecoverableConnections returns [] for empty / non-array input", () => { + assert.deepEqual(selectRecoverableConnections([], NOW), []); + assert.deepEqual( + selectRecoverableConnections(undefined as unknown as RecoverableConnectionInput[], NOW), + [] + ); +}); + +test("selectRecoverableConnections does not mutate the input array", () => { + const connections: RecoverableConnectionInput[] = [ + conn({ id: "a", rateLimitedUntil: PAST }), + conn({ id: "b", rateLimitedUntil: FUTURE }), + ]; + const before = connections.length; + selectRecoverableConnections(connections, NOW); + assert.equal(connections.length, before); +}); + +test("runConnectionRecoveryTick clears only the elapsed-cooldown connections (injected deps, no DB)", async () => { + const cleared: string[] = []; + const result = await runConnectionRecoveryTick({ + nowMs: NOW, + loadConnections: async () => [ + conn({ id: "elapsed", testStatus: "unavailable", rateLimitedUntil: PAST }), + conn({ id: "still-cooling", testStatus: "unavailable", rateLimitedUntil: FUTURE }), + conn({ id: "banned", testStatus: "banned", rateLimitedUntil: PAST }), + conn({ id: "active", testStatus: "active", rateLimitedUntil: PAST }), + ], + clearConnectionError: async (connectionId) => { + cleared.push(connectionId); + }, + }); + + assert.deepEqual(cleared, ["elapsed"]); + assert.equal(result.scanned, 4); + assert.equal(result.recovered, 1); + assert.deepEqual(result.recoveredIds, ["elapsed"]); +}); + +test("runConnectionRecoveryTick isolates a per-connection clear failure (others still recovered)", async () => { + const cleared: string[] = []; + const warnings: string[] = []; + const result = await runConnectionRecoveryTick({ + nowMs: NOW, + loadConnections: async () => [ + conn({ id: "boom", testStatus: "unavailable", rateLimitedUntil: PAST }), + conn({ id: "ok", testStatus: "unavailable", rateLimitedUntil: PAST }), + ], + clearConnectionError: async (connectionId) => { + if (connectionId === "boom") throw new Error("db write failed"); + cleared.push(connectionId); + }, + logger: { warn: (m) => warnings.push(m) }, + }); + + assert.deepEqual(cleared, ["ok"]); + assert.equal(result.recovered, 1); + assert.equal(warnings.length, 1); +}); + +test("runConnectionRecoveryTick returns a zero result and never throws when loading fails", async () => { + const result = await runConnectionRecoveryTick({ + nowMs: NOW, + loadConnections: async () => { + throw new Error("DB unavailable"); + }, + clearConnectionError: async () => { + throw new Error("must not be called"); + }, + }); + assert.deepEqual(result, { scanned: 0, recovered: 0, recoveredIds: [] }); +}); + +test("resolveConnectionRecoveryIntervalMs defaults to 60s and clamps to a floor", () => { + assert.equal(resolveConnectionRecoveryIntervalMs(undefined), 60_000); + assert.equal(resolveConnectionRecoveryIntervalMs(""), 60_000); + assert.equal(resolveConnectionRecoveryIntervalMs("not-a-number"), 60_000); + assert.equal(resolveConnectionRecoveryIntervalMs("0"), 60_000); + assert.equal(resolveConnectionRecoveryIntervalMs("-5"), 60_000); + assert.equal(resolveConnectionRecoveryIntervalMs("120000"), 120_000); + assert.equal(resolveConnectionRecoveryIntervalMs("1000"), 5_000); // clamped up to MIN_TICK_MS +});