diff --git a/src/app/api/providers/[id]/test/route.ts b/src/app/api/providers/[id]/test/route.ts index 1ae613dab8..215771104c 100644 --- a/src/app/api/providers/[id]/test/route.ts +++ b/src/app/api/providers/[id]/test/route.ts @@ -30,6 +30,7 @@ import { providerAllowsOptionalApiKey } from "@/shared/constants/providers"; import { shouldUseApiKeyConnectionTest } from "./webSessionTestDispatch"; import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHealth"; import { removeConnectionHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts"; +import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits"; import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation"; import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth"; import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult"; @@ -1082,23 +1083,46 @@ export async function testSingleConnection(connectionId: string, validationModel terminalTestStatuses.has(String(diagnosis.code ?? diagnosis.type ?? "").toLowerCase()); const testFailureCooldownMs = result.valid ? 0 : 30_000; // 30s retry window + // A successful credential probe proves the KEY is valid. It does NOT prove the + // quota window reopened: the probe is a cheap auth/models call that never touches + // the chat quota a weekly cap applies to. Clearing an ACTIVE cooldown here — which + // the credential-health scheduler triggers for every connection every 300s — put + // `zai/glm-5.3` back to `active` / `rate_limited_until = NULL` within 30s of every + // restart, so combo dispatched it straight into the same weekly 429. Same rule as + // maybeClearRecoveredQuotaState: a future rateLimitedUntil is the 429 handler's + // hard statement and no poller may overrule it. Once it elapses, the next probe + // clears it normally. + const clearErrorState = shouldClearErrorStateOnValidProbe( + connection as { rateLimitedUntil?: string | null }, + result.valid + ); + const updateData: Record = { - testStatus: result.valid ? "active" : "error", - lastError: result.valid ? null : result.error, - lastErrorAt: result.valid ? null : now, + testStatus: clearErrorState ? "active" : result.valid ? connection.testStatus : "error", + lastError: clearErrorState ? null : result.valid ? connection.lastError : result.error, + lastErrorAt: clearErrorState ? null : result.valid ? connection.lastErrorAt : now, lastTested: now, - lastErrorType: result.valid ? null : diagnosis.type, - lastErrorSource: result.valid ? null : diagnosis.source, - errorCode: result.valid ? null : diagnosis.code || result.statusCode || null, - rateLimitedUntil: - result.valid || isTerminalFailure - ? result.valid - ? null - : connection.rateLimitedUntil || null - : new Date(Date.now() + testFailureCooldownMs).toISOString(), + lastErrorType: clearErrorState ? null : result.valid ? connection.lastErrorType : diagnosis.type, + lastErrorSource: clearErrorState + ? null + : result.valid + ? connection.lastErrorSource + : diagnosis.source, + errorCode: clearErrorState + ? null + : result.valid + ? connection.errorCode + : diagnosis.code || result.statusCode || null, + rateLimitedUntil: clearErrorState + ? null + : isTerminalFailure + ? connection.rateLimitedUntil || null + : result.valid + ? connection.rateLimitedUntil || null + : new Date(Date.now() + testFailureCooldownMs).toISOString(), }; - if (result.valid) { + if (clearErrorState) { updateData.backoffLevel = 0; const psd = connection?.providerSpecificData as Record | undefined; diff --git a/src/lib/db/providers/rateLimit.ts b/src/lib/db/providers/rateLimit.ts index e9447152d3..7812080a0f 100644 --- a/src/lib/db/providers/rateLimit.ts +++ b/src/lib/db/providers/rateLimit.ts @@ -124,6 +124,27 @@ export function getEffectiveQuotaUsage( return used; } +/** + * Normalize a persisted `rate_limited_until` to epoch ms. + * + * The column is written in two shapes: epoch ms by `setConnectionRateLimitUntil` + * (the chat path) and an ISO-8601 string by `updateProviderConnection` (the + * dashboard/AUTH path). Returns null when the value is absent or unparseable — + * callers treat that as "no usable deadline". + */ +function parseCooldownUntilMs(value: string | number | null | undefined): number | null { + if (value == null || value === "") return null; + if (typeof value === "number") return Number.isFinite(value) ? value : null; + const raw = String(value).trim(); + if (raw === "") return null; + if (/^\d+$/.test(raw)) { + const numeric = Number(raw); + return Number.isFinite(numeric) ? numeric : null; + } + const parsed = Date.parse(raw); + return Number.isFinite(parsed) ? parsed : null; +} + /** * T05: Startup crash-recovery — clear stale transient connection cooldowns. * @@ -138,9 +159,19 @@ export function getEffectiveQuotaUsage( * - Only connections with `rate_limited_until IS NOT NULL` are touched. * - Terminal states (`banned`, `expired`, `credits_exhausted`) are skipped — * those require a deliberate credential change or operator reset. - * - Past timestamps are also cleared: they are already expired in the lazy + * - Past timestamps are cleared: they are already expired in the lazy * expiry sense, but clearing them resets `backoffLevel` / transient error - * fields so the connection gets a clean slate on this fresh process. + * fields so the connection gets a clean slate on this fresh process. An + * unparseable timestamp is treated the same way — it can never expire + * lazily, so leaving it would strand the connection forever. + * - FUTURE timestamps are NEVER cleared. Clearing them was the original + * behaviour and it wiped legitimate multi-day quota cooldowns on every + * container recreate: a GLM weekly cap persisted until 2026-08-29 came + * back `active` with `rate_limited_until = NULL`, combo dispatched it + * immediately, and the connection re-earned a real upstream 429. A stale + * crash-backoff value is bounded by the engine's own cooldown cap, so + * honouring it costs at most that window — far less than burning quota + * against an upstream that is provably exhausted. * * Must be called once, early in the startup sequence, before any request * is handled. Returns the number of connections that were cleared. @@ -148,6 +179,7 @@ export function getEffectiveQuotaUsage( export function clearStaleCrashCooldowns(): { cleared: number } { const db = getDbInstance() as unknown as DbLike; const now = new Date().toISOString(); + const nowMs = Date.now(); // Fetch all connections that have a rate_limited_until set and are NOT in // a terminal state. We do the terminal-status filter in JS to reuse the @@ -156,13 +188,20 @@ export function clearStaleCrashCooldowns(): { cleared: number } { const rows = db .prepare( - `SELECT id, test_status FROM provider_connections WHERE rate_limited_until IS NOT NULL` + `SELECT id, test_status, rate_limited_until FROM provider_connections WHERE rate_limited_until IS NOT NULL` ) - .all() as Array<{ id: string; test_status: string | null }>; + .all() as Array<{ + id: string; + test_status: string | null; + rate_limited_until: string | number | null; + }>; const toReset = rows.filter((r) => { const status = (r.test_status || "").trim().toLowerCase(); - return !TERMINAL_STATUSES.has(status); + if (TERMINAL_STATUSES.has(status)) return false; + const untilMs = parseCooldownUntilMs(r.rate_limited_until); + // Unparseable → clear (cannot expire lazily). Future → keep. + return untilMs === null || untilMs <= nowMs; }); if (toReset.length === 0) return { cleared: 0 }; diff --git a/src/lib/usage/providerLimits.ts b/src/lib/usage/providerLimits.ts index ae39205674..8fa230fc5d 100644 --- a/src/lib/usage/providerLimits.ts +++ b/src/lib/usage/providerLimits.ts @@ -26,6 +26,7 @@ import { USAGE_SUPPORTED_PROVIDERS } from "@/shared/constants/providers"; import { mergeProviderLimitsCacheEntry, toProviderLimitsCacheEntry } from "./providerLimitsCache"; import { getExecutor } from "@omniroute/open-sse/executors/index.ts"; import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts"; +import { cooldownUntilMs } from "@omniroute/open-sse/services/accountFallback.ts"; import { rotationGroupFor, serializeRefresh, @@ -459,66 +460,57 @@ function windowStillExhaustedAfterRealReset(value: unknown, nowMs: number): bool return resetMs > nowMs; } +/** + * Is an explicit cooldown still in the future? + * + * A rateLimitedUntil set by the upstream 429 handler is a hard statement and + * must never be overruled by a quota poll. + * + * Gate on the timestamp alone; lastErrorType stays irrelevant here. + */ +export function hasActiveCooldown( + connection: Pick, + now: number = Date.now() +): boolean { + if (!connection.rateLimitedUntil) return false; + // #3954: the rate_limited_until TEXT column holds an ISO string (dashboard/AUTH + // path) OR numeric epoch ms (setConnectionRateLimitUntil, the chat path). A bare + // `new Date(String(...))` yields Invalid Date for the numeric form, which read as + // "no cooldown" and let every poller wipe a chat-path-written lockout. Use the + // canonical parser connectionRecovery.ts already relies on. + const until = cooldownUntilMs(connection.rateLimitedUntil as string | number | null | undefined); + return Number.isFinite(until) && until > now; +} + +/** + * Whether a connection test may wipe the persisted error/cooldown state. + * + * A successful probe proves the CREDENTIAL is valid; it does not prove an + * exhausted quota window reopened — the probe is a cheap auth/models call that + * never touches the chat quota a weekly cap applies to. The credential-health + * scheduler runs that probe against every connection every 300s, so without this + * gate a weekly-capped connection was reset to `active` / `rateLimitedUntil=null` + * within 30s of every restart and dispatched straight back into the same 429. + * + * Same rule as `maybeClearRecoveredQuotaState`: a future `rateLimitedUntil` is + * the 429 handler's hard statement and no poller may overrule it. Once the + * window elapses, the next probe clears the state normally. + */ +export function shouldClearErrorStateOnValidProbe( + connection: Pick, + probeValid: boolean, + now: number = Date.now() +): boolean { + return probeValid && !hasActiveCooldown(connection, now); +} + export async function maybeClearRecoveredQuotaState( connection: ProviderConnectionLike, usage: JsonRecord ): Promise { if (!hasUsableQuota(usage)) return connection; if (isTerminalStatusForQuotaRecovery(connection.testStatus)) return connection; - if (connection.lastErrorType === "quota_exhausted") { - if ( - connection.lastErrorSource === CLAUDE_EXTRA_USAGE_ERROR_SOURCE && - isClaudeExtraUsageBlockEnabled(connection.provider, connection.providerSpecificData) && - isClaudeExtraUsageQueued(usage) - ) { - // Claude's pay-as-you-go extra-usage block is orthogonal to the - // session/weekly quota windows checked below: the upstream can report a - // fully recovered quota window while extraUsage.queued is still true. - // Only syncClaudeExtraUsageStateIfNeeded (buildClaudeExtraUsageConnectionUpdate) - // owns clearing this specific state — the general window-recovery logic - // below must not release it just because some quota window looks fresh. - return connection; - } - - const quotas = usage?.quotas; - if (isRecord(quotas)) { - // Honor the REAL per-window resetAt from the freshly fetched quota - // instead of the synthetic cooldown persisted at failure time (e.g. - // Claude's flat 1h SUBSCRIPTION_QUOTA_COOLDOWN_MS when no upstream - // reset was parseable). Only stay locked if some window that governs - // this connection's quota is still demonstrably exhausted. - const anyStillBlocking = Object.values(quotas).some((value) => - windowStillExhaustedAfterRealReset(value, Date.now()) - ); - if (anyStillBlocking) return connection; - } else if ( - connection.rateLimitedUntil && - new Date(connection.rateLimitedUntil).getTime() > Date.now() - ) { - // No quota object at all (degraded/failed fetch shape) — fall back to - // the previous synthetic-cooldown guard. - return connection; - } - } else if ( - connection.rateLimitedUntil && - new Date(connection.rateLimitedUntil).getTime() > Date.now() - ) { - // Universal fallback guard for every lastErrorType other than - // "quota_exhausted" (which gets the more precise per-window check above, - // and may legitimately release early once the REAL window has reset even - // while a synthetic rateLimitedUntil is still in the future). A future - // rateLimitedUntil is a hard statement made by the 429/error handler that - // persisted it (src/sse/services/auth.ts, src/app/api/providers/[id]/test/ - // route.ts) — no quota poll finding *some* usable window elsewhere should - // be able to overrule it. Before this fix, ANY lastErrorType other than - // "quota_exhausted" skipped straight to hasTransientState/ - // clearRecoveredProviderState() below with no rateLimitedUntil check at - // all, so a multi-day cooldown (observed: 146h, Z.AI weekly quota) got - // cleared on the very next quota sync a few minutes later — a - // self-restart/burn loop that kept burning real upstream calls against a - // known-exhausted connection (#11277). - return connection; - } + if (hasActiveCooldown(connection)) return connection; const hasTransientState = connection.testStatus === "unavailable" || diff --git a/tests/unit/repro-zai-cooldown-cleared-by-connection-test.test.ts b/tests/unit/repro-zai-cooldown-cleared-by-connection-test.test.ts new file mode 100644 index 0000000000..94768373e6 --- /dev/null +++ b/tests/unit/repro-zai-cooldown-cleared-by-connection-test.test.ts @@ -0,0 +1,87 @@ +/** + * Regression: the connection TEST path cleared a still-active cooldown. + * + * Sibling of repro-zai-cooldown-cleared-by-quota-poll.test.ts — same symptom, + * a different writer. testSingleConnection() (src/app/api/providers/[id]/test/ + * route.ts) built its update payload as: + * + * testStatus: result.valid ? "active" : "error", + * rateLimitedUntil: result.valid ? null : connection.rateLimitedUntil || null, + * + * so ANY successful probe wiped the persisted cooldown. That probe is not a + * chat call — it is a cheap auth/models validation that never touches the chat + * quota a weekly cap applies to, so it succeeds even while the weekly window is + * exhausted. The credential-health scheduler (src/lib/credentialHealth/ + * scheduler.ts) runs it against every connection 30s after startup and every + * 300s thereafter. + * + * Observed in production (2026-08-23) right after deploying the ISO-reset / + * pre-skip / crash-clear patch: the GLM connection carried a valid future + * rate_limited_until, "[CredentialHealth] Testing 10/10 connections..." ran, + * and the row came back testStatus="active", rate_limited_until=NULL — so combo + * dispatched zai/glm-5.3 straight back into the same weekly 429. This writer + * alone defeats every other cooldown fix. + * + * The gate is shouldClearErrorStateOnValidProbe(): a future rateLimitedUntil is + * the 429 handler's hard statement and a credential probe may not overrule it. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { + hasActiveCooldown, + shouldClearErrorStateOnValidProbe, +} from "../../src/lib/usage/providerLimits.ts"; + +const HOUR_MS = 60 * 60 * 1000; +const NOW = Date.UTC(2026, 7, 23, 21, 47, 0); // 2026-08-23 21:47 UTC + +/** The production row: zai/glm-5.3 held until the weekly reset on 2026-08-29. */ +const GLM_COOLDOWN = { rateLimitedUntil: "2026-08-29T21:01:21.000Z" }; + +describe("connection test must not clear an active cooldown", () => { + it("keeps the GLM weekly cooldown when the credential probe succeeds", () => { + assert.equal(hasActiveCooldown(GLM_COOLDOWN, NOW), true); + assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, true, NOW), false); + }); + + it("keeps a cooldown that is only one second away from elapsing", () => { + const conn = { rateLimitedUntil: new Date(NOW + 1000).toISOString() }; + assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false); + }); + + it("clears the error state once the cooldown has elapsed", () => { + const conn = { rateLimitedUntil: new Date(NOW - 1000).toISOString() }; + assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true); + }); + + it("clears the error state at the exact reset instant", () => { + const conn = { rateLimitedUntil: new Date(NOW).toISOString() }; + assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true); + }); + + it("clears the error state for a connection with no cooldown", () => { + assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, true, NOW), true); + assert.equal( + shouldClearErrorStateOnValidProbe({ rateLimitedUntil: undefined }, true, NOW), + true + ); + }); + + it("never clears on a FAILED probe, cooldown or not", () => { + assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, false, NOW), false); + assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, false, NOW), false); + }); + + it("fails open on an unparseable timestamp so a broken value cannot strand a connection", () => { + const conn = { rateLimitedUntil: "not-a-date" }; + assert.equal(hasActiveCooldown(conn, NOW), false); + assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true); + }); + + it("honours a numeric-epoch timestamp (the chat path writes epoch ms)", () => { + const conn = { rateLimitedUntil: String(NOW + 146 * HOUR_MS) }; + assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false); + }); +}); diff --git a/tests/unit/startup-stale-cooldown-recovery.test.ts b/tests/unit/startup-stale-cooldown-recovery.test.ts index 3c637eae85..8adec926a2 100644 --- a/tests/unit/startup-stale-cooldown-recovery.test.ts +++ b/tests/unit/startup-stale-cooldown-recovery.test.ts @@ -1,15 +1,17 @@ /** - * TDD regression guard for issue #3625 (Part A). + * TDD regression guard for issue #3625 (Part A) and future quota cooldown preservation. * * After an unclean process crash (SIGKILL / large-body burst), provider - * connections can be left in the DB with a far-future `rate_limited_until` - * (stale exponential-backoff value). On restart, getProviderCredentials() - * skips those connections and Bottleneck queues time out at 120 s. + * connections can be left in the DB with expired transient cooldowns. + * On startup, scan `provider_connections` and clear stale transient + * cooldown fields for any non-terminal connection that has an EXPIRED or + * unparseable `rate_limited_until`. * - * The fix: on startup, scan `provider_connections` and clear transient - * cooldown fields for any non-terminal connection that has a - * `rate_limited_until` set (past *or* future). Terminal states - * (banned / expired / credits_exhausted) must not be touched. + * FUTURE timestamps (such as weekly/monthly quota cooldowns) MUST be + * preserved so that restarts/recreates do not wipe active cooldowns and + * immediately dispatch into upstream 429s. + * + * Terminal states (banned / expired / credits_exhausted) must not be touched. */ import test from "node:test"; import assert from "node:assert/strict"; @@ -55,51 +57,43 @@ test.after(async () => { // ─── helpers ──────────────────────────────────────────────────────────────── -/** Far-future epoch ms (simulates a crash-burst backoff). */ -const FAR_FUTURE = Date.now() + 60 * 60 * 1000; // +1 hour +/** Far-future epoch ms (simulates a multi-day quota reset or active cooldown). */ +const FAR_FUTURE = Date.now() + 6 * 24 * 60 * 60 * 1000; // +6 days -/** Slightly past timestamp (normal lazy expiry — also cleared on startup). */ +/** Slightly past timestamp (normal lazy expiry — cleared on startup). */ const JUST_PAST = Date.now() - 10_000; // -10 s // ─── tests ────────────────────────────────────────────────────────────────── -test("clearStaleCrashCooldowns clears far-future transient cooldown on restart", async () => { +test("clearStaleCrashCooldowns PRESERVES future transient cooldown on restart", async () => { const conn = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", - name: "Stale Cooldown", + name: "Future Cooldown", apiKey: "sk-test", }); - // Simulate crash-burst state: far-future cooldown, transient error fields await providersDb.updateProviderConnection(conn.id, { ...conn, rateLimitedUntil: new Date(FAR_FUTURE).toISOString(), testStatus: "unavailable", - lastError: "upstream timeout", - lastErrorType: "timeout", + lastError: "upstream weekly quota exhausted", + lastErrorType: "quota_exhausted", backoffLevel: 3, }); - // Verify pre-condition: connection has a far-future cooldown persisted const pre = await providersDb.getProviderConnectionById(conn.id); assert.ok( pre?.rateLimitedUntil && new Date(pre.rateLimitedUntil as string).getTime() > Date.now(), "connection should have a future rate_limited_until before recovery" ); - // Run startup recovery const result = providersDb.clearStaleCrashCooldowns(); + assert.equal(result.cleared, 0, "future cooldown must NOT be cleared on startup"); - assert.ok(result.cleared >= 1, `expected at least 1 cleared, got ${result.cleared}`); - - // Verify post-condition: cooldown is gone (cleanNulls strips null → undefined) const updated = await providersDb.getProviderConnectionById(conn.id); - assert.ok(!updated?.rateLimitedUntil, "rateLimitedUntil should be absent/falsy after recovery"); - assert.equal(updated?.testStatus, "active", "testStatus should be 'active' after recovery"); - assert.equal(updated?.backoffLevel, 0, "backoffLevel should be 0 after recovery"); - assert.ok(!updated?.lastError, "lastError should be absent/falsy after recovery"); - assert.ok(!updated?.lastErrorType, "lastErrorType should be absent/falsy after recovery"); + assert.ok(updated?.rateLimitedUntil, "future rateLimitedUntil must remain intact"); + assert.equal(updated?.testStatus, "unavailable", "testStatus should remain unavailable"); }); test("clearStaleCrashCooldowns clears past-dated transient cooldown on restart", async () => { @@ -144,14 +138,12 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (banned)", async ( const result = providersDb.clearStaleCrashCooldowns(); - // The banned connection must NOT be cleared const updated = await providersDb.getProviderConnectionById(conn.id); assert.equal(updated?.testStatus, "banned", "banned connection must not be touched"); assert.ok( updated?.rateLimitedUntil, "rate_limited_until on a banned connection must not be cleared" ); - // cleared count should be 0 (only the banned conn exists in this test) assert.equal(result.cleared, 0, "no transient connections to clear"); }); @@ -201,7 +193,6 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (credits_exhausted }); test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exist", async () => { - // Create a clean connection (no cooldown) await providersDb.createProviderConnection({ provider: "gemini", authType: "apikey", @@ -215,28 +206,29 @@ test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exi }); test("clearStaleCrashCooldowns handles mixed transient + terminal connections correctly", async () => { - // Transient — should be cleared - const transient1 = await providersDb.createProviderConnection({ + // Future transient — should be PRESERVED + const futureTransient = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", - name: "Transient 1", + name: "Future Transient", apiKey: "sk-t1", }); - await providersDb.updateProviderConnection(transient1.id, { - ...transient1, + await providersDb.updateProviderConnection(futureTransient.id, { + ...futureTransient, rateLimitedUntil: new Date(FAR_FUTURE).toISOString(), testStatus: "unavailable", backoffLevel: 2, }); - const transient2 = await providersDb.createProviderConnection({ + // Past transient — should be CLEARED + const pastTransient = await providersDb.createProviderConnection({ provider: "anthropic", authType: "apikey", - name: "Transient 2", + name: "Past Transient", apiKey: "sk-t2", }); - await providersDb.updateProviderConnection(transient2.id, { - ...transient2, + await providersDb.updateProviderConnection(pastTransient.id, { + ...pastTransient, rateLimitedUntil: new Date(JUST_PAST).toISOString(), testStatus: "unavailable", backoffLevel: 1, @@ -258,15 +250,15 @@ test("clearStaleCrashCooldowns handles mixed transient + terminal connections co const result = providersDb.clearStaleCrashCooldowns(); - assert.equal(result.cleared, 2, "exactly 2 transient connections cleared"); + assert.equal(result.cleared, 1, "only 1 past transient connection cleared"); - const updatedT1 = await providersDb.getProviderConnectionById(transient1.id); - assert.ok(!updatedT1?.rateLimitedUntil, "transient1 cooldown cleared"); - assert.equal(updatedT1?.testStatus, "active", "transient1 status active"); + const updatedFuture = await providersDb.getProviderConnectionById(futureTransient.id); + assert.ok(updatedFuture?.rateLimitedUntil, "future cooldown preserved"); + assert.equal(updatedFuture?.testStatus, "unavailable", "future transient status preserved"); - const updatedT2 = await providersDb.getProviderConnectionById(transient2.id); - assert.ok(!updatedT2?.rateLimitedUntil, "transient2 cooldown cleared"); - assert.equal(updatedT2?.testStatus, "active", "transient2 status active"); + const updatedPast = await providersDb.getProviderConnectionById(pastTransient.id); + assert.ok(!updatedPast?.rateLimitedUntil, "past transient cooldown cleared"); + assert.equal(updatedPast?.testStatus, "active", "past transient status active"); const updatedTerminal = await providersDb.getProviderConnectionById(terminal.id); assert.equal(updatedTerminal?.testStatus, "banned", "terminal connection untouched");