fix(resilience): preserve active cooldowns during recovery and probes (#11355)

Merged via consolidated batch validation (fix applied for a cross-PR interaction with #11360, both boarded in the same batch — see combo.ts reconciliation commit). Startup crash recovery cleared every non-terminal transient cooldown unconditionally, erasing legitimate multi-day weekly quota cooldowns on restart. Now only clears expired/unparseable ones. Own repro tests pass.
This commit is contained in:
sprintberlin
2026-08-24 17:12:30 +02:00
committed by GitHub
parent e1c2b347f9
commit 315b0a94e1
5 changed files with 251 additions and 117 deletions

View File

@@ -30,6 +30,7 @@ import { providerAllowsOptionalApiKey } from "@/shared/constants/providers";
import { shouldUseApiKeyConnectionTest } from "./webSessionTestDispatch";
import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHealth";
import { removeConnectionHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts";
import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits";
import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation";
import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth";
import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult";
@@ -1082,23 +1083,46 @@ export async function testSingleConnection(connectionId: string, validationModel
terminalTestStatuses.has(String(diagnosis.code ?? diagnosis.type ?? "").toLowerCase());
const testFailureCooldownMs = result.valid ? 0 : 30_000; // 30s retry window
// A successful credential probe proves the KEY is valid. It does NOT prove the
// quota window reopened: the probe is a cheap auth/models call that never touches
// the chat quota a weekly cap applies to. Clearing an ACTIVE cooldown here — which
// the credential-health scheduler triggers for every connection every 300s — put
// `zai/glm-5.3` back to `active` / `rate_limited_until = NULL` within 30s of every
// restart, so combo dispatched it straight into the same weekly 429. Same rule as
// maybeClearRecoveredQuotaState: a future rateLimitedUntil is the 429 handler's
// hard statement and no poller may overrule it. Once it elapses, the next probe
// clears it normally.
const clearErrorState = shouldClearErrorStateOnValidProbe(
connection as { rateLimitedUntil?: string | null },
result.valid
);
const updateData: Record<string, any> = {
testStatus: result.valid ? "active" : "error",
lastError: result.valid ? null : result.error,
lastErrorAt: result.valid ? null : now,
testStatus: clearErrorState ? "active" : result.valid ? connection.testStatus : "error",
lastError: clearErrorState ? null : result.valid ? connection.lastError : result.error,
lastErrorAt: clearErrorState ? null : result.valid ? connection.lastErrorAt : now,
lastTested: now,
lastErrorType: result.valid ? null : diagnosis.type,
lastErrorSource: result.valid ? null : diagnosis.source,
errorCode: result.valid ? null : diagnosis.code || result.statusCode || null,
rateLimitedUntil:
result.valid || isTerminalFailure
? result.valid
? null
: connection.rateLimitedUntil || null
: new Date(Date.now() + testFailureCooldownMs).toISOString(),
lastErrorType: clearErrorState ? null : result.valid ? connection.lastErrorType : diagnosis.type,
lastErrorSource: clearErrorState
? null
: result.valid
? connection.lastErrorSource
: diagnosis.source,
errorCode: clearErrorState
? null
: result.valid
? connection.errorCode
: diagnosis.code || result.statusCode || null,
rateLimitedUntil: clearErrorState
? null
: isTerminalFailure
? connection.rateLimitedUntil || null
: result.valid
? connection.rateLimitedUntil || null
: new Date(Date.now() + testFailureCooldownMs).toISOString(),
};
if (result.valid) {
if (clearErrorState) {
updateData.backoffLevel = 0;
const psd = connection?.providerSpecificData as Record<string, unknown> | undefined;

View File

@@ -124,6 +124,27 @@ export function getEffectiveQuotaUsage(
return used;
}
/**
* Normalize a persisted `rate_limited_until` to epoch ms.
*
* The column is written in two shapes: epoch ms by `setConnectionRateLimitUntil`
* (the chat path) and an ISO-8601 string by `updateProviderConnection` (the
* dashboard/AUTH path). Returns null when the value is absent or unparseable —
* callers treat that as "no usable deadline".
*/
function parseCooldownUntilMs(value: string | number | null | undefined): number | null {
if (value == null || value === "") return null;
if (typeof value === "number") return Number.isFinite(value) ? value : null;
const raw = String(value).trim();
if (raw === "") return null;
if (/^\d+$/.test(raw)) {
const numeric = Number(raw);
return Number.isFinite(numeric) ? numeric : null;
}
const parsed = Date.parse(raw);
return Number.isFinite(parsed) ? parsed : null;
}
/**
* T05: Startup crash-recovery — clear stale transient connection cooldowns.
*
@@ -138,9 +159,19 @@ export function getEffectiveQuotaUsage(
* - Only connections with `rate_limited_until IS NOT NULL` are touched.
* - Terminal states (`banned`, `expired`, `credits_exhausted`) are skipped —
* those require a deliberate credential change or operator reset.
* - Past timestamps are also cleared: they are already expired in the lazy
* - Past timestamps are cleared: they are already expired in the lazy
* expiry sense, but clearing them resets `backoffLevel` / transient error
* fields so the connection gets a clean slate on this fresh process.
* fields so the connection gets a clean slate on this fresh process. An
* unparseable timestamp is treated the same way — it can never expire
* lazily, so leaving it would strand the connection forever.
* - FUTURE timestamps are NEVER cleared. Clearing them was the original
* behaviour and it wiped legitimate multi-day quota cooldowns on every
* container recreate: a GLM weekly cap persisted until 2026-08-29 came
* back `active` with `rate_limited_until = NULL`, combo dispatched it
* immediately, and the connection re-earned a real upstream 429. A stale
* crash-backoff value is bounded by the engine's own cooldown cap, so
* honouring it costs at most that window — far less than burning quota
* against an upstream that is provably exhausted.
*
* Must be called once, early in the startup sequence, before any request
* is handled. Returns the number of connections that were cleared.
@@ -148,6 +179,7 @@ export function getEffectiveQuotaUsage(
export function clearStaleCrashCooldowns(): { cleared: number } {
const db = getDbInstance() as unknown as DbLike;
const now = new Date().toISOString();
const nowMs = Date.now();
// Fetch all connections that have a rate_limited_until set and are NOT in
// a terminal state. We do the terminal-status filter in JS to reuse the
@@ -156,13 +188,20 @@ export function clearStaleCrashCooldowns(): { cleared: number } {
const rows = db
.prepare(
`SELECT id, test_status FROM provider_connections WHERE rate_limited_until IS NOT NULL`
`SELECT id, test_status, rate_limited_until FROM provider_connections WHERE rate_limited_until IS NOT NULL`
)
.all() as Array<{ id: string; test_status: string | null }>;
.all() as Array<{
id: string;
test_status: string | null;
rate_limited_until: string | number | null;
}>;
const toReset = rows.filter((r) => {
const status = (r.test_status || "").trim().toLowerCase();
return !TERMINAL_STATUSES.has(status);
if (TERMINAL_STATUSES.has(status)) return false;
const untilMs = parseCooldownUntilMs(r.rate_limited_until);
// Unparseable → clear (cannot expire lazily). Future → keep.
return untilMs === null || untilMs <= nowMs;
});
if (toReset.length === 0) return { cleared: 0 };

View File

@@ -26,6 +26,7 @@ import { USAGE_SUPPORTED_PROVIDERS } from "@/shared/constants/providers";
import { mergeProviderLimitsCacheEntry, toProviderLimitsCacheEntry } from "./providerLimitsCache";
import { getExecutor } from "@omniroute/open-sse/executors/index.ts";
import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts";
import { cooldownUntilMs } from "@omniroute/open-sse/services/accountFallback.ts";
import {
rotationGroupFor,
serializeRefresh,
@@ -459,66 +460,57 @@ function windowStillExhaustedAfterRealReset(value: unknown, nowMs: number): bool
return resetMs > nowMs;
}
/**
* Is an explicit cooldown still in the future?
*
* A rateLimitedUntil set by the upstream 429 handler is a hard statement and
* must never be overruled by a quota poll.
*
* Gate on the timestamp alone; lastErrorType stays irrelevant here.
*/
export function hasActiveCooldown(
connection: Pick<ProviderConnectionLike, "rateLimitedUntil">,
now: number = Date.now()
): boolean {
if (!connection.rateLimitedUntil) return false;
// #3954: the rate_limited_until TEXT column holds an ISO string (dashboard/AUTH
// path) OR numeric epoch ms (setConnectionRateLimitUntil, the chat path). A bare
// `new Date(String(...))` yields Invalid Date for the numeric form, which read as
// "no cooldown" and let every poller wipe a chat-path-written lockout. Use the
// canonical parser connectionRecovery.ts already relies on.
const until = cooldownUntilMs(connection.rateLimitedUntil as string | number | null | undefined);
return Number.isFinite(until) && until > now;
}
/**
* Whether a connection test may wipe the persisted error/cooldown state.
*
* A successful probe proves the CREDENTIAL is valid; it does not prove an
* exhausted quota window reopened — the probe is a cheap auth/models call that
* never touches the chat quota a weekly cap applies to. The credential-health
* scheduler runs that probe against every connection every 300s, so without this
* gate a weekly-capped connection was reset to `active` / `rateLimitedUntil=null`
* within 30s of every restart and dispatched straight back into the same 429.
*
* Same rule as `maybeClearRecoveredQuotaState`: a future `rateLimitedUntil` is
* the 429 handler's hard statement and no poller may overrule it. Once the
* window elapses, the next probe clears the state normally.
*/
export function shouldClearErrorStateOnValidProbe(
connection: Pick<ProviderConnectionLike, "rateLimitedUntil">,
probeValid: boolean,
now: number = Date.now()
): boolean {
return probeValid && !hasActiveCooldown(connection, now);
}
export async function maybeClearRecoveredQuotaState(
connection: ProviderConnectionLike,
usage: JsonRecord
): Promise<ProviderConnectionLike> {
if (!hasUsableQuota(usage)) return connection;
if (isTerminalStatusForQuotaRecovery(connection.testStatus)) return connection;
if (connection.lastErrorType === "quota_exhausted") {
if (
connection.lastErrorSource === CLAUDE_EXTRA_USAGE_ERROR_SOURCE &&
isClaudeExtraUsageBlockEnabled(connection.provider, connection.providerSpecificData) &&
isClaudeExtraUsageQueued(usage)
) {
// Claude's pay-as-you-go extra-usage block is orthogonal to the
// session/weekly quota windows checked below: the upstream can report a
// fully recovered quota window while extraUsage.queued is still true.
// Only syncClaudeExtraUsageStateIfNeeded (buildClaudeExtraUsageConnectionUpdate)
// owns clearing this specific state — the general window-recovery logic
// below must not release it just because some quota window looks fresh.
return connection;
}
const quotas = usage?.quotas;
if (isRecord(quotas)) {
// Honor the REAL per-window resetAt from the freshly fetched quota
// instead of the synthetic cooldown persisted at failure time (e.g.
// Claude's flat 1h SUBSCRIPTION_QUOTA_COOLDOWN_MS when no upstream
// reset was parseable). Only stay locked if some window that governs
// this connection's quota is still demonstrably exhausted.
const anyStillBlocking = Object.values(quotas).some((value) =>
windowStillExhaustedAfterRealReset(value, Date.now())
);
if (anyStillBlocking) return connection;
} else if (
connection.rateLimitedUntil &&
new Date(connection.rateLimitedUntil).getTime() > Date.now()
) {
// No quota object at all (degraded/failed fetch shape) — fall back to
// the previous synthetic-cooldown guard.
return connection;
}
} else if (
connection.rateLimitedUntil &&
new Date(connection.rateLimitedUntil).getTime() > Date.now()
) {
// Universal fallback guard for every lastErrorType other than
// "quota_exhausted" (which gets the more precise per-window check above,
// and may legitimately release early once the REAL window has reset even
// while a synthetic rateLimitedUntil is still in the future). A future
// rateLimitedUntil is a hard statement made by the 429/error handler that
// persisted it (src/sse/services/auth.ts, src/app/api/providers/[id]/test/
// route.ts) — no quota poll finding *some* usable window elsewhere should
// be able to overrule it. Before this fix, ANY lastErrorType other than
// "quota_exhausted" skipped straight to hasTransientState/
// clearRecoveredProviderState() below with no rateLimitedUntil check at
// all, so a multi-day cooldown (observed: 146h, Z.AI weekly quota) got
// cleared on the very next quota sync a few minutes later — a
// self-restart/burn loop that kept burning real upstream calls against a
// known-exhausted connection (#11277).
return connection;
}
if (hasActiveCooldown(connection)) return connection;
const hasTransientState =
connection.testStatus === "unavailable" ||

View File

@@ -0,0 +1,87 @@
/**
* Regression: the connection TEST path cleared a still-active cooldown.
*
* Sibling of repro-zai-cooldown-cleared-by-quota-poll.test.ts — same symptom,
* a different writer. testSingleConnection() (src/app/api/providers/[id]/test/
* route.ts) built its update payload as:
*
* testStatus: result.valid ? "active" : "error",
* rateLimitedUntil: result.valid ? null : connection.rateLimitedUntil || null,
*
* so ANY successful probe wiped the persisted cooldown. That probe is not a
* chat call — it is a cheap auth/models validation that never touches the chat
* quota a weekly cap applies to, so it succeeds even while the weekly window is
* exhausted. The credential-health scheduler (src/lib/credentialHealth/
* scheduler.ts) runs it against every connection 30s after startup and every
* 300s thereafter.
*
* Observed in production (2026-08-23) right after deploying the ISO-reset /
* pre-skip / crash-clear patch: the GLM connection carried a valid future
* rate_limited_until, "[CredentialHealth] Testing 10/10 connections..." ran,
* and the row came back testStatus="active", rate_limited_until=NULL — so combo
* dispatched zai/glm-5.3 straight back into the same weekly 429. This writer
* alone defeats every other cooldown fix.
*
* The gate is shouldClearErrorStateOnValidProbe(): a future rateLimitedUntil is
* the 429 handler's hard statement and a credential probe may not overrule it.
*/
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import {
hasActiveCooldown,
shouldClearErrorStateOnValidProbe,
} from "../../src/lib/usage/providerLimits.ts";
const HOUR_MS = 60 * 60 * 1000;
const NOW = Date.UTC(2026, 7, 23, 21, 47, 0); // 2026-08-23 21:47 UTC
/** The production row: zai/glm-5.3 held until the weekly reset on 2026-08-29. */
const GLM_COOLDOWN = { rateLimitedUntil: "2026-08-29T21:01:21.000Z" };
describe("connection test must not clear an active cooldown", () => {
it("keeps the GLM weekly cooldown when the credential probe succeeds", () => {
assert.equal(hasActiveCooldown(GLM_COOLDOWN, NOW), true);
assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, true, NOW), false);
});
it("keeps a cooldown that is only one second away from elapsing", () => {
const conn = { rateLimitedUntil: new Date(NOW + 1000).toISOString() };
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false);
});
it("clears the error state once the cooldown has elapsed", () => {
const conn = { rateLimitedUntil: new Date(NOW - 1000).toISOString() };
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
});
it("clears the error state at the exact reset instant", () => {
const conn = { rateLimitedUntil: new Date(NOW).toISOString() };
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
});
it("clears the error state for a connection with no cooldown", () => {
assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, true, NOW), true);
assert.equal(
shouldClearErrorStateOnValidProbe({ rateLimitedUntil: undefined }, true, NOW),
true
);
});
it("never clears on a FAILED probe, cooldown or not", () => {
assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, false, NOW), false);
assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, false, NOW), false);
});
it("fails open on an unparseable timestamp so a broken value cannot strand a connection", () => {
const conn = { rateLimitedUntil: "not-a-date" };
assert.equal(hasActiveCooldown(conn, NOW), false);
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
});
it("honours a numeric-epoch timestamp (the chat path writes epoch ms)", () => {
const conn = { rateLimitedUntil: String(NOW + 146 * HOUR_MS) };
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false);
});
});

View File

@@ -1,15 +1,17 @@
/**
* TDD regression guard for issue #3625 (Part A).
* TDD regression guard for issue #3625 (Part A) and future quota cooldown preservation.
*
* After an unclean process crash (SIGKILL / large-body burst), provider
* connections can be left in the DB with a far-future `rate_limited_until`
* (stale exponential-backoff value). On restart, getProviderCredentials()
* skips those connections and Bottleneck queues time out at 120 s.
* connections can be left in the DB with expired transient cooldowns.
* On startup, scan `provider_connections` and clear stale transient
* cooldown fields for any non-terminal connection that has an EXPIRED or
* unparseable `rate_limited_until`.
*
* The fix: on startup, scan `provider_connections` and clear transient
* cooldown fields for any non-terminal connection that has a
* `rate_limited_until` set (past *or* future). Terminal states
* (banned / expired / credits_exhausted) must not be touched.
* FUTURE timestamps (such as weekly/monthly quota cooldowns) MUST be
* preserved so that restarts/recreates do not wipe active cooldowns and
* immediately dispatch into upstream 429s.
*
* Terminal states (banned / expired / credits_exhausted) must not be touched.
*/
import test from "node:test";
import assert from "node:assert/strict";
@@ -55,51 +57,43 @@ test.after(async () => {
// ─── helpers ────────────────────────────────────────────────────────────────
/** Far-future epoch ms (simulates a crash-burst backoff). */
const FAR_FUTURE = Date.now() + 60 * 60 * 1000; // +1 hour
/** Far-future epoch ms (simulates a multi-day quota reset or active cooldown). */
const FAR_FUTURE = Date.now() + 6 * 24 * 60 * 60 * 1000; // +6 days
/** Slightly past timestamp (normal lazy expiry — also cleared on startup). */
/** Slightly past timestamp (normal lazy expiry — cleared on startup). */
const JUST_PAST = Date.now() - 10_000; // -10 s
// ─── tests ──────────────────────────────────────────────────────────────────
test("clearStaleCrashCooldowns clears far-future transient cooldown on restart", async () => {
test("clearStaleCrashCooldowns PRESERVES future transient cooldown on restart", async () => {
const conn = await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Stale Cooldown",
name: "Future Cooldown",
apiKey: "sk-test",
});
// Simulate crash-burst state: far-future cooldown, transient error fields
await providersDb.updateProviderConnection(conn.id, {
...conn,
rateLimitedUntil: new Date(FAR_FUTURE).toISOString(),
testStatus: "unavailable",
lastError: "upstream timeout",
lastErrorType: "timeout",
lastError: "upstream weekly quota exhausted",
lastErrorType: "quota_exhausted",
backoffLevel: 3,
});
// Verify pre-condition: connection has a far-future cooldown persisted
const pre = await providersDb.getProviderConnectionById(conn.id);
assert.ok(
pre?.rateLimitedUntil && new Date(pre.rateLimitedUntil as string).getTime() > Date.now(),
"connection should have a future rate_limited_until before recovery"
);
// Run startup recovery
const result = providersDb.clearStaleCrashCooldowns();
assert.equal(result.cleared, 0, "future cooldown must NOT be cleared on startup");
assert.ok(result.cleared >= 1, `expected at least 1 cleared, got ${result.cleared}`);
// Verify post-condition: cooldown is gone (cleanNulls strips null → undefined)
const updated = await providersDb.getProviderConnectionById(conn.id);
assert.ok(!updated?.rateLimitedUntil, "rateLimitedUntil should be absent/falsy after recovery");
assert.equal(updated?.testStatus, "active", "testStatus should be 'active' after recovery");
assert.equal(updated?.backoffLevel, 0, "backoffLevel should be 0 after recovery");
assert.ok(!updated?.lastError, "lastError should be absent/falsy after recovery");
assert.ok(!updated?.lastErrorType, "lastErrorType should be absent/falsy after recovery");
assert.ok(updated?.rateLimitedUntil, "future rateLimitedUntil must remain intact");
assert.equal(updated?.testStatus, "unavailable", "testStatus should remain unavailable");
});
test("clearStaleCrashCooldowns clears past-dated transient cooldown on restart", async () => {
@@ -144,14 +138,12 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (banned)", async (
const result = providersDb.clearStaleCrashCooldowns();
// The banned connection must NOT be cleared
const updated = await providersDb.getProviderConnectionById(conn.id);
assert.equal(updated?.testStatus, "banned", "banned connection must not be touched");
assert.ok(
updated?.rateLimitedUntil,
"rate_limited_until on a banned connection must not be cleared"
);
// cleared count should be 0 (only the banned conn exists in this test)
assert.equal(result.cleared, 0, "no transient connections to clear");
});
@@ -201,7 +193,6 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (credits_exhausted
});
test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exist", async () => {
// Create a clean connection (no cooldown)
await providersDb.createProviderConnection({
provider: "gemini",
authType: "apikey",
@@ -215,28 +206,29 @@ test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exi
});
test("clearStaleCrashCooldowns handles mixed transient + terminal connections correctly", async () => {
// Transient — should be cleared
const transient1 = await providersDb.createProviderConnection({
// Future transient — should be PRESERVED
const futureTransient = await providersDb.createProviderConnection({
provider: "openai",
authType: "apikey",
name: "Transient 1",
name: "Future Transient",
apiKey: "sk-t1",
});
await providersDb.updateProviderConnection(transient1.id, {
...transient1,
await providersDb.updateProviderConnection(futureTransient.id, {
...futureTransient,
rateLimitedUntil: new Date(FAR_FUTURE).toISOString(),
testStatus: "unavailable",
backoffLevel: 2,
});
const transient2 = await providersDb.createProviderConnection({
// Past transient — should be CLEARED
const pastTransient = await providersDb.createProviderConnection({
provider: "anthropic",
authType: "apikey",
name: "Transient 2",
name: "Past Transient",
apiKey: "sk-t2",
});
await providersDb.updateProviderConnection(transient2.id, {
...transient2,
await providersDb.updateProviderConnection(pastTransient.id, {
...pastTransient,
rateLimitedUntil: new Date(JUST_PAST).toISOString(),
testStatus: "unavailable",
backoffLevel: 1,
@@ -258,15 +250,15 @@ test("clearStaleCrashCooldowns handles mixed transient + terminal connections co
const result = providersDb.clearStaleCrashCooldowns();
assert.equal(result.cleared, 2, "exactly 2 transient connections cleared");
assert.equal(result.cleared, 1, "only 1 past transient connection cleared");
const updatedT1 = await providersDb.getProviderConnectionById(transient1.id);
assert.ok(!updatedT1?.rateLimitedUntil, "transient1 cooldown cleared");
assert.equal(updatedT1?.testStatus, "active", "transient1 status active");
const updatedFuture = await providersDb.getProviderConnectionById(futureTransient.id);
assert.ok(updatedFuture?.rateLimitedUntil, "future cooldown preserved");
assert.equal(updatedFuture?.testStatus, "unavailable", "future transient status preserved");
const updatedT2 = await providersDb.getProviderConnectionById(transient2.id);
assert.ok(!updatedT2?.rateLimitedUntil, "transient2 cooldown cleared");
assert.equal(updatedT2?.testStatus, "active", "transient2 status active");
const updatedPast = await providersDb.getProviderConnectionById(pastTransient.id);
assert.ok(!updatedPast?.rateLimitedUntil, "past transient cooldown cleared");
assert.equal(updatedPast?.testStatus, "active", "past transient status active");
const updatedTerminal = await providersDb.getProviderConnectionById(terminal.id);
assert.equal(updatedTerminal?.testStatus, "banned", "terminal connection untouched");