mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-25 08:32:11 +03:00
fix(resilience): preserve active cooldowns during recovery and probes (#11355)
Merged via consolidated batch validation (fix applied for a cross-PR interaction with #11360, both boarded in the same batch — see combo.ts reconciliation commit). Startup crash recovery cleared every non-terminal transient cooldown unconditionally, erasing legitimate multi-day weekly quota cooldowns on restart. Now only clears expired/unparseable ones. Own repro tests pass.
This commit is contained in:
@@ -30,6 +30,7 @@ import { providerAllowsOptionalApiKey } from "@/shared/constants/providers";
|
||||
import { shouldUseApiKeyConnectionTest } from "./webSessionTestDispatch";
|
||||
import { testCodexAppServerConnection, makeDiagnosis } from "./codexAppServerHealth";
|
||||
import { removeConnectionHealth } from "@omniroute/open-sse/services/apiKeyRotator.ts";
|
||||
import { shouldClearErrorStateOnValidProbe } from "@/lib/usage/providerLimits";
|
||||
import { isConnectionUnavailableToAuxiliaryActivity } from "@/lib/exclusiveLeaseIsolation";
|
||||
import { classifyAmbiguousOrAuthError, type ClassifyFailureArgs } from "./mistralAmbiguousAuth";
|
||||
import { buildApiKeyConnectionTestResult } from "./apiKeyTestResult";
|
||||
@@ -1082,23 +1083,46 @@ export async function testSingleConnection(connectionId: string, validationModel
|
||||
terminalTestStatuses.has(String(diagnosis.code ?? diagnosis.type ?? "").toLowerCase());
|
||||
const testFailureCooldownMs = result.valid ? 0 : 30_000; // 30s retry window
|
||||
|
||||
// A successful credential probe proves the KEY is valid. It does NOT prove the
|
||||
// quota window reopened: the probe is a cheap auth/models call that never touches
|
||||
// the chat quota a weekly cap applies to. Clearing an ACTIVE cooldown here — which
|
||||
// the credential-health scheduler triggers for every connection every 300s — put
|
||||
// `zai/glm-5.3` back to `active` / `rate_limited_until = NULL` within 30s of every
|
||||
// restart, so combo dispatched it straight into the same weekly 429. Same rule as
|
||||
// maybeClearRecoveredQuotaState: a future rateLimitedUntil is the 429 handler's
|
||||
// hard statement and no poller may overrule it. Once it elapses, the next probe
|
||||
// clears it normally.
|
||||
const clearErrorState = shouldClearErrorStateOnValidProbe(
|
||||
connection as { rateLimitedUntil?: string | null },
|
||||
result.valid
|
||||
);
|
||||
|
||||
const updateData: Record<string, any> = {
|
||||
testStatus: result.valid ? "active" : "error",
|
||||
lastError: result.valid ? null : result.error,
|
||||
lastErrorAt: result.valid ? null : now,
|
||||
testStatus: clearErrorState ? "active" : result.valid ? connection.testStatus : "error",
|
||||
lastError: clearErrorState ? null : result.valid ? connection.lastError : result.error,
|
||||
lastErrorAt: clearErrorState ? null : result.valid ? connection.lastErrorAt : now,
|
||||
lastTested: now,
|
||||
lastErrorType: result.valid ? null : diagnosis.type,
|
||||
lastErrorSource: result.valid ? null : diagnosis.source,
|
||||
errorCode: result.valid ? null : diagnosis.code || result.statusCode || null,
|
||||
rateLimitedUntil:
|
||||
result.valid || isTerminalFailure
|
||||
? result.valid
|
||||
? null
|
||||
: connection.rateLimitedUntil || null
|
||||
: new Date(Date.now() + testFailureCooldownMs).toISOString(),
|
||||
lastErrorType: clearErrorState ? null : result.valid ? connection.lastErrorType : diagnosis.type,
|
||||
lastErrorSource: clearErrorState
|
||||
? null
|
||||
: result.valid
|
||||
? connection.lastErrorSource
|
||||
: diagnosis.source,
|
||||
errorCode: clearErrorState
|
||||
? null
|
||||
: result.valid
|
||||
? connection.errorCode
|
||||
: diagnosis.code || result.statusCode || null,
|
||||
rateLimitedUntil: clearErrorState
|
||||
? null
|
||||
: isTerminalFailure
|
||||
? connection.rateLimitedUntil || null
|
||||
: result.valid
|
||||
? connection.rateLimitedUntil || null
|
||||
: new Date(Date.now() + testFailureCooldownMs).toISOString(),
|
||||
};
|
||||
|
||||
if (result.valid) {
|
||||
if (clearErrorState) {
|
||||
updateData.backoffLevel = 0;
|
||||
|
||||
const psd = connection?.providerSpecificData as Record<string, unknown> | undefined;
|
||||
|
||||
@@ -124,6 +124,27 @@ export function getEffectiveQuotaUsage(
|
||||
return used;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a persisted `rate_limited_until` to epoch ms.
|
||||
*
|
||||
* The column is written in two shapes: epoch ms by `setConnectionRateLimitUntil`
|
||||
* (the chat path) and an ISO-8601 string by `updateProviderConnection` (the
|
||||
* dashboard/AUTH path). Returns null when the value is absent or unparseable —
|
||||
* callers treat that as "no usable deadline".
|
||||
*/
|
||||
function parseCooldownUntilMs(value: string | number | null | undefined): number | null {
|
||||
if (value == null || value === "") return null;
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
||||
const raw = String(value).trim();
|
||||
if (raw === "") return null;
|
||||
if (/^\d+$/.test(raw)) {
|
||||
const numeric = Number(raw);
|
||||
return Number.isFinite(numeric) ? numeric : null;
|
||||
}
|
||||
const parsed = Date.parse(raw);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* T05: Startup crash-recovery — clear stale transient connection cooldowns.
|
||||
*
|
||||
@@ -138,9 +159,19 @@ export function getEffectiveQuotaUsage(
|
||||
* - Only connections with `rate_limited_until IS NOT NULL` are touched.
|
||||
* - Terminal states (`banned`, `expired`, `credits_exhausted`) are skipped —
|
||||
* those require a deliberate credential change or operator reset.
|
||||
* - Past timestamps are also cleared: they are already expired in the lazy
|
||||
* - Past timestamps are cleared: they are already expired in the lazy
|
||||
* expiry sense, but clearing them resets `backoffLevel` / transient error
|
||||
* fields so the connection gets a clean slate on this fresh process.
|
||||
* fields so the connection gets a clean slate on this fresh process. An
|
||||
* unparseable timestamp is treated the same way — it can never expire
|
||||
* lazily, so leaving it would strand the connection forever.
|
||||
* - FUTURE timestamps are NEVER cleared. Clearing them was the original
|
||||
* behaviour and it wiped legitimate multi-day quota cooldowns on every
|
||||
* container recreate: a GLM weekly cap persisted until 2026-08-29 came
|
||||
* back `active` with `rate_limited_until = NULL`, combo dispatched it
|
||||
* immediately, and the connection re-earned a real upstream 429. A stale
|
||||
* crash-backoff value is bounded by the engine's own cooldown cap, so
|
||||
* honouring it costs at most that window — far less than burning quota
|
||||
* against an upstream that is provably exhausted.
|
||||
*
|
||||
* Must be called once, early in the startup sequence, before any request
|
||||
* is handled. Returns the number of connections that were cleared.
|
||||
@@ -148,6 +179,7 @@ export function getEffectiveQuotaUsage(
|
||||
export function clearStaleCrashCooldowns(): { cleared: number } {
|
||||
const db = getDbInstance() as unknown as DbLike;
|
||||
const now = new Date().toISOString();
|
||||
const nowMs = Date.now();
|
||||
|
||||
// Fetch all connections that have a rate_limited_until set and are NOT in
|
||||
// a terminal state. We do the terminal-status filter in JS to reuse the
|
||||
@@ -156,13 +188,20 @@ export function clearStaleCrashCooldowns(): { cleared: number } {
|
||||
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT id, test_status FROM provider_connections WHERE rate_limited_until IS NOT NULL`
|
||||
`SELECT id, test_status, rate_limited_until FROM provider_connections WHERE rate_limited_until IS NOT NULL`
|
||||
)
|
||||
.all() as Array<{ id: string; test_status: string | null }>;
|
||||
.all() as Array<{
|
||||
id: string;
|
||||
test_status: string | null;
|
||||
rate_limited_until: string | number | null;
|
||||
}>;
|
||||
|
||||
const toReset = rows.filter((r) => {
|
||||
const status = (r.test_status || "").trim().toLowerCase();
|
||||
return !TERMINAL_STATUSES.has(status);
|
||||
if (TERMINAL_STATUSES.has(status)) return false;
|
||||
const untilMs = parseCooldownUntilMs(r.rate_limited_until);
|
||||
// Unparseable → clear (cannot expire lazily). Future → keep.
|
||||
return untilMs === null || untilMs <= nowMs;
|
||||
});
|
||||
|
||||
if (toReset.length === 0) return { cleared: 0 };
|
||||
|
||||
@@ -26,6 +26,7 @@ import { USAGE_SUPPORTED_PROVIDERS } from "@/shared/constants/providers";
|
||||
import { mergeProviderLimitsCacheEntry, toProviderLimitsCacheEntry } from "./providerLimitsCache";
|
||||
import { getExecutor } from "@omniroute/open-sse/executors/index.ts";
|
||||
import { getUsageForProvider } from "@omniroute/open-sse/services/usage.ts";
|
||||
import { cooldownUntilMs } from "@omniroute/open-sse/services/accountFallback.ts";
|
||||
import {
|
||||
rotationGroupFor,
|
||||
serializeRefresh,
|
||||
@@ -459,66 +460,57 @@ function windowStillExhaustedAfterRealReset(value: unknown, nowMs: number): bool
|
||||
return resetMs > nowMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is an explicit cooldown still in the future?
|
||||
*
|
||||
* A rateLimitedUntil set by the upstream 429 handler is a hard statement and
|
||||
* must never be overruled by a quota poll.
|
||||
*
|
||||
* Gate on the timestamp alone; lastErrorType stays irrelevant here.
|
||||
*/
|
||||
export function hasActiveCooldown(
|
||||
connection: Pick<ProviderConnectionLike, "rateLimitedUntil">,
|
||||
now: number = Date.now()
|
||||
): boolean {
|
||||
if (!connection.rateLimitedUntil) return false;
|
||||
// #3954: the rate_limited_until TEXT column holds an ISO string (dashboard/AUTH
|
||||
// path) OR numeric epoch ms (setConnectionRateLimitUntil, the chat path). A bare
|
||||
// `new Date(String(...))` yields Invalid Date for the numeric form, which read as
|
||||
// "no cooldown" and let every poller wipe a chat-path-written lockout. Use the
|
||||
// canonical parser connectionRecovery.ts already relies on.
|
||||
const until = cooldownUntilMs(connection.rateLimitedUntil as string | number | null | undefined);
|
||||
return Number.isFinite(until) && until > now;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a connection test may wipe the persisted error/cooldown state.
|
||||
*
|
||||
* A successful probe proves the CREDENTIAL is valid; it does not prove an
|
||||
* exhausted quota window reopened — the probe is a cheap auth/models call that
|
||||
* never touches the chat quota a weekly cap applies to. The credential-health
|
||||
* scheduler runs that probe against every connection every 300s, so without this
|
||||
* gate a weekly-capped connection was reset to `active` / `rateLimitedUntil=null`
|
||||
* within 30s of every restart and dispatched straight back into the same 429.
|
||||
*
|
||||
* Same rule as `maybeClearRecoveredQuotaState`: a future `rateLimitedUntil` is
|
||||
* the 429 handler's hard statement and no poller may overrule it. Once the
|
||||
* window elapses, the next probe clears the state normally.
|
||||
*/
|
||||
export function shouldClearErrorStateOnValidProbe(
|
||||
connection: Pick<ProviderConnectionLike, "rateLimitedUntil">,
|
||||
probeValid: boolean,
|
||||
now: number = Date.now()
|
||||
): boolean {
|
||||
return probeValid && !hasActiveCooldown(connection, now);
|
||||
}
|
||||
|
||||
export async function maybeClearRecoveredQuotaState(
|
||||
connection: ProviderConnectionLike,
|
||||
usage: JsonRecord
|
||||
): Promise<ProviderConnectionLike> {
|
||||
if (!hasUsableQuota(usage)) return connection;
|
||||
if (isTerminalStatusForQuotaRecovery(connection.testStatus)) return connection;
|
||||
if (connection.lastErrorType === "quota_exhausted") {
|
||||
if (
|
||||
connection.lastErrorSource === CLAUDE_EXTRA_USAGE_ERROR_SOURCE &&
|
||||
isClaudeExtraUsageBlockEnabled(connection.provider, connection.providerSpecificData) &&
|
||||
isClaudeExtraUsageQueued(usage)
|
||||
) {
|
||||
// Claude's pay-as-you-go extra-usage block is orthogonal to the
|
||||
// session/weekly quota windows checked below: the upstream can report a
|
||||
// fully recovered quota window while extraUsage.queued is still true.
|
||||
// Only syncClaudeExtraUsageStateIfNeeded (buildClaudeExtraUsageConnectionUpdate)
|
||||
// owns clearing this specific state — the general window-recovery logic
|
||||
// below must not release it just because some quota window looks fresh.
|
||||
return connection;
|
||||
}
|
||||
|
||||
const quotas = usage?.quotas;
|
||||
if (isRecord(quotas)) {
|
||||
// Honor the REAL per-window resetAt from the freshly fetched quota
|
||||
// instead of the synthetic cooldown persisted at failure time (e.g.
|
||||
// Claude's flat 1h SUBSCRIPTION_QUOTA_COOLDOWN_MS when no upstream
|
||||
// reset was parseable). Only stay locked if some window that governs
|
||||
// this connection's quota is still demonstrably exhausted.
|
||||
const anyStillBlocking = Object.values(quotas).some((value) =>
|
||||
windowStillExhaustedAfterRealReset(value, Date.now())
|
||||
);
|
||||
if (anyStillBlocking) return connection;
|
||||
} else if (
|
||||
connection.rateLimitedUntil &&
|
||||
new Date(connection.rateLimitedUntil).getTime() > Date.now()
|
||||
) {
|
||||
// No quota object at all (degraded/failed fetch shape) — fall back to
|
||||
// the previous synthetic-cooldown guard.
|
||||
return connection;
|
||||
}
|
||||
} else if (
|
||||
connection.rateLimitedUntil &&
|
||||
new Date(connection.rateLimitedUntil).getTime() > Date.now()
|
||||
) {
|
||||
// Universal fallback guard for every lastErrorType other than
|
||||
// "quota_exhausted" (which gets the more precise per-window check above,
|
||||
// and may legitimately release early once the REAL window has reset even
|
||||
// while a synthetic rateLimitedUntil is still in the future). A future
|
||||
// rateLimitedUntil is a hard statement made by the 429/error handler that
|
||||
// persisted it (src/sse/services/auth.ts, src/app/api/providers/[id]/test/
|
||||
// route.ts) — no quota poll finding *some* usable window elsewhere should
|
||||
// be able to overrule it. Before this fix, ANY lastErrorType other than
|
||||
// "quota_exhausted" skipped straight to hasTransientState/
|
||||
// clearRecoveredProviderState() below with no rateLimitedUntil check at
|
||||
// all, so a multi-day cooldown (observed: 146h, Z.AI weekly quota) got
|
||||
// cleared on the very next quota sync a few minutes later — a
|
||||
// self-restart/burn loop that kept burning real upstream calls against a
|
||||
// known-exhausted connection (#11277).
|
||||
return connection;
|
||||
}
|
||||
if (hasActiveCooldown(connection)) return connection;
|
||||
|
||||
const hasTransientState =
|
||||
connection.testStatus === "unavailable" ||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Regression: the connection TEST path cleared a still-active cooldown.
|
||||
*
|
||||
* Sibling of repro-zai-cooldown-cleared-by-quota-poll.test.ts — same symptom,
|
||||
* a different writer. testSingleConnection() (src/app/api/providers/[id]/test/
|
||||
* route.ts) built its update payload as:
|
||||
*
|
||||
* testStatus: result.valid ? "active" : "error",
|
||||
* rateLimitedUntil: result.valid ? null : connection.rateLimitedUntil || null,
|
||||
*
|
||||
* so ANY successful probe wiped the persisted cooldown. That probe is not a
|
||||
* chat call — it is a cheap auth/models validation that never touches the chat
|
||||
* quota a weekly cap applies to, so it succeeds even while the weekly window is
|
||||
* exhausted. The credential-health scheduler (src/lib/credentialHealth/
|
||||
* scheduler.ts) runs it against every connection 30s after startup and every
|
||||
* 300s thereafter.
|
||||
*
|
||||
* Observed in production (2026-08-23) right after deploying the ISO-reset /
|
||||
* pre-skip / crash-clear patch: the GLM connection carried a valid future
|
||||
* rate_limited_until, "[CredentialHealth] Testing 10/10 connections..." ran,
|
||||
* and the row came back testStatus="active", rate_limited_until=NULL — so combo
|
||||
* dispatched zai/glm-5.3 straight back into the same weekly 429. This writer
|
||||
* alone defeats every other cooldown fix.
|
||||
*
|
||||
* The gate is shouldClearErrorStateOnValidProbe(): a future rateLimitedUntil is
|
||||
* the 429 handler's hard statement and a credential probe may not overrule it.
|
||||
*/
|
||||
|
||||
import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
hasActiveCooldown,
|
||||
shouldClearErrorStateOnValidProbe,
|
||||
} from "../../src/lib/usage/providerLimits.ts";
|
||||
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
const NOW = Date.UTC(2026, 7, 23, 21, 47, 0); // 2026-08-23 21:47 UTC
|
||||
|
||||
/** The production row: zai/glm-5.3 held until the weekly reset on 2026-08-29. */
|
||||
const GLM_COOLDOWN = { rateLimitedUntil: "2026-08-29T21:01:21.000Z" };
|
||||
|
||||
describe("connection test must not clear an active cooldown", () => {
|
||||
it("keeps the GLM weekly cooldown when the credential probe succeeds", () => {
|
||||
assert.equal(hasActiveCooldown(GLM_COOLDOWN, NOW), true);
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, true, NOW), false);
|
||||
});
|
||||
|
||||
it("keeps a cooldown that is only one second away from elapsing", () => {
|
||||
const conn = { rateLimitedUntil: new Date(NOW + 1000).toISOString() };
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false);
|
||||
});
|
||||
|
||||
it("clears the error state once the cooldown has elapsed", () => {
|
||||
const conn = { rateLimitedUntil: new Date(NOW - 1000).toISOString() };
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
|
||||
});
|
||||
|
||||
it("clears the error state at the exact reset instant", () => {
|
||||
const conn = { rateLimitedUntil: new Date(NOW).toISOString() };
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
|
||||
});
|
||||
|
||||
it("clears the error state for a connection with no cooldown", () => {
|
||||
assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, true, NOW), true);
|
||||
assert.equal(
|
||||
shouldClearErrorStateOnValidProbe({ rateLimitedUntil: undefined }, true, NOW),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
it("never clears on a FAILED probe, cooldown or not", () => {
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(GLM_COOLDOWN, false, NOW), false);
|
||||
assert.equal(shouldClearErrorStateOnValidProbe({ rateLimitedUntil: null }, false, NOW), false);
|
||||
});
|
||||
|
||||
it("fails open on an unparseable timestamp so a broken value cannot strand a connection", () => {
|
||||
const conn = { rateLimitedUntil: "not-a-date" };
|
||||
assert.equal(hasActiveCooldown(conn, NOW), false);
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), true);
|
||||
});
|
||||
|
||||
it("honours a numeric-epoch timestamp (the chat path writes epoch ms)", () => {
|
||||
const conn = { rateLimitedUntil: String(NOW + 146 * HOUR_MS) };
|
||||
assert.equal(shouldClearErrorStateOnValidProbe(conn, true, NOW), false);
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,17 @@
|
||||
/**
|
||||
* TDD regression guard for issue #3625 (Part A).
|
||||
* TDD regression guard for issue #3625 (Part A) and future quota cooldown preservation.
|
||||
*
|
||||
* After an unclean process crash (SIGKILL / large-body burst), provider
|
||||
* connections can be left in the DB with a far-future `rate_limited_until`
|
||||
* (stale exponential-backoff value). On restart, getProviderCredentials()
|
||||
* skips those connections and Bottleneck queues time out at 120 s.
|
||||
* connections can be left in the DB with expired transient cooldowns.
|
||||
* On startup, scan `provider_connections` and clear stale transient
|
||||
* cooldown fields for any non-terminal connection that has an EXPIRED or
|
||||
* unparseable `rate_limited_until`.
|
||||
*
|
||||
* The fix: on startup, scan `provider_connections` and clear transient
|
||||
* cooldown fields for any non-terminal connection that has a
|
||||
* `rate_limited_until` set (past *or* future). Terminal states
|
||||
* (banned / expired / credits_exhausted) must not be touched.
|
||||
* FUTURE timestamps (such as weekly/monthly quota cooldowns) MUST be
|
||||
* preserved so that restarts/recreates do not wipe active cooldowns and
|
||||
* immediately dispatch into upstream 429s.
|
||||
*
|
||||
* Terminal states (banned / expired / credits_exhausted) must not be touched.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
@@ -55,51 +57,43 @@ test.after(async () => {
|
||||
|
||||
// ─── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Far-future epoch ms (simulates a crash-burst backoff). */
|
||||
const FAR_FUTURE = Date.now() + 60 * 60 * 1000; // +1 hour
|
||||
/** Far-future epoch ms (simulates a multi-day quota reset or active cooldown). */
|
||||
const FAR_FUTURE = Date.now() + 6 * 24 * 60 * 60 * 1000; // +6 days
|
||||
|
||||
/** Slightly past timestamp (normal lazy expiry — also cleared on startup). */
|
||||
/** Slightly past timestamp (normal lazy expiry — cleared on startup). */
|
||||
const JUST_PAST = Date.now() - 10_000; // -10 s
|
||||
|
||||
// ─── tests ──────────────────────────────────────────────────────────────────
|
||||
|
||||
test("clearStaleCrashCooldowns clears far-future transient cooldown on restart", async () => {
|
||||
test("clearStaleCrashCooldowns PRESERVES future transient cooldown on restart", async () => {
|
||||
const conn = await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Stale Cooldown",
|
||||
name: "Future Cooldown",
|
||||
apiKey: "sk-test",
|
||||
});
|
||||
|
||||
// Simulate crash-burst state: far-future cooldown, transient error fields
|
||||
await providersDb.updateProviderConnection(conn.id, {
|
||||
...conn,
|
||||
rateLimitedUntil: new Date(FAR_FUTURE).toISOString(),
|
||||
testStatus: "unavailable",
|
||||
lastError: "upstream timeout",
|
||||
lastErrorType: "timeout",
|
||||
lastError: "upstream weekly quota exhausted",
|
||||
lastErrorType: "quota_exhausted",
|
||||
backoffLevel: 3,
|
||||
});
|
||||
|
||||
// Verify pre-condition: connection has a far-future cooldown persisted
|
||||
const pre = await providersDb.getProviderConnectionById(conn.id);
|
||||
assert.ok(
|
||||
pre?.rateLimitedUntil && new Date(pre.rateLimitedUntil as string).getTime() > Date.now(),
|
||||
"connection should have a future rate_limited_until before recovery"
|
||||
);
|
||||
|
||||
// Run startup recovery
|
||||
const result = providersDb.clearStaleCrashCooldowns();
|
||||
assert.equal(result.cleared, 0, "future cooldown must NOT be cleared on startup");
|
||||
|
||||
assert.ok(result.cleared >= 1, `expected at least 1 cleared, got ${result.cleared}`);
|
||||
|
||||
// Verify post-condition: cooldown is gone (cleanNulls strips null → undefined)
|
||||
const updated = await providersDb.getProviderConnectionById(conn.id);
|
||||
assert.ok(!updated?.rateLimitedUntil, "rateLimitedUntil should be absent/falsy after recovery");
|
||||
assert.equal(updated?.testStatus, "active", "testStatus should be 'active' after recovery");
|
||||
assert.equal(updated?.backoffLevel, 0, "backoffLevel should be 0 after recovery");
|
||||
assert.ok(!updated?.lastError, "lastError should be absent/falsy after recovery");
|
||||
assert.ok(!updated?.lastErrorType, "lastErrorType should be absent/falsy after recovery");
|
||||
assert.ok(updated?.rateLimitedUntil, "future rateLimitedUntil must remain intact");
|
||||
assert.equal(updated?.testStatus, "unavailable", "testStatus should remain unavailable");
|
||||
});
|
||||
|
||||
test("clearStaleCrashCooldowns clears past-dated transient cooldown on restart", async () => {
|
||||
@@ -144,14 +138,12 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (banned)", async (
|
||||
|
||||
const result = providersDb.clearStaleCrashCooldowns();
|
||||
|
||||
// The banned connection must NOT be cleared
|
||||
const updated = await providersDb.getProviderConnectionById(conn.id);
|
||||
assert.equal(updated?.testStatus, "banned", "banned connection must not be touched");
|
||||
assert.ok(
|
||||
updated?.rateLimitedUntil,
|
||||
"rate_limited_until on a banned connection must not be cleared"
|
||||
);
|
||||
// cleared count should be 0 (only the banned conn exists in this test)
|
||||
assert.equal(result.cleared, 0, "no transient connections to clear");
|
||||
});
|
||||
|
||||
@@ -201,7 +193,6 @@ test("clearStaleCrashCooldowns does NOT clear terminal states (credits_exhausted
|
||||
});
|
||||
|
||||
test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exist", async () => {
|
||||
// Create a clean connection (no cooldown)
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "gemini",
|
||||
authType: "apikey",
|
||||
@@ -215,28 +206,29 @@ test("clearStaleCrashCooldowns returns cleared=0 when no transient cooldowns exi
|
||||
});
|
||||
|
||||
test("clearStaleCrashCooldowns handles mixed transient + terminal connections correctly", async () => {
|
||||
// Transient — should be cleared
|
||||
const transient1 = await providersDb.createProviderConnection({
|
||||
// Future transient — should be PRESERVED
|
||||
const futureTransient = await providersDb.createProviderConnection({
|
||||
provider: "openai",
|
||||
authType: "apikey",
|
||||
name: "Transient 1",
|
||||
name: "Future Transient",
|
||||
apiKey: "sk-t1",
|
||||
});
|
||||
await providersDb.updateProviderConnection(transient1.id, {
|
||||
...transient1,
|
||||
await providersDb.updateProviderConnection(futureTransient.id, {
|
||||
...futureTransient,
|
||||
rateLimitedUntil: new Date(FAR_FUTURE).toISOString(),
|
||||
testStatus: "unavailable",
|
||||
backoffLevel: 2,
|
||||
});
|
||||
|
||||
const transient2 = await providersDb.createProviderConnection({
|
||||
// Past transient — should be CLEARED
|
||||
const pastTransient = await providersDb.createProviderConnection({
|
||||
provider: "anthropic",
|
||||
authType: "apikey",
|
||||
name: "Transient 2",
|
||||
name: "Past Transient",
|
||||
apiKey: "sk-t2",
|
||||
});
|
||||
await providersDb.updateProviderConnection(transient2.id, {
|
||||
...transient2,
|
||||
await providersDb.updateProviderConnection(pastTransient.id, {
|
||||
...pastTransient,
|
||||
rateLimitedUntil: new Date(JUST_PAST).toISOString(),
|
||||
testStatus: "unavailable",
|
||||
backoffLevel: 1,
|
||||
@@ -258,15 +250,15 @@ test("clearStaleCrashCooldowns handles mixed transient + terminal connections co
|
||||
|
||||
const result = providersDb.clearStaleCrashCooldowns();
|
||||
|
||||
assert.equal(result.cleared, 2, "exactly 2 transient connections cleared");
|
||||
assert.equal(result.cleared, 1, "only 1 past transient connection cleared");
|
||||
|
||||
const updatedT1 = await providersDb.getProviderConnectionById(transient1.id);
|
||||
assert.ok(!updatedT1?.rateLimitedUntil, "transient1 cooldown cleared");
|
||||
assert.equal(updatedT1?.testStatus, "active", "transient1 status active");
|
||||
const updatedFuture = await providersDb.getProviderConnectionById(futureTransient.id);
|
||||
assert.ok(updatedFuture?.rateLimitedUntil, "future cooldown preserved");
|
||||
assert.equal(updatedFuture?.testStatus, "unavailable", "future transient status preserved");
|
||||
|
||||
const updatedT2 = await providersDb.getProviderConnectionById(transient2.id);
|
||||
assert.ok(!updatedT2?.rateLimitedUntil, "transient2 cooldown cleared");
|
||||
assert.equal(updatedT2?.testStatus, "active", "transient2 status active");
|
||||
const updatedPast = await providersDb.getProviderConnectionById(pastTransient.id);
|
||||
assert.ok(!updatedPast?.rateLimitedUntil, "past transient cooldown cleared");
|
||||
assert.equal(updatedPast?.testStatus, "active", "past transient status active");
|
||||
|
||||
const updatedTerminal = await providersDb.getProviderConnectionById(terminal.id);
|
||||
assert.equal(updatedTerminal?.testStatus, "banned", "terminal connection untouched");
|
||||
|
||||
Reference in New Issue
Block a user