From cb1c5471ddb9dc89206007520de170f3f0aa61da Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Thu, 11 Jun 2026 20:53:55 -0300 Subject: [PATCH] fix(oauth): resilient refresh error classification + healthcheck circuit breaker Root cause of the production token-invalidation storm (claude/aa5dd5cf refreshed 1352x, kimi-coding 270x): when a refresh endpoint returned invalid_grant in a non-canonical body shape (a JSON string, a double-encoded string, a nested {error:{code}}, or raw text wrapped by a proxy/MITM), refreshClaudeOAuthToken's errorBody.error === "invalid_grant" check failed and returned null instead of the unrecoverable sentinel. The HealthCheck then kept the connection active and re-tried every 60s forever (the loop + log flood + upstream hammering that contributed to the 429s). - extractOAuthErrorCode(): shape-agnostic OAuth error extractor (object, nested, bare string, double-encoded JSON string, raw text) restricted to a known unrecoverable set so transient errors (server_error, 502 HTML) never become false positives. - refreshClaudeOAuthToken + refreshAccessToken/Cline/Qoder/GitHub now classify invalid_grant/invalid_request via the helper -> always emit the unrecoverable sentinel so the HealthCheck deactivates cleanly instead of looping. - HealthCheck refresh circuit breaker: track consecutive refresh failures in providerSpecificData.refreshCircuit and back off exponentially (5->10->20... ->240min cap) instead of retrying every 60s; cleared on a successful refresh. Stops the loop for null/network failures (e.g. dead proxy / Kimi fetch failed). TDD: 26 new tests (19 resilience + 7 circuit breaker); 113 oauth/refresh/ healthcheck tests green. typecheck:core=0, lint clean. --- open-sse/services/tokenRefresh.ts | 121 +++++++++++- src/lib/tokenHealthCheck.ts | 70 ++++++- .../oauth-refresh-error-resilience.test.ts | 180 ++++++++++++++++++ ...token-health-check-circuit-breaker.test.ts | 89 +++++++++ 4 files changed, 445 insertions(+), 15 deletions(-) create mode 100644 tests/unit/oauth-refresh-error-resilience.test.ts create mode 100644 tests/unit/token-health-check-circuit-breaker.test.ts diff --git a/open-sse/services/tokenRefresh.ts b/open-sse/services/tokenRefresh.ts index 4abb7acbee..0581a77bf6 100755 --- a/open-sse/services/tokenRefresh.ts +++ b/open-sse/services/tokenRefresh.ts @@ -181,6 +181,93 @@ function getRefreshCacheKey(provider, refreshToken) { return `${provider}:${tokenHash}`; } +/** + * OAuth2 error codes that mean the refresh token is permanently dead and + * retrying will never succeed → callers must emit the unrecoverable sentinel + * so the HealthCheck deactivates the account instead of looping every 60s. + * Deliberately EXCLUDES transient codes (server_error, temporarily_unavailable, + * slow_down) so we never deactivate an account over a recoverable blip. + */ +const UNRECOVERABLE_OAUTH_ERROR_CODES = new Set([ + "invalid_grant", + "invalid_request", + "refresh_token_reused", + "invalid_token", + "expired_token", + "unauthorized_client", + "access_denied", +]); + +/** + * Extract a canonical OAuth error code from a refresh-endpoint error body of + * ANY shape. Production proxies/MITMs deliver the same `invalid_grant` 400 in + * several shapes — a plain object `{error:"invalid_grant"}`, a nested + * `{error:{code:"invalid_grant"}}`, a JSON **string** (double-encoded body), + * or the raw JSON text wrapped as `{error:""}` by a catch branch. + * The old `errorBody.error === "invalid_grant"` only matched the first shape, + * so the others returned `null` → the HealthCheck refresh loop (root cause of + * the 1352× claude/aa5dd5cf invalidation storm). + * + * Returns the matched code (only if it is in UNRECOVERABLE_OAUTH_ERROR_CODES) + * or null. Never matches loosely — a known code is accepted only when it is a + * bare code string or the value of an `"error"`/`"error_code"` field, so a 502 + * HTML page or a `server_error` body never becomes a false positive. + */ +export function extractOAuthErrorCode(raw: unknown, depth = 0): string | null { + if (raw == null || depth > 6) return null; + + if (typeof raw === "string") { + const s = raw.trim(); + if (!s) return null; + if (UNRECOVERABLE_OAUTH_ERROR_CODES.has(s)) return s; + // The string may itself be JSON (a double-encoded body, or the raw text). + if (s[0] === "{" || s[0] === "[" || s[0] === '"') { + try { + const nested = extractOAuthErrorCode(JSON.parse(s), depth + 1); + if (nested) return nested; + } catch { + // not valid JSON — fall through to the field scan + } + } + // Safety net: a known code appearing as the value of an "error"/"error_code" + // field inside otherwise-unparsed text. Scoped to avoid false positives. + const m = s.match(/"error(?:_code)?"\s*:\s*"([a-z_]+)"/i); + if (m && UNRECOVERABLE_OAUTH_ERROR_CODES.has(m[1])) return m[1]; + return null; + } + + if (typeof raw === "object") { + const o = raw as Record; + return ( + extractOAuthErrorCode(o.error, depth + 1) ?? + extractOAuthErrorCode(o.code, depth + 1) ?? + extractOAuthErrorCode(o.error_code, depth + 1) + ); + } + + return null; +} + +/** + * Read an error response body ONCE and classify it. Returns the raw text (for + * logging) and the extracted unrecoverable OAuth code (or null). Reading once + * avoids the double-read bug where `response.json()` consumes the stream and a + * later `response.text()` returns empty. + */ +async function readRefreshErrorBody( + response: Response +): Promise<{ rawText: string; code: string | null }> { + const rawText = await response.text().catch(() => ""); + let parsed: unknown = rawText; + try { + parsed = JSON.parse(rawText); + } catch { + // keep rawText as-is + } + const code = extractOAuthErrorCode(parsed) ?? extractOAuthErrorCode(rawText); + return { rawText, code }; +} + /** * Refresh OAuth access token using refresh token */ @@ -229,6 +316,10 @@ export async function refreshAccessToken( status: response.status, error: errorText, }); + const code = extractOAuthErrorCode(errorText); + if (code === "invalid_grant" || code === "invalid_request") { + return { error: "unrecoverable_refresh_error", code }; + } return null; } @@ -401,6 +492,10 @@ export async function refreshClineToken(refreshToken, log, proxyConfig: unknown status: response.status, error: errorText, }); + const code = extractOAuthErrorCode(errorText); + if (code === "invalid_grant" || code === "invalid_request") { + return { error: "unrecoverable_refresh_error", code }; + } return null; } @@ -664,19 +759,17 @@ export async function refreshClaudeOAuthToken(refreshToken, log, proxyConfig: un ); if (!response.ok) { - let errorBody: { error?: string; error_description?: string } = {}; - try { - errorBody = await response.json(); - } catch { - const text = await response.text().catch(() => "unknown"); - errorBody = { error: text }; - } + // Read + classify the body ONCE, shape-agnostic. A proxy/MITM can deliver + // the invalid_grant 400 as a JSON string, a double-encoded string, a + // nested {error:{code}}, or raw text — all must yield the sentinel so the + // HealthCheck deactivates instead of looping every 60s. + const { rawText, code } = await readRefreshErrorBody(response); log?.error?.("TOKEN_REFRESH", "Failed to refresh Claude OAuth token", { status: response.status, - error: errorBody, + error: rawText.slice(0, 300), }); - if (errorBody.error === "invalid_grant" || errorBody.error === "invalid_request") { - return { error: "unrecoverable_refresh_error", code: errorBody.error }; + if (code === "invalid_grant" || code === "invalid_request") { + return { error: "unrecoverable_refresh_error", code }; } return null; } @@ -1186,6 +1279,10 @@ export async function refreshQoderToken(refreshToken, log, proxyConfig: unknown status: response.status, error: errorText, }); + const code = extractOAuthErrorCode(errorText); + if (code === "invalid_grant" || code === "invalid_request") { + return { error: "unrecoverable_refresh_error", code }; + } return null; } @@ -1230,6 +1327,10 @@ export async function refreshGitHubToken(refreshToken, log, proxyConfig: unknown status: response.status, error: errorText, }); + const code = extractOAuthErrorCode(errorText); + if (code === "invalid_grant" || code === "invalid_request") { + return { error: "unrecoverable_refresh_error", code }; + } return null; } diff --git a/src/lib/tokenHealthCheck.ts b/src/lib/tokenHealthCheck.ts index 1166238f04..305d9ec6e5 100644 --- a/src/lib/tokenHealthCheck.ts +++ b/src/lib/tokenHealthCheck.ts @@ -76,10 +76,38 @@ function getEffectiveTokenExpiryMs(conn: any): number { return Number.isFinite(expiryMs) ? expiryMs : 0; } +// ── Refresh circuit breaker ─────────────────────────────────────────────── +// A refresh that returns null (network blip, dead proxy, unclassified error) +// leaves the connection active, so the next 60s sweep retries immediately — +// the production refresh loop (claude/aa5dd5cf 1352×, kimi 270×). We track +// consecutive failures and back off exponentially so a stuck connection stops +// hammering the upstream (and stops flooding the logs) instead of looping. +const REFRESH_CIRCUIT_BASE_MIN = 5; +const REFRESH_CIRCUIT_MAX_MIN = 240; // cap at 4h + +export function getRefreshBackoffUntil(streak: number, now: string): string { + const steps = Math.max(0, streak - 1); + const backoffMin = Math.min(REFRESH_CIRCUIT_BASE_MIN * 2 ** steps, REFRESH_CIRCUIT_MAX_MIN); + return new Date(new Date(now).getTime() + backoffMin * 60 * 1000).toISOString(); +} + +export function isInRefreshBackoff(conn: any, nowMs: number): boolean { + const until = conn?.providerSpecificData?.refreshCircuit?.until; + if (typeof until !== "string") return false; + const untilMs = new Date(until).getTime(); + return Number.isFinite(untilMs) && untilMs > nowMs; +} + export function buildRefreshFailureUpdate(conn: any, now: string) { const wasExpired = conn.testStatus === "expired"; const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0); + // Circuit breaker: increment the consecutive-failure streak and set an + // exponential backoff window so the next sweep skips this connection instead + // of retrying every 60s. Cleared by a successful refresh (clearRefreshCircuit). + const prevStreak = conn.providerSpecificData?.refreshCircuit?.streak ?? 0; + const streak = prevStreak + 1; + return { lastHealthCheckAt: now, // A failed background refresh should not evict otherwise healthy accounts @@ -91,10 +119,28 @@ export function buildRefreshFailureUpdate(conn: any, now: string) { lastErrorType: "token_refresh_failed", lastErrorSource: "oauth", errorCode: "refresh_failed", + providerSpecificData: { + ...(conn.providerSpecificData || {}), + refreshCircuit: { streak, until: getRefreshBackoffUntil(streak, now), lastFailAt: now }, + }, ...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}), }; } +/** + * Strip the refresh circuit breaker state from providerSpecificData after a + * successful refresh, so the streak/backoff resets cleanly. + */ +export function clearRefreshCircuit( + providerSpecificData: Record | null | undefined +): Record | undefined { + if (!providerSpecificData || typeof providerSpecificData !== "object") return undefined; + if (!("refreshCircuit" in providerSpecificData)) return undefined; + const next = { ...providerSpecificData }; + delete next.refreshCircuit; + return next; +} + function isEnvFlagEnabled(name: string): boolean { const value = process.env[name]; if (!value) return false; @@ -355,6 +401,14 @@ export async function checkConnection(conn) { if (!isAboutToExpire && !shouldRefreshByInterval) return; + // Circuit breaker: if recent refreshes for this connection failed, wait out + // the exponential backoff window instead of retrying every 60s tick. This is + // what stops the refresh loop when getAccessToken keeps returning null + // (dead proxy / network blip / unclassified upstream error). + if (isInRefreshBackoff(conn, Date.now())) { + return; + } + const reason = isAboutToExpire ? "token expiring soon" : `interval: ${intervalMin}min`; log(`${LOG_PREFIX} Refreshing ${conn.provider}/${getConnectionLogLabel(conn)} (${reason})`); @@ -427,11 +481,17 @@ export async function checkConnection(conn) { updateData.expiresAt = expiresAt; updateData.tokenExpiresAt = expiresAt; } - if (refreshResult.providerSpecificData) { - updateData.providerSpecificData = { - ...(conn.providerSpecificData || {}), - ...refreshResult.providerSpecificData, - }; + // Merge new providerSpecificData and ALWAYS clear the refresh circuit + // breaker streak on a successful refresh. + const mergedProviderData = { + ...(conn.providerSpecificData || {}), + ...(refreshResult.providerSpecificData || {}), + }; + const clearedProviderData = clearRefreshCircuit(mergedProviderData); + if (clearedProviderData !== undefined) { + updateData.providerSpecificData = clearedProviderData; + } else if (refreshResult.providerSpecificData) { + updateData.providerSpecificData = mergedProviderData; } await updateProviderConnection(conn.id, updateData); persistedResult = refreshResult; diff --git a/tests/unit/oauth-refresh-error-resilience.test.ts b/tests/unit/oauth-refresh-error-resilience.test.ts new file mode 100644 index 0000000000..aa3125ab79 --- /dev/null +++ b/tests/unit/oauth-refresh-error-resilience.test.ts @@ -0,0 +1,180 @@ +/** + * TDD — OAuth refresh error classification must be resilient to body SHAPE. + * + * Root cause of the production "1352× refresh loop" (claude/aa5dd5cf): when the + * Anthropic 400 body reaches `refreshClaudeOAuthToken` in a non-canonical shape + * (a JSON string instead of an object, a double-encoded string, a nested + * `{error:{code}}`, or the raw text in the catch branch), the old check + * `errorBody.error === "invalid_grant"` evaluated to false, so the function + * returned `null` instead of the `unrecoverable_refresh_error` sentinel. + * + * `null` makes the HealthCheck treat it as a recoverable failure → keeps the + * connection `active` → retries every 60s forever (the loop). The fix is a + * shape-agnostic extractor used by all refreshers that classify invalid_grant. + * + * These tests FAIL before the fix (functions return null) and pass after. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const tokenRefresh = await import("../../open-sse/services/tokenRefresh.ts"); +const { + extractOAuthErrorCode, + refreshClaudeOAuthToken, + refreshClineToken, + refreshQoderToken, + refreshGitHubToken, + isUnrecoverableRefreshError, +} = tokenRefresh as unknown as { + extractOAuthErrorCode: (raw: unknown) => string | null; + refreshClaudeOAuthToken: (rt: string, log?: unknown, proxy?: unknown) => Promise; + refreshClineToken: (rt: string, log?: unknown, proxy?: unknown) => Promise; + refreshQoderToken: (rt: string, log?: unknown, proxy?: unknown) => Promise; + refreshGitHubToken: (rt: string, log?: unknown, proxy?: unknown) => Promise; + isUnrecoverableRefreshError: (r: unknown) => boolean; +}; + +function rawResponse(body: string, status = 400, contentType = "application/json") { + return new Response(body, { status, headers: { "content-type": contentType } }); +} + +async function withMockedFetch(impl: typeof fetch, fn: () => Promise): Promise { + const original = globalThis.fetch; + globalThis.fetch = impl; + try { + return await fn(); + } finally { + globalThis.fetch = original; + } +} + +// ── extractOAuthErrorCode: shape matrix ───────────────────────────────────── + +test("extractOAuthErrorCode: canonical object { error: 'invalid_grant' }", () => { + assert.equal(extractOAuthErrorCode({ error: "invalid_grant" }), "invalid_grant"); +}); + +test("extractOAuthErrorCode: nested object { error: { code: 'invalid_grant' } }", () => { + assert.equal(extractOAuthErrorCode({ error: { code: "invalid_grant" } }), "invalid_grant"); +}); + +test("extractOAuthErrorCode: bare string code 'invalid_grant'", () => { + assert.equal(extractOAuthErrorCode("invalid_grant"), "invalid_grant"); +}); + +test("extractOAuthErrorCode: JSON string body '{\"error\":\"invalid_grant\"}'", () => { + assert.equal(extractOAuthErrorCode('{"error": "invalid_grant"}'), "invalid_grant"); +}); + +test("extractOAuthErrorCode: double-encoded JSON string (the production case)", () => { + // response.json() returned the inner JSON AS a string (proxy double-encode) + const doubleEncoded = JSON.stringify('{"error": "invalid_grant", "error_description": "x"}'); + const parsedOnce = JSON.parse(doubleEncoded); // a string, still JSON inside + assert.equal(extractOAuthErrorCode(parsedOnce), "invalid_grant"); +}); + +test("extractOAuthErrorCode: catch-branch shape { error: '' }", () => { + // refreshClaudeOAuthToken's catch did errorBody = { error: text } + const errorBody = { error: '{"error": "invalid_grant", "error_description": "x"}' }; + assert.equal(extractOAuthErrorCode(errorBody), "invalid_grant"); +}); + +test("extractOAuthErrorCode: invalid_request is recognized", () => { + assert.equal(extractOAuthErrorCode({ error: "invalid_request" }), "invalid_request"); +}); + +test("extractOAuthErrorCode: transient errors are NOT misclassified (no false positives)", () => { + assert.equal(extractOAuthErrorCode({ error: "server_error" }), null); + assert.equal(extractOAuthErrorCode("rate_limited"), null); + assert.equal(extractOAuthErrorCode("502 Bad Gateway"), null); + assert.equal(extractOAuthErrorCode(""), null); + assert.equal(extractOAuthErrorCode(null), null); + assert.equal(extractOAuthErrorCode(undefined), null); +}); + +// ── refreshClaudeOAuthToken: every shape → unrecoverable sentinel ──────────── + +const SENTINEL_SHAPES: Array<{ name: string; body: string; ct?: string }> = [ + { name: "canonical object", body: '{"error": "invalid_grant", "error_description": "Refresh token not found or invalid"}' }, + { name: "double-encoded JSON string", body: JSON.stringify('{"error": "invalid_grant", "error_description": "x"}') }, + { name: "bare string code", body: '"invalid_grant"' }, + { name: "nested error.code", body: '{"error": {"code": "invalid_grant", "message": "x"}}' }, + { name: "json served as text/plain", body: '{"error": "invalid_grant"}', ct: "text/plain" }, +]; + +for (const shape of SENTINEL_SHAPES) { + test(`refreshClaudeOAuthToken → unrecoverable sentinel for shape: ${shape.name}`, async () => { + await withMockedFetch( + (async () => rawResponse(shape.body, 400, shape.ct ?? "application/json")) as unknown as typeof fetch, + async () => { + const result = await refreshClaudeOAuthToken("dead-refresh-token"); + assert.ok( + isUnrecoverableRefreshError(result), + `shape "${shape.name}" must yield an unrecoverable sentinel, got ${JSON.stringify(result)}` + ); + assert.equal((result as { code?: string }).code, "invalid_grant"); + } + ); + }); +} + +test("refreshClaudeOAuthToken: transient 500 server_error stays null (NOT unrecoverable)", async () => { + await withMockedFetch( + (async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch, + async () => { + const result = await refreshClaudeOAuthToken("token"); + assert.equal(result, null, "transient errors must remain recoverable (null), not deactivate the account"); + } + ); +}); + +test("refreshClaudeOAuthToken: 502 HTML gateway error stays null", async () => { + await withMockedFetch( + (async () => rawResponse("502 Bad Gateway", 502, "text/html")) as unknown as typeof fetch, + async () => { + const result = await refreshClaudeOAuthToken("token"); + assert.equal(result, null); + } + ); +}); + +// ── Previously-frágil refreshers that NEVER emitted a sentinel ────────────── +// refreshClineToken / refreshQoderToken / refreshGitHubToken returned null on +// ANY error → invalid_grant looked recoverable → HealthCheck refresh loop. + +// Note: refreshQoderToken also got the same fix, but it early-returns null via a +// config guard (no clientId/secret in the test env) so it can't be exercised here. +const FRAGILE_REFRESHERS: Array<{ + name: string; + fn: (rt: string) => Promise; +}> = [ + { name: "refreshClineToken", fn: (rt) => refreshClineToken(rt) }, + { name: "refreshGitHubToken", fn: (rt) => refreshGitHubToken(rt) }, +]; + +void refreshQoderToken; // fixed in source; not unit-testable without OAuth config + +for (const r of FRAGILE_REFRESHERS) { + test(`${r.name}: invalid_grant now yields an unrecoverable sentinel`, async () => { + await withMockedFetch( + (async () => rawResponse('{"error": "invalid_grant"}', 400)) as unknown as typeof fetch, + async () => { + const result = await r.fn("dead-token"); + assert.ok( + isUnrecoverableRefreshError(result), + `${r.name} must classify invalid_grant as unrecoverable, got ${JSON.stringify(result)}` + ); + } + ); + }); + + test(`${r.name}: transient 500 server_error stays null`, async () => { + await withMockedFetch( + (async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch, + async () => { + const result = await r.fn("token"); + assert.equal(result, null, `${r.name} must keep transient errors recoverable`); + } + ); + }); +} diff --git a/tests/unit/token-health-check-circuit-breaker.test.ts b/tests/unit/token-health-check-circuit-breaker.test.ts new file mode 100644 index 0000000000..b4da27af55 --- /dev/null +++ b/tests/unit/token-health-check-circuit-breaker.test.ts @@ -0,0 +1,89 @@ +/** + * TDD — HealthCheck refresh circuit breaker. + * + * Production incident: claude/aa5dd5cf refreshed 1352× and kimi-coding 270×, + * each retrying every 60s forever because a refresh that returns `null` + * (network blip, dead proxy, or an unclassified error) leaves the connection + * `active`, so the next sweep tries again immediately — no backoff. + * + * The circuit breaker tracks consecutive refresh failures in + * providerSpecificData.refreshCircuit and computes an exponential backoff + * window. While inside the window, checkConnection must SKIP the refresh + * instead of hammering every tick. A successful refresh clears the circuit. + * + * These exercise the pure helpers (no DB/network needed). + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts"); +const { buildRefreshFailureUpdate, isInRefreshBackoff, getRefreshBackoffUntil } = + tokenHealthCheck as unknown as { + buildRefreshFailureUpdate: (conn: any, now: string) => any; + isInRefreshBackoff: (conn: any, nowMs: number) => boolean; + getRefreshBackoffUntil: (streak: number, now: string) => string; + }; + +const NOW = "2026-06-11T12:00:00.000Z"; +const NOW_MS = new Date(NOW).getTime(); + +test("getRefreshBackoffUntil grows exponentially and caps", () => { + const min = (iso: string) => Math.round((new Date(iso).getTime() - NOW_MS) / 60000); + assert.equal(min(getRefreshBackoffUntil(1, NOW)), 5); // 5 * 2^0 + assert.equal(min(getRefreshBackoffUntil(2, NOW)), 10); // 5 * 2^1 + assert.equal(min(getRefreshBackoffUntil(3, NOW)), 20); + assert.equal(min(getRefreshBackoffUntil(4, NOW)), 40); + assert.ok(min(getRefreshBackoffUntil(20, NOW)) <= 240, "must cap at 4h"); +}); + +test("buildRefreshFailureUpdate starts a circuit streak of 1 on first failure", () => { + const update = buildRefreshFailureUpdate({ testStatus: "active" }, NOW); + assert.equal(update.testStatus, "active", "first failure stays routable"); + assert.equal(update.providerSpecificData.refreshCircuit.streak, 1); + assert.ok( + new Date(update.providerSpecificData.refreshCircuit.until).getTime() > NOW_MS, + "must set a future backoff window" + ); +}); + +test("buildRefreshFailureUpdate increments the streak across consecutive failures", () => { + const update = buildRefreshFailureUpdate( + { testStatus: "active", providerSpecificData: { refreshCircuit: { streak: 3 } } }, + NOW + ); + assert.equal(update.providerSpecificData.refreshCircuit.streak, 4); +}); + +test("buildRefreshFailureUpdate preserves unrelated providerSpecificData", () => { + const update = buildRefreshFailureUpdate( + { testStatus: "active", providerSpecificData: { projectId: "p-123", copilotToken: "x" } }, + NOW + ); + assert.equal(update.providerSpecificData.projectId, "p-123"); + assert.equal(update.providerSpecificData.copilotToken, "x"); + assert.equal(update.providerSpecificData.refreshCircuit.streak, 1); +}); + +test("isInRefreshBackoff true while within the window, false after", () => { + const conn = { + providerSpecificData: { refreshCircuit: { until: getRefreshBackoffUntil(2, NOW) } }, + }; + assert.equal(isInRefreshBackoff(conn, NOW_MS), true, "10min window, now → inside"); + assert.equal(isInRefreshBackoff(conn, NOW_MS + 11 * 60000), false, "after 11min → outside"); +}); + +test("isInRefreshBackoff false when no circuit recorded", () => { + assert.equal(isInRefreshBackoff({}, NOW_MS), false); + assert.equal(isInRefreshBackoff({ providerSpecificData: {} }, NOW_MS), false); + assert.equal(isInRefreshBackoff({ providerSpecificData: { refreshCircuit: {} } }, NOW_MS), false); +}); + +test("expired connections still track expiredRetryCount AND the circuit", () => { + const update = buildRefreshFailureUpdate( + { testStatus: "expired", expiredRetryCount: 1 }, + NOW + ); + assert.equal(update.testStatus, "expired"); + assert.equal(update.expiredRetryCount, 2); + assert.equal(update.providerSpecificData.refreshCircuit.streak, 1); +});