mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-03 13:52:09 +03:00
fix(oauth): resilient refresh error classification + healthcheck circuit breaker
Root cause of the production token-invalidation storm (claude/aa5dd5cf
refreshed 1352x, kimi-coding 270x): when a refresh endpoint returned
invalid_grant in a non-canonical body shape (a JSON string, a double-encoded
string, a nested {error:{code}}, or raw text wrapped by a proxy/MITM),
refreshClaudeOAuthToken's errorBody.error === "invalid_grant" check failed and
returned null instead of the unrecoverable sentinel. The HealthCheck then kept
the connection active and re-tried every 60s forever (the loop + log flood +
upstream hammering that contributed to the 429s).
- extractOAuthErrorCode(): shape-agnostic OAuth error extractor (object, nested,
bare string, double-encoded JSON string, raw text) restricted to a known
unrecoverable set so transient errors (server_error, 502 HTML) never become
false positives.
- refreshClaudeOAuthToken + refreshAccessToken/Cline/Qoder/GitHub now classify
invalid_grant/invalid_request via the helper -> always emit the unrecoverable
sentinel so the HealthCheck deactivates cleanly instead of looping.
- HealthCheck refresh circuit breaker: track consecutive refresh failures in
providerSpecificData.refreshCircuit and back off exponentially (5->10->20...
->240min cap) instead of retrying every 60s; cleared on a successful refresh.
Stops the loop for null/network failures (e.g. dead proxy / Kimi fetch failed).
TDD: 26 new tests (19 resilience + 7 circuit breaker); 113 oauth/refresh/
healthcheck tests green. typecheck:core=0, lint clean.
This commit is contained in:
@@ -181,6 +181,93 @@ function getRefreshCacheKey(provider, refreshToken) {
|
||||
return `${provider}:${tokenHash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* OAuth2 error codes that mean the refresh token is permanently dead and
|
||||
* retrying will never succeed → callers must emit the unrecoverable sentinel
|
||||
* so the HealthCheck deactivates the account instead of looping every 60s.
|
||||
* Deliberately EXCLUDES transient codes (server_error, temporarily_unavailable,
|
||||
* slow_down) so we never deactivate an account over a recoverable blip.
|
||||
*/
|
||||
const UNRECOVERABLE_OAUTH_ERROR_CODES = new Set([
|
||||
"invalid_grant",
|
||||
"invalid_request",
|
||||
"refresh_token_reused",
|
||||
"invalid_token",
|
||||
"expired_token",
|
||||
"unauthorized_client",
|
||||
"access_denied",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Extract a canonical OAuth error code from a refresh-endpoint error body of
|
||||
* ANY shape. Production proxies/MITMs deliver the same `invalid_grant` 400 in
|
||||
* several shapes — a plain object `{error:"invalid_grant"}`, a nested
|
||||
* `{error:{code:"invalid_grant"}}`, a JSON **string** (double-encoded body),
|
||||
* or the raw JSON text wrapped as `{error:"<json text>"}` by a catch branch.
|
||||
* The old `errorBody.error === "invalid_grant"` only matched the first shape,
|
||||
* so the others returned `null` → the HealthCheck refresh loop (root cause of
|
||||
* the 1352× claude/aa5dd5cf invalidation storm).
|
||||
*
|
||||
* Returns the matched code (only if it is in UNRECOVERABLE_OAUTH_ERROR_CODES)
|
||||
* or null. Never matches loosely — a known code is accepted only when it is a
|
||||
* bare code string or the value of an `"error"`/`"error_code"` field, so a 502
|
||||
* HTML page or a `server_error` body never becomes a false positive.
|
||||
*/
|
||||
export function extractOAuthErrorCode(raw: unknown, depth = 0): string | null {
|
||||
if (raw == null || depth > 6) return null;
|
||||
|
||||
if (typeof raw === "string") {
|
||||
const s = raw.trim();
|
||||
if (!s) return null;
|
||||
if (UNRECOVERABLE_OAUTH_ERROR_CODES.has(s)) return s;
|
||||
// The string may itself be JSON (a double-encoded body, or the raw text).
|
||||
if (s[0] === "{" || s[0] === "[" || s[0] === '"') {
|
||||
try {
|
||||
const nested = extractOAuthErrorCode(JSON.parse(s), depth + 1);
|
||||
if (nested) return nested;
|
||||
} catch {
|
||||
// not valid JSON — fall through to the field scan
|
||||
}
|
||||
}
|
||||
// Safety net: a known code appearing as the value of an "error"/"error_code"
|
||||
// field inside otherwise-unparsed text. Scoped to avoid false positives.
|
||||
const m = s.match(/"error(?:_code)?"\s*:\s*"([a-z_]+)"/i);
|
||||
if (m && UNRECOVERABLE_OAUTH_ERROR_CODES.has(m[1])) return m[1];
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof raw === "object") {
|
||||
const o = raw as Record<string, unknown>;
|
||||
return (
|
||||
extractOAuthErrorCode(o.error, depth + 1) ??
|
||||
extractOAuthErrorCode(o.code, depth + 1) ??
|
||||
extractOAuthErrorCode(o.error_code, depth + 1)
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an error response body ONCE and classify it. Returns the raw text (for
|
||||
* logging) and the extracted unrecoverable OAuth code (or null). Reading once
|
||||
* avoids the double-read bug where `response.json()` consumes the stream and a
|
||||
* later `response.text()` returns empty.
|
||||
*/
|
||||
async function readRefreshErrorBody(
|
||||
response: Response
|
||||
): Promise<{ rawText: string; code: string | null }> {
|
||||
const rawText = await response.text().catch(() => "");
|
||||
let parsed: unknown = rawText;
|
||||
try {
|
||||
parsed = JSON.parse(rawText);
|
||||
} catch {
|
||||
// keep rawText as-is
|
||||
}
|
||||
const code = extractOAuthErrorCode(parsed) ?? extractOAuthErrorCode(rawText);
|
||||
return { rawText, code };
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh OAuth access token using refresh token
|
||||
*/
|
||||
@@ -229,6 +316,10 @@ export async function refreshAccessToken(
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
});
|
||||
const code = extractOAuthErrorCode(errorText);
|
||||
if (code === "invalid_grant" || code === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -401,6 +492,10 @@ export async function refreshClineToken(refreshToken, log, proxyConfig: unknown
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
});
|
||||
const code = extractOAuthErrorCode(errorText);
|
||||
if (code === "invalid_grant" || code === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -664,19 +759,17 @@ export async function refreshClaudeOAuthToken(refreshToken, log, proxyConfig: un
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
let errorBody: { error?: string; error_description?: string } = {};
|
||||
try {
|
||||
errorBody = await response.json();
|
||||
} catch {
|
||||
const text = await response.text().catch(() => "unknown");
|
||||
errorBody = { error: text };
|
||||
}
|
||||
// Read + classify the body ONCE, shape-agnostic. A proxy/MITM can deliver
|
||||
// the invalid_grant 400 as a JSON string, a double-encoded string, a
|
||||
// nested {error:{code}}, or raw text — all must yield the sentinel so the
|
||||
// HealthCheck deactivates instead of looping every 60s.
|
||||
const { rawText, code } = await readRefreshErrorBody(response);
|
||||
log?.error?.("TOKEN_REFRESH", "Failed to refresh Claude OAuth token", {
|
||||
status: response.status,
|
||||
error: errorBody,
|
||||
error: rawText.slice(0, 300),
|
||||
});
|
||||
if (errorBody.error === "invalid_grant" || errorBody.error === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code: errorBody.error };
|
||||
if (code === "invalid_grant" || code === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -1186,6 +1279,10 @@ export async function refreshQoderToken(refreshToken, log, proxyConfig: unknown
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
});
|
||||
const code = extractOAuthErrorCode(errorText);
|
||||
if (code === "invalid_grant" || code === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1230,6 +1327,10 @@ export async function refreshGitHubToken(refreshToken, log, proxyConfig: unknown
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
});
|
||||
const code = extractOAuthErrorCode(errorText);
|
||||
if (code === "invalid_grant" || code === "invalid_request") {
|
||||
return { error: "unrecoverable_refresh_error", code };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -76,10 +76,38 @@ function getEffectiveTokenExpiryMs(conn: any): number {
|
||||
return Number.isFinite(expiryMs) ? expiryMs : 0;
|
||||
}
|
||||
|
||||
// ── Refresh circuit breaker ───────────────────────────────────────────────
|
||||
// A refresh that returns null (network blip, dead proxy, unclassified error)
|
||||
// leaves the connection active, so the next 60s sweep retries immediately —
|
||||
// the production refresh loop (claude/aa5dd5cf 1352×, kimi 270×). We track
|
||||
// consecutive failures and back off exponentially so a stuck connection stops
|
||||
// hammering the upstream (and stops flooding the logs) instead of looping.
|
||||
const REFRESH_CIRCUIT_BASE_MIN = 5;
|
||||
const REFRESH_CIRCUIT_MAX_MIN = 240; // cap at 4h
|
||||
|
||||
export function getRefreshBackoffUntil(streak: number, now: string): string {
|
||||
const steps = Math.max(0, streak - 1);
|
||||
const backoffMin = Math.min(REFRESH_CIRCUIT_BASE_MIN * 2 ** steps, REFRESH_CIRCUIT_MAX_MIN);
|
||||
return new Date(new Date(now).getTime() + backoffMin * 60 * 1000).toISOString();
|
||||
}
|
||||
|
||||
export function isInRefreshBackoff(conn: any, nowMs: number): boolean {
|
||||
const until = conn?.providerSpecificData?.refreshCircuit?.until;
|
||||
if (typeof until !== "string") return false;
|
||||
const untilMs = new Date(until).getTime();
|
||||
return Number.isFinite(untilMs) && untilMs > nowMs;
|
||||
}
|
||||
|
||||
export function buildRefreshFailureUpdate(conn: any, now: string) {
|
||||
const wasExpired = conn.testStatus === "expired";
|
||||
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
|
||||
|
||||
// Circuit breaker: increment the consecutive-failure streak and set an
|
||||
// exponential backoff window so the next sweep skips this connection instead
|
||||
// of retrying every 60s. Cleared by a successful refresh (clearRefreshCircuit).
|
||||
const prevStreak = conn.providerSpecificData?.refreshCircuit?.streak ?? 0;
|
||||
const streak = prevStreak + 1;
|
||||
|
||||
return {
|
||||
lastHealthCheckAt: now,
|
||||
// A failed background refresh should not evict otherwise healthy accounts
|
||||
@@ -91,10 +119,28 @@ export function buildRefreshFailureUpdate(conn: any, now: string) {
|
||||
lastErrorType: "token_refresh_failed",
|
||||
lastErrorSource: "oauth",
|
||||
errorCode: "refresh_failed",
|
||||
providerSpecificData: {
|
||||
...(conn.providerSpecificData || {}),
|
||||
refreshCircuit: { streak, until: getRefreshBackoffUntil(streak, now), lastFailAt: now },
|
||||
},
|
||||
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip the refresh circuit breaker state from providerSpecificData after a
|
||||
* successful refresh, so the streak/backoff resets cleanly.
|
||||
*/
|
||||
export function clearRefreshCircuit(
|
||||
providerSpecificData: Record<string, unknown> | null | undefined
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!providerSpecificData || typeof providerSpecificData !== "object") return undefined;
|
||||
if (!("refreshCircuit" in providerSpecificData)) return undefined;
|
||||
const next = { ...providerSpecificData };
|
||||
delete next.refreshCircuit;
|
||||
return next;
|
||||
}
|
||||
|
||||
function isEnvFlagEnabled(name: string): boolean {
|
||||
const value = process.env[name];
|
||||
if (!value) return false;
|
||||
@@ -355,6 +401,14 @@ export async function checkConnection(conn) {
|
||||
|
||||
if (!isAboutToExpire && !shouldRefreshByInterval) return;
|
||||
|
||||
// Circuit breaker: if recent refreshes for this connection failed, wait out
|
||||
// the exponential backoff window instead of retrying every 60s tick. This is
|
||||
// what stops the refresh loop when getAccessToken keeps returning null
|
||||
// (dead proxy / network blip / unclassified upstream error).
|
||||
if (isInRefreshBackoff(conn, Date.now())) {
|
||||
return;
|
||||
}
|
||||
|
||||
const reason = isAboutToExpire ? "token expiring soon" : `interval: ${intervalMin}min`;
|
||||
log(`${LOG_PREFIX} Refreshing ${conn.provider}/${getConnectionLogLabel(conn)} (${reason})`);
|
||||
|
||||
@@ -427,11 +481,17 @@ export async function checkConnection(conn) {
|
||||
updateData.expiresAt = expiresAt;
|
||||
updateData.tokenExpiresAt = expiresAt;
|
||||
}
|
||||
if (refreshResult.providerSpecificData) {
|
||||
updateData.providerSpecificData = {
|
||||
...(conn.providerSpecificData || {}),
|
||||
...refreshResult.providerSpecificData,
|
||||
};
|
||||
// Merge new providerSpecificData and ALWAYS clear the refresh circuit
|
||||
// breaker streak on a successful refresh.
|
||||
const mergedProviderData = {
|
||||
...(conn.providerSpecificData || {}),
|
||||
...(refreshResult.providerSpecificData || {}),
|
||||
};
|
||||
const clearedProviderData = clearRefreshCircuit(mergedProviderData);
|
||||
if (clearedProviderData !== undefined) {
|
||||
updateData.providerSpecificData = clearedProviderData;
|
||||
} else if (refreshResult.providerSpecificData) {
|
||||
updateData.providerSpecificData = mergedProviderData;
|
||||
}
|
||||
await updateProviderConnection(conn.id, updateData);
|
||||
persistedResult = refreshResult;
|
||||
|
||||
180
tests/unit/oauth-refresh-error-resilience.test.ts
Normal file
180
tests/unit/oauth-refresh-error-resilience.test.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* TDD — OAuth refresh error classification must be resilient to body SHAPE.
|
||||
*
|
||||
* Root cause of the production "1352× refresh loop" (claude/aa5dd5cf): when the
|
||||
* Anthropic 400 body reaches `refreshClaudeOAuthToken` in a non-canonical shape
|
||||
* (a JSON string instead of an object, a double-encoded string, a nested
|
||||
* `{error:{code}}`, or the raw text in the catch branch), the old check
|
||||
* `errorBody.error === "invalid_grant"` evaluated to false, so the function
|
||||
* returned `null` instead of the `unrecoverable_refresh_error` sentinel.
|
||||
*
|
||||
* `null` makes the HealthCheck treat it as a recoverable failure → keeps the
|
||||
* connection `active` → retries every 60s forever (the loop). The fix is a
|
||||
* shape-agnostic extractor used by all refreshers that classify invalid_grant.
|
||||
*
|
||||
* These tests FAIL before the fix (functions return null) and pass after.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const tokenRefresh = await import("../../open-sse/services/tokenRefresh.ts");
|
||||
const {
|
||||
extractOAuthErrorCode,
|
||||
refreshClaudeOAuthToken,
|
||||
refreshClineToken,
|
||||
refreshQoderToken,
|
||||
refreshGitHubToken,
|
||||
isUnrecoverableRefreshError,
|
||||
} = tokenRefresh as unknown as {
|
||||
extractOAuthErrorCode: (raw: unknown) => string | null;
|
||||
refreshClaudeOAuthToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
|
||||
refreshClineToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
|
||||
refreshQoderToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
|
||||
refreshGitHubToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
|
||||
isUnrecoverableRefreshError: (r: unknown) => boolean;
|
||||
};
|
||||
|
||||
function rawResponse(body: string, status = 400, contentType = "application/json") {
|
||||
return new Response(body, { status, headers: { "content-type": contentType } });
|
||||
}
|
||||
|
||||
async function withMockedFetch<T>(impl: typeof fetch, fn: () => Promise<T>): Promise<T> {
|
||||
const original = globalThis.fetch;
|
||||
globalThis.fetch = impl;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
globalThis.fetch = original;
|
||||
}
|
||||
}
|
||||
|
||||
// ── extractOAuthErrorCode: shape matrix ─────────────────────────────────────
|
||||
|
||||
test("extractOAuthErrorCode: canonical object { error: 'invalid_grant' }", () => {
|
||||
assert.equal(extractOAuthErrorCode({ error: "invalid_grant" }), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: nested object { error: { code: 'invalid_grant' } }", () => {
|
||||
assert.equal(extractOAuthErrorCode({ error: { code: "invalid_grant" } }), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: bare string code 'invalid_grant'", () => {
|
||||
assert.equal(extractOAuthErrorCode("invalid_grant"), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: JSON string body '{\"error\":\"invalid_grant\"}'", () => {
|
||||
assert.equal(extractOAuthErrorCode('{"error": "invalid_grant"}'), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: double-encoded JSON string (the production case)", () => {
|
||||
// response.json() returned the inner JSON AS a string (proxy double-encode)
|
||||
const doubleEncoded = JSON.stringify('{"error": "invalid_grant", "error_description": "x"}');
|
||||
const parsedOnce = JSON.parse(doubleEncoded); // a string, still JSON inside
|
||||
assert.equal(extractOAuthErrorCode(parsedOnce), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: catch-branch shape { error: '<raw json text>' }", () => {
|
||||
// refreshClaudeOAuthToken's catch did errorBody = { error: text }
|
||||
const errorBody = { error: '{"error": "invalid_grant", "error_description": "x"}' };
|
||||
assert.equal(extractOAuthErrorCode(errorBody), "invalid_grant");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: invalid_request is recognized", () => {
|
||||
assert.equal(extractOAuthErrorCode({ error: "invalid_request" }), "invalid_request");
|
||||
});
|
||||
|
||||
test("extractOAuthErrorCode: transient errors are NOT misclassified (no false positives)", () => {
|
||||
assert.equal(extractOAuthErrorCode({ error: "server_error" }), null);
|
||||
assert.equal(extractOAuthErrorCode("rate_limited"), null);
|
||||
assert.equal(extractOAuthErrorCode("<!DOCTYPE html><html>502 Bad Gateway</html>"), null);
|
||||
assert.equal(extractOAuthErrorCode(""), null);
|
||||
assert.equal(extractOAuthErrorCode(null), null);
|
||||
assert.equal(extractOAuthErrorCode(undefined), null);
|
||||
});
|
||||
|
||||
// ── refreshClaudeOAuthToken: every shape → unrecoverable sentinel ────────────
|
||||
|
||||
const SENTINEL_SHAPES: Array<{ name: string; body: string; ct?: string }> = [
|
||||
{ name: "canonical object", body: '{"error": "invalid_grant", "error_description": "Refresh token not found or invalid"}' },
|
||||
{ name: "double-encoded JSON string", body: JSON.stringify('{"error": "invalid_grant", "error_description": "x"}') },
|
||||
{ name: "bare string code", body: '"invalid_grant"' },
|
||||
{ name: "nested error.code", body: '{"error": {"code": "invalid_grant", "message": "x"}}' },
|
||||
{ name: "json served as text/plain", body: '{"error": "invalid_grant"}', ct: "text/plain" },
|
||||
];
|
||||
|
||||
for (const shape of SENTINEL_SHAPES) {
|
||||
test(`refreshClaudeOAuthToken → unrecoverable sentinel for shape: ${shape.name}`, async () => {
|
||||
await withMockedFetch(
|
||||
(async () => rawResponse(shape.body, 400, shape.ct ?? "application/json")) as unknown as typeof fetch,
|
||||
async () => {
|
||||
const result = await refreshClaudeOAuthToken("dead-refresh-token");
|
||||
assert.ok(
|
||||
isUnrecoverableRefreshError(result),
|
||||
`shape "${shape.name}" must yield an unrecoverable sentinel, got ${JSON.stringify(result)}`
|
||||
);
|
||||
assert.equal((result as { code?: string }).code, "invalid_grant");
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test("refreshClaudeOAuthToken: transient 500 server_error stays null (NOT unrecoverable)", async () => {
|
||||
await withMockedFetch(
|
||||
(async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch,
|
||||
async () => {
|
||||
const result = await refreshClaudeOAuthToken("token");
|
||||
assert.equal(result, null, "transient errors must remain recoverable (null), not deactivate the account");
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test("refreshClaudeOAuthToken: 502 HTML gateway error stays null", async () => {
|
||||
await withMockedFetch(
|
||||
(async () => rawResponse("<html>502 Bad Gateway</html>", 502, "text/html")) as unknown as typeof fetch,
|
||||
async () => {
|
||||
const result = await refreshClaudeOAuthToken("token");
|
||||
assert.equal(result, null);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
// ── Previously-frágil refreshers that NEVER emitted a sentinel ──────────────
|
||||
// refreshClineToken / refreshQoderToken / refreshGitHubToken returned null on
|
||||
// ANY error → invalid_grant looked recoverable → HealthCheck refresh loop.
|
||||
|
||||
// Note: refreshQoderToken also got the same fix, but it early-returns null via a
|
||||
// config guard (no clientId/secret in the test env) so it can't be exercised here.
|
||||
const FRAGILE_REFRESHERS: Array<{
|
||||
name: string;
|
||||
fn: (rt: string) => Promise<unknown>;
|
||||
}> = [
|
||||
{ name: "refreshClineToken", fn: (rt) => refreshClineToken(rt) },
|
||||
{ name: "refreshGitHubToken", fn: (rt) => refreshGitHubToken(rt) },
|
||||
];
|
||||
|
||||
void refreshQoderToken; // fixed in source; not unit-testable without OAuth config
|
||||
|
||||
for (const r of FRAGILE_REFRESHERS) {
|
||||
test(`${r.name}: invalid_grant now yields an unrecoverable sentinel`, async () => {
|
||||
await withMockedFetch(
|
||||
(async () => rawResponse('{"error": "invalid_grant"}', 400)) as unknown as typeof fetch,
|
||||
async () => {
|
||||
const result = await r.fn("dead-token");
|
||||
assert.ok(
|
||||
isUnrecoverableRefreshError(result),
|
||||
`${r.name} must classify invalid_grant as unrecoverable, got ${JSON.stringify(result)}`
|
||||
);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test(`${r.name}: transient 500 server_error stays null`, async () => {
|
||||
await withMockedFetch(
|
||||
(async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch,
|
||||
async () => {
|
||||
const result = await r.fn("token");
|
||||
assert.equal(result, null, `${r.name} must keep transient errors recoverable`);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
89
tests/unit/token-health-check-circuit-breaker.test.ts
Normal file
89
tests/unit/token-health-check-circuit-breaker.test.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* TDD — HealthCheck refresh circuit breaker.
|
||||
*
|
||||
* Production incident: claude/aa5dd5cf refreshed 1352× and kimi-coding 270×,
|
||||
* each retrying every 60s forever because a refresh that returns `null`
|
||||
* (network blip, dead proxy, or an unclassified error) leaves the connection
|
||||
* `active`, so the next sweep tries again immediately — no backoff.
|
||||
*
|
||||
* The circuit breaker tracks consecutive refresh failures in
|
||||
* providerSpecificData.refreshCircuit and computes an exponential backoff
|
||||
* window. While inside the window, checkConnection must SKIP the refresh
|
||||
* instead of hammering every tick. A successful refresh clears the circuit.
|
||||
*
|
||||
* These exercise the pure helpers (no DB/network needed).
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts");
|
||||
const { buildRefreshFailureUpdate, isInRefreshBackoff, getRefreshBackoffUntil } =
|
||||
tokenHealthCheck as unknown as {
|
||||
buildRefreshFailureUpdate: (conn: any, now: string) => any;
|
||||
isInRefreshBackoff: (conn: any, nowMs: number) => boolean;
|
||||
getRefreshBackoffUntil: (streak: number, now: string) => string;
|
||||
};
|
||||
|
||||
const NOW = "2026-06-11T12:00:00.000Z";
|
||||
const NOW_MS = new Date(NOW).getTime();
|
||||
|
||||
test("getRefreshBackoffUntil grows exponentially and caps", () => {
|
||||
const min = (iso: string) => Math.round((new Date(iso).getTime() - NOW_MS) / 60000);
|
||||
assert.equal(min(getRefreshBackoffUntil(1, NOW)), 5); // 5 * 2^0
|
||||
assert.equal(min(getRefreshBackoffUntil(2, NOW)), 10); // 5 * 2^1
|
||||
assert.equal(min(getRefreshBackoffUntil(3, NOW)), 20);
|
||||
assert.equal(min(getRefreshBackoffUntil(4, NOW)), 40);
|
||||
assert.ok(min(getRefreshBackoffUntil(20, NOW)) <= 240, "must cap at 4h");
|
||||
});
|
||||
|
||||
test("buildRefreshFailureUpdate starts a circuit streak of 1 on first failure", () => {
|
||||
const update = buildRefreshFailureUpdate({ testStatus: "active" }, NOW);
|
||||
assert.equal(update.testStatus, "active", "first failure stays routable");
|
||||
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
|
||||
assert.ok(
|
||||
new Date(update.providerSpecificData.refreshCircuit.until).getTime() > NOW_MS,
|
||||
"must set a future backoff window"
|
||||
);
|
||||
});
|
||||
|
||||
test("buildRefreshFailureUpdate increments the streak across consecutive failures", () => {
|
||||
const update = buildRefreshFailureUpdate(
|
||||
{ testStatus: "active", providerSpecificData: { refreshCircuit: { streak: 3 } } },
|
||||
NOW
|
||||
);
|
||||
assert.equal(update.providerSpecificData.refreshCircuit.streak, 4);
|
||||
});
|
||||
|
||||
test("buildRefreshFailureUpdate preserves unrelated providerSpecificData", () => {
|
||||
const update = buildRefreshFailureUpdate(
|
||||
{ testStatus: "active", providerSpecificData: { projectId: "p-123", copilotToken: "x" } },
|
||||
NOW
|
||||
);
|
||||
assert.equal(update.providerSpecificData.projectId, "p-123");
|
||||
assert.equal(update.providerSpecificData.copilotToken, "x");
|
||||
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
|
||||
});
|
||||
|
||||
test("isInRefreshBackoff true while within the window, false after", () => {
|
||||
const conn = {
|
||||
providerSpecificData: { refreshCircuit: { until: getRefreshBackoffUntil(2, NOW) } },
|
||||
};
|
||||
assert.equal(isInRefreshBackoff(conn, NOW_MS), true, "10min window, now → inside");
|
||||
assert.equal(isInRefreshBackoff(conn, NOW_MS + 11 * 60000), false, "after 11min → outside");
|
||||
});
|
||||
|
||||
test("isInRefreshBackoff false when no circuit recorded", () => {
|
||||
assert.equal(isInRefreshBackoff({}, NOW_MS), false);
|
||||
assert.equal(isInRefreshBackoff({ providerSpecificData: {} }, NOW_MS), false);
|
||||
assert.equal(isInRefreshBackoff({ providerSpecificData: { refreshCircuit: {} } }, NOW_MS), false);
|
||||
});
|
||||
|
||||
test("expired connections still track expiredRetryCount AND the circuit", () => {
|
||||
const update = buildRefreshFailureUpdate(
|
||||
{ testStatus: "expired", expiredRetryCount: 1 },
|
||||
NOW
|
||||
);
|
||||
assert.equal(update.testStatus, "expired");
|
||||
assert.equal(update.expiredRetryCount, 2);
|
||||
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
|
||||
});
|
||||
Reference in New Issue
Block a user