fix(oauth): resilient refresh error classification + healthcheck circuit breaker

Root cause of the production token-invalidation storm (claude/aa5dd5cf
refreshed 1352x, kimi-coding 270x): when a refresh endpoint returned
invalid_grant in a non-canonical body shape (a JSON string, a double-encoded
string, a nested {error:{code}}, or raw text wrapped by a proxy/MITM),
refreshClaudeOAuthToken's errorBody.error === "invalid_grant" check failed and
returned null instead of the unrecoverable sentinel. The HealthCheck then kept
the connection active and re-tried every 60s forever (the loop + log flood +
upstream hammering that contributed to the 429s).

- extractOAuthErrorCode(): shape-agnostic OAuth error extractor (object, nested,
  bare string, double-encoded JSON string, raw text) restricted to a known
  unrecoverable set so transient errors (server_error, 502 HTML) never become
  false positives.
- refreshClaudeOAuthToken + refreshAccessToken/Cline/Qoder/GitHub now classify
  invalid_grant/invalid_request via the helper -> always emit the unrecoverable
  sentinel so the HealthCheck deactivates cleanly instead of looping.
- HealthCheck refresh circuit breaker: track consecutive refresh failures in
  providerSpecificData.refreshCircuit and back off exponentially (5->10->20...
  ->240min cap) instead of retrying every 60s; cleared on a successful refresh.
  Stops the loop for null/network failures (e.g. dead proxy / Kimi fetch failed).

TDD: 26 new tests (19 resilience + 7 circuit breaker); 113 oauth/refresh/
healthcheck tests green. typecheck:core=0, lint clean.
This commit is contained in:
diegosouzapw
2026-06-11 20:53:55 -03:00
parent e771776b67
commit cb1c5471dd
4 changed files with 445 additions and 15 deletions

View File

@@ -181,6 +181,93 @@ function getRefreshCacheKey(provider, refreshToken) {
return `${provider}:${tokenHash}`;
}
/**
* OAuth2 error codes that mean the refresh token is permanently dead and
* retrying will never succeed → callers must emit the unrecoverable sentinel
* so the HealthCheck deactivates the account instead of looping every 60s.
* Deliberately EXCLUDES transient codes (server_error, temporarily_unavailable,
* slow_down) so we never deactivate an account over a recoverable blip.
*/
const UNRECOVERABLE_OAUTH_ERROR_CODES = new Set([
"invalid_grant",
"invalid_request",
"refresh_token_reused",
"invalid_token",
"expired_token",
"unauthorized_client",
"access_denied",
]);
/**
* Extract a canonical OAuth error code from a refresh-endpoint error body of
* ANY shape. Production proxies/MITMs deliver the same `invalid_grant` 400 in
* several shapes — a plain object `{error:"invalid_grant"}`, a nested
* `{error:{code:"invalid_grant"}}`, a JSON **string** (double-encoded body),
* or the raw JSON text wrapped as `{error:"<json text>"}` by a catch branch.
* The old `errorBody.error === "invalid_grant"` only matched the first shape,
* so the others returned `null` → the HealthCheck refresh loop (root cause of
* the 1352× claude/aa5dd5cf invalidation storm).
*
* Returns the matched code (only if it is in UNRECOVERABLE_OAUTH_ERROR_CODES)
* or null. Never matches loosely — a known code is accepted only when it is a
* bare code string or the value of an `"error"`/`"error_code"` field, so a 502
* HTML page or a `server_error` body never becomes a false positive.
*/
export function extractOAuthErrorCode(raw: unknown, depth = 0): string | null {
if (raw == null || depth > 6) return null;
if (typeof raw === "string") {
const s = raw.trim();
if (!s) return null;
if (UNRECOVERABLE_OAUTH_ERROR_CODES.has(s)) return s;
// The string may itself be JSON (a double-encoded body, or the raw text).
if (s[0] === "{" || s[0] === "[" || s[0] === '"') {
try {
const nested = extractOAuthErrorCode(JSON.parse(s), depth + 1);
if (nested) return nested;
} catch {
// not valid JSON — fall through to the field scan
}
}
// Safety net: a known code appearing as the value of an "error"/"error_code"
// field inside otherwise-unparsed text. Scoped to avoid false positives.
const m = s.match(/"error(?:_code)?"\s*:\s*"([a-z_]+)"/i);
if (m && UNRECOVERABLE_OAUTH_ERROR_CODES.has(m[1])) return m[1];
return null;
}
if (typeof raw === "object") {
const o = raw as Record<string, unknown>;
return (
extractOAuthErrorCode(o.error, depth + 1) ??
extractOAuthErrorCode(o.code, depth + 1) ??
extractOAuthErrorCode(o.error_code, depth + 1)
);
}
return null;
}
/**
* Read an error response body ONCE and classify it. Returns the raw text (for
* logging) and the extracted unrecoverable OAuth code (or null). Reading once
* avoids the double-read bug where `response.json()` consumes the stream and a
* later `response.text()` returns empty.
*/
async function readRefreshErrorBody(
response: Response
): Promise<{ rawText: string; code: string | null }> {
const rawText = await response.text().catch(() => "");
let parsed: unknown = rawText;
try {
parsed = JSON.parse(rawText);
} catch {
// keep rawText as-is
}
const code = extractOAuthErrorCode(parsed) ?? extractOAuthErrorCode(rawText);
return { rawText, code };
}
/**
* Refresh OAuth access token using refresh token
*/
@@ -229,6 +316,10 @@ export async function refreshAccessToken(
status: response.status,
error: errorText,
});
const code = extractOAuthErrorCode(errorText);
if (code === "invalid_grant" || code === "invalid_request") {
return { error: "unrecoverable_refresh_error", code };
}
return null;
}
@@ -401,6 +492,10 @@ export async function refreshClineToken(refreshToken, log, proxyConfig: unknown
status: response.status,
error: errorText,
});
const code = extractOAuthErrorCode(errorText);
if (code === "invalid_grant" || code === "invalid_request") {
return { error: "unrecoverable_refresh_error", code };
}
return null;
}
@@ -664,19 +759,17 @@ export async function refreshClaudeOAuthToken(refreshToken, log, proxyConfig: un
);
if (!response.ok) {
let errorBody: { error?: string; error_description?: string } = {};
try {
errorBody = await response.json();
} catch {
const text = await response.text().catch(() => "unknown");
errorBody = { error: text };
}
// Read + classify the body ONCE, shape-agnostic. A proxy/MITM can deliver
// the invalid_grant 400 as a JSON string, a double-encoded string, a
// nested {error:{code}}, or raw text — all must yield the sentinel so the
// HealthCheck deactivates instead of looping every 60s.
const { rawText, code } = await readRefreshErrorBody(response);
log?.error?.("TOKEN_REFRESH", "Failed to refresh Claude OAuth token", {
status: response.status,
error: errorBody,
error: rawText.slice(0, 300),
});
if (errorBody.error === "invalid_grant" || errorBody.error === "invalid_request") {
return { error: "unrecoverable_refresh_error", code: errorBody.error };
if (code === "invalid_grant" || code === "invalid_request") {
return { error: "unrecoverable_refresh_error", code };
}
return null;
}
@@ -1186,6 +1279,10 @@ export async function refreshQoderToken(refreshToken, log, proxyConfig: unknown
status: response.status,
error: errorText,
});
const code = extractOAuthErrorCode(errorText);
if (code === "invalid_grant" || code === "invalid_request") {
return { error: "unrecoverable_refresh_error", code };
}
return null;
}
@@ -1230,6 +1327,10 @@ export async function refreshGitHubToken(refreshToken, log, proxyConfig: unknown
status: response.status,
error: errorText,
});
const code = extractOAuthErrorCode(errorText);
if (code === "invalid_grant" || code === "invalid_request") {
return { error: "unrecoverable_refresh_error", code };
}
return null;
}

View File

@@ -76,10 +76,38 @@ function getEffectiveTokenExpiryMs(conn: any): number {
return Number.isFinite(expiryMs) ? expiryMs : 0;
}
// ── Refresh circuit breaker ───────────────────────────────────────────────
// A refresh that returns null (network blip, dead proxy, unclassified error)
// leaves the connection active, so the next 60s sweep retries immediately —
// the production refresh loop (claude/aa5dd5cf 1352×, kimi 270×). We track
// consecutive failures and back off exponentially so a stuck connection stops
// hammering the upstream (and stops flooding the logs) instead of looping.
const REFRESH_CIRCUIT_BASE_MIN = 5;
const REFRESH_CIRCUIT_MAX_MIN = 240; // cap at 4h
export function getRefreshBackoffUntil(streak: number, now: string): string {
const steps = Math.max(0, streak - 1);
const backoffMin = Math.min(REFRESH_CIRCUIT_BASE_MIN * 2 ** steps, REFRESH_CIRCUIT_MAX_MIN);
return new Date(new Date(now).getTime() + backoffMin * 60 * 1000).toISOString();
}
export function isInRefreshBackoff(conn: any, nowMs: number): boolean {
const until = conn?.providerSpecificData?.refreshCircuit?.until;
if (typeof until !== "string") return false;
const untilMs = new Date(until).getTime();
return Number.isFinite(untilMs) && untilMs > nowMs;
}
export function buildRefreshFailureUpdate(conn: any, now: string) {
const wasExpired = conn.testStatus === "expired";
const retryCount = (conn.expiredRetryCount ?? 0) + (wasExpired ? 1 : 0);
// Circuit breaker: increment the consecutive-failure streak and set an
// exponential backoff window so the next sweep skips this connection instead
// of retrying every 60s. Cleared by a successful refresh (clearRefreshCircuit).
const prevStreak = conn.providerSpecificData?.refreshCircuit?.streak ?? 0;
const streak = prevStreak + 1;
return {
lastHealthCheckAt: now,
// A failed background refresh should not evict otherwise healthy accounts
@@ -91,10 +119,28 @@ export function buildRefreshFailureUpdate(conn: any, now: string) {
lastErrorType: "token_refresh_failed",
lastErrorSource: "oauth",
errorCode: "refresh_failed",
providerSpecificData: {
...(conn.providerSpecificData || {}),
refreshCircuit: { streak, until: getRefreshBackoffUntil(streak, now), lastFailAt: now },
},
...(wasExpired ? { expiredRetryCount: retryCount, expiredRetryAt: now } : {}),
};
}
/**
* Strip the refresh circuit breaker state from providerSpecificData after a
* successful refresh, so the streak/backoff resets cleanly.
*/
export function clearRefreshCircuit(
providerSpecificData: Record<string, unknown> | null | undefined
): Record<string, unknown> | undefined {
if (!providerSpecificData || typeof providerSpecificData !== "object") return undefined;
if (!("refreshCircuit" in providerSpecificData)) return undefined;
const next = { ...providerSpecificData };
delete next.refreshCircuit;
return next;
}
function isEnvFlagEnabled(name: string): boolean {
const value = process.env[name];
if (!value) return false;
@@ -355,6 +401,14 @@ export async function checkConnection(conn) {
if (!isAboutToExpire && !shouldRefreshByInterval) return;
// Circuit breaker: if recent refreshes for this connection failed, wait out
// the exponential backoff window instead of retrying every 60s tick. This is
// what stops the refresh loop when getAccessToken keeps returning null
// (dead proxy / network blip / unclassified upstream error).
if (isInRefreshBackoff(conn, Date.now())) {
return;
}
const reason = isAboutToExpire ? "token expiring soon" : `interval: ${intervalMin}min`;
log(`${LOG_PREFIX} Refreshing ${conn.provider}/${getConnectionLogLabel(conn)} (${reason})`);
@@ -427,11 +481,17 @@ export async function checkConnection(conn) {
updateData.expiresAt = expiresAt;
updateData.tokenExpiresAt = expiresAt;
}
if (refreshResult.providerSpecificData) {
updateData.providerSpecificData = {
...(conn.providerSpecificData || {}),
...refreshResult.providerSpecificData,
};
// Merge new providerSpecificData and ALWAYS clear the refresh circuit
// breaker streak on a successful refresh.
const mergedProviderData = {
...(conn.providerSpecificData || {}),
...(refreshResult.providerSpecificData || {}),
};
const clearedProviderData = clearRefreshCircuit(mergedProviderData);
if (clearedProviderData !== undefined) {
updateData.providerSpecificData = clearedProviderData;
} else if (refreshResult.providerSpecificData) {
updateData.providerSpecificData = mergedProviderData;
}
await updateProviderConnection(conn.id, updateData);
persistedResult = refreshResult;

View File

@@ -0,0 +1,180 @@
/**
* TDD — OAuth refresh error classification must be resilient to body SHAPE.
*
* Root cause of the production "1352× refresh loop" (claude/aa5dd5cf): when the
* Anthropic 400 body reaches `refreshClaudeOAuthToken` in a non-canonical shape
* (a JSON string instead of an object, a double-encoded string, a nested
* `{error:{code}}`, or the raw text in the catch branch), the old check
* `errorBody.error === "invalid_grant"` evaluated to false, so the function
* returned `null` instead of the `unrecoverable_refresh_error` sentinel.
*
* `null` makes the HealthCheck treat it as a recoverable failure → keeps the
* connection `active` → retries every 60s forever (the loop). The fix is a
* shape-agnostic extractor used by all refreshers that classify invalid_grant.
*
* These tests FAIL before the fix (functions return null) and pass after.
*/
import test from "node:test";
import assert from "node:assert/strict";
const tokenRefresh = await import("../../open-sse/services/tokenRefresh.ts");
const {
extractOAuthErrorCode,
refreshClaudeOAuthToken,
refreshClineToken,
refreshQoderToken,
refreshGitHubToken,
isUnrecoverableRefreshError,
} = tokenRefresh as unknown as {
extractOAuthErrorCode: (raw: unknown) => string | null;
refreshClaudeOAuthToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
refreshClineToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
refreshQoderToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
refreshGitHubToken: (rt: string, log?: unknown, proxy?: unknown) => Promise<unknown>;
isUnrecoverableRefreshError: (r: unknown) => boolean;
};
function rawResponse(body: string, status = 400, contentType = "application/json") {
return new Response(body, { status, headers: { "content-type": contentType } });
}
async function withMockedFetch<T>(impl: typeof fetch, fn: () => Promise<T>): Promise<T> {
const original = globalThis.fetch;
globalThis.fetch = impl;
try {
return await fn();
} finally {
globalThis.fetch = original;
}
}
// ── extractOAuthErrorCode: shape matrix ─────────────────────────────────────
test("extractOAuthErrorCode: canonical object { error: 'invalid_grant' }", () => {
assert.equal(extractOAuthErrorCode({ error: "invalid_grant" }), "invalid_grant");
});
test("extractOAuthErrorCode: nested object { error: { code: 'invalid_grant' } }", () => {
assert.equal(extractOAuthErrorCode({ error: { code: "invalid_grant" } }), "invalid_grant");
});
test("extractOAuthErrorCode: bare string code 'invalid_grant'", () => {
assert.equal(extractOAuthErrorCode("invalid_grant"), "invalid_grant");
});
test("extractOAuthErrorCode: JSON string body '{\"error\":\"invalid_grant\"}'", () => {
assert.equal(extractOAuthErrorCode('{"error": "invalid_grant"}'), "invalid_grant");
});
test("extractOAuthErrorCode: double-encoded JSON string (the production case)", () => {
// response.json() returned the inner JSON AS a string (proxy double-encode)
const doubleEncoded = JSON.stringify('{"error": "invalid_grant", "error_description": "x"}');
const parsedOnce = JSON.parse(doubleEncoded); // a string, still JSON inside
assert.equal(extractOAuthErrorCode(parsedOnce), "invalid_grant");
});
test("extractOAuthErrorCode: catch-branch shape { error: '<raw json text>' }", () => {
// refreshClaudeOAuthToken's catch did errorBody = { error: text }
const errorBody = { error: '{"error": "invalid_grant", "error_description": "x"}' };
assert.equal(extractOAuthErrorCode(errorBody), "invalid_grant");
});
test("extractOAuthErrorCode: invalid_request is recognized", () => {
assert.equal(extractOAuthErrorCode({ error: "invalid_request" }), "invalid_request");
});
test("extractOAuthErrorCode: transient errors are NOT misclassified (no false positives)", () => {
assert.equal(extractOAuthErrorCode({ error: "server_error" }), null);
assert.equal(extractOAuthErrorCode("rate_limited"), null);
assert.equal(extractOAuthErrorCode("<!DOCTYPE html><html>502 Bad Gateway</html>"), null);
assert.equal(extractOAuthErrorCode(""), null);
assert.equal(extractOAuthErrorCode(null), null);
assert.equal(extractOAuthErrorCode(undefined), null);
});
// ── refreshClaudeOAuthToken: every shape → unrecoverable sentinel ────────────
const SENTINEL_SHAPES: Array<{ name: string; body: string; ct?: string }> = [
{ name: "canonical object", body: '{"error": "invalid_grant", "error_description": "Refresh token not found or invalid"}' },
{ name: "double-encoded JSON string", body: JSON.stringify('{"error": "invalid_grant", "error_description": "x"}') },
{ name: "bare string code", body: '"invalid_grant"' },
{ name: "nested error.code", body: '{"error": {"code": "invalid_grant", "message": "x"}}' },
{ name: "json served as text/plain", body: '{"error": "invalid_grant"}', ct: "text/plain" },
];
for (const shape of SENTINEL_SHAPES) {
test(`refreshClaudeOAuthToken → unrecoverable sentinel for shape: ${shape.name}`, async () => {
await withMockedFetch(
(async () => rawResponse(shape.body, 400, shape.ct ?? "application/json")) as unknown as typeof fetch,
async () => {
const result = await refreshClaudeOAuthToken("dead-refresh-token");
assert.ok(
isUnrecoverableRefreshError(result),
`shape "${shape.name}" must yield an unrecoverable sentinel, got ${JSON.stringify(result)}`
);
assert.equal((result as { code?: string }).code, "invalid_grant");
}
);
});
}
test("refreshClaudeOAuthToken: transient 500 server_error stays null (NOT unrecoverable)", async () => {
await withMockedFetch(
(async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch,
async () => {
const result = await refreshClaudeOAuthToken("token");
assert.equal(result, null, "transient errors must remain recoverable (null), not deactivate the account");
}
);
});
test("refreshClaudeOAuthToken: 502 HTML gateway error stays null", async () => {
await withMockedFetch(
(async () => rawResponse("<html>502 Bad Gateway</html>", 502, "text/html")) as unknown as typeof fetch,
async () => {
const result = await refreshClaudeOAuthToken("token");
assert.equal(result, null);
}
);
});
// ── Previously-frágil refreshers that NEVER emitted a sentinel ──────────────
// refreshClineToken / refreshQoderToken / refreshGitHubToken returned null on
// ANY error → invalid_grant looked recoverable → HealthCheck refresh loop.
// Note: refreshQoderToken also got the same fix, but it early-returns null via a
// config guard (no clientId/secret in the test env) so it can't be exercised here.
const FRAGILE_REFRESHERS: Array<{
name: string;
fn: (rt: string) => Promise<unknown>;
}> = [
{ name: "refreshClineToken", fn: (rt) => refreshClineToken(rt) },
{ name: "refreshGitHubToken", fn: (rt) => refreshGitHubToken(rt) },
];
void refreshQoderToken; // fixed in source; not unit-testable without OAuth config
for (const r of FRAGILE_REFRESHERS) {
test(`${r.name}: invalid_grant now yields an unrecoverable sentinel`, async () => {
await withMockedFetch(
(async () => rawResponse('{"error": "invalid_grant"}', 400)) as unknown as typeof fetch,
async () => {
const result = await r.fn("dead-token");
assert.ok(
isUnrecoverableRefreshError(result),
`${r.name} must classify invalid_grant as unrecoverable, got ${JSON.stringify(result)}`
);
}
);
});
test(`${r.name}: transient 500 server_error stays null`, async () => {
await withMockedFetch(
(async () => rawResponse('{"error": "server_error"}', 500)) as unknown as typeof fetch,
async () => {
const result = await r.fn("token");
assert.equal(result, null, `${r.name} must keep transient errors recoverable`);
}
);
});
}

View File

@@ -0,0 +1,89 @@
/**
* TDD — HealthCheck refresh circuit breaker.
*
* Production incident: claude/aa5dd5cf refreshed 1352× and kimi-coding 270×,
* each retrying every 60s forever because a refresh that returns `null`
* (network blip, dead proxy, or an unclassified error) leaves the connection
* `active`, so the next sweep tries again immediately — no backoff.
*
* The circuit breaker tracks consecutive refresh failures in
* providerSpecificData.refreshCircuit and computes an exponential backoff
* window. While inside the window, checkConnection must SKIP the refresh
* instead of hammering every tick. A successful refresh clears the circuit.
*
* These exercise the pure helpers (no DB/network needed).
*/
import test from "node:test";
import assert from "node:assert/strict";
const tokenHealthCheck = await import("../../src/lib/tokenHealthCheck.ts");
const { buildRefreshFailureUpdate, isInRefreshBackoff, getRefreshBackoffUntil } =
tokenHealthCheck as unknown as {
buildRefreshFailureUpdate: (conn: any, now: string) => any;
isInRefreshBackoff: (conn: any, nowMs: number) => boolean;
getRefreshBackoffUntil: (streak: number, now: string) => string;
};
const NOW = "2026-06-11T12:00:00.000Z";
const NOW_MS = new Date(NOW).getTime();
test("getRefreshBackoffUntil grows exponentially and caps", () => {
const min = (iso: string) => Math.round((new Date(iso).getTime() - NOW_MS) / 60000);
assert.equal(min(getRefreshBackoffUntil(1, NOW)), 5); // 5 * 2^0
assert.equal(min(getRefreshBackoffUntil(2, NOW)), 10); // 5 * 2^1
assert.equal(min(getRefreshBackoffUntil(3, NOW)), 20);
assert.equal(min(getRefreshBackoffUntil(4, NOW)), 40);
assert.ok(min(getRefreshBackoffUntil(20, NOW)) <= 240, "must cap at 4h");
});
test("buildRefreshFailureUpdate starts a circuit streak of 1 on first failure", () => {
const update = buildRefreshFailureUpdate({ testStatus: "active" }, NOW);
assert.equal(update.testStatus, "active", "first failure stays routable");
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
assert.ok(
new Date(update.providerSpecificData.refreshCircuit.until).getTime() > NOW_MS,
"must set a future backoff window"
);
});
test("buildRefreshFailureUpdate increments the streak across consecutive failures", () => {
const update = buildRefreshFailureUpdate(
{ testStatus: "active", providerSpecificData: { refreshCircuit: { streak: 3 } } },
NOW
);
assert.equal(update.providerSpecificData.refreshCircuit.streak, 4);
});
test("buildRefreshFailureUpdate preserves unrelated providerSpecificData", () => {
const update = buildRefreshFailureUpdate(
{ testStatus: "active", providerSpecificData: { projectId: "p-123", copilotToken: "x" } },
NOW
);
assert.equal(update.providerSpecificData.projectId, "p-123");
assert.equal(update.providerSpecificData.copilotToken, "x");
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
});
test("isInRefreshBackoff true while within the window, false after", () => {
const conn = {
providerSpecificData: { refreshCircuit: { until: getRefreshBackoffUntil(2, NOW) } },
};
assert.equal(isInRefreshBackoff(conn, NOW_MS), true, "10min window, now → inside");
assert.equal(isInRefreshBackoff(conn, NOW_MS + 11 * 60000), false, "after 11min → outside");
});
test("isInRefreshBackoff false when no circuit recorded", () => {
assert.equal(isInRefreshBackoff({}, NOW_MS), false);
assert.equal(isInRefreshBackoff({ providerSpecificData: {} }, NOW_MS), false);
assert.equal(isInRefreshBackoff({ providerSpecificData: { refreshCircuit: {} } }, NOW_MS), false);
});
test("expired connections still track expiredRetryCount AND the circuit", () => {
const update = buildRefreshFailureUpdate(
{ testStatus: "expired", expiredRetryCount: 1 },
NOW
);
assert.equal(update.testStatus, "expired");
assert.equal(update.expiredRetryCount, 2);
assert.equal(update.providerSpecificData.refreshCircuit.streak, 1);
});