From d62b128144fe2ca9e5153fdd34a043dd900426f0 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Mon, 18 May 2026 09:11:32 -0300 Subject: [PATCH] fix(account-fallback): classify Anthropic 'Usage Limit Reached' as quota (#2321) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code Pro/Team users hit a 429 cascade where every Claude account got marked rate-limited for only a few seconds, retried, hit 429 again, and the cycle exhausted all accounts until the 5h subscription window finally reset. Root cause: Anthropic OAuth 429 bodies carry phrases like 'Usage Limit Reached' or 'Claude Pro usage limit reached' that don't contain the word 'quota'. classifyErrorText fell through to RATE_LIMIT_EXCEEDED with the OAuth ~5s base cooldown. Three targeted changes in open-sse/services/accountFallback.ts: 1. classifyErrorText: recognize the Anthropic OAuth phrases ('usage limit reached', 'claude pro usage limit', 'you've reached your usage limit' and variants) and return QUOTA_EXHAUSTED. 2. checkFallbackError: new branch that, when the classifier flags QUOTA_EXHAUSTED for a non-credits/non-daily case, applies a deliberate 1h cooldown (subscription quotas need real time to reset; the existing COOLDOWN_MS.paymentRequired is only 2 minutes). 3. parseRetryFromErrorText: parse absolute ISO 8601 timestamps in the body ('Try again at 2026-05-17T10:00:00Z', 'Please wait until ...'). When present, honor the upstream's stated recovery time instead of the 1h default. Cross-provider fallback for direct (non-combo) calls remains a separate feature — operators wanting that should route Claude through a combo that includes openrouter/claude or another fallback target. --- open-sse/services/accountFallback.ts | 59 ++++++++++- .../account-fallback-anthropic-quota.test.ts | 100 ++++++++++++++++++ 2 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 tests/unit/account-fallback-anthropic-quota.test.ts diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 12271a22cf..0133793b9b 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -786,6 +786,21 @@ function parseDelayString(value: unknown): number | null { export function parseRetryFromErrorText(errorText: unknown): number | null { if (!errorText || typeof errorText !== "string") return null; + // Issue #2321: Anthropic OAuth occasionally embeds an absolute ISO 8601 + // timestamp instead of a relative duration (e.g. "Try again at + // 2026-05-17T10:00:00Z" or "Please wait until 2026-05-17T10:00:00.000Z"). + // Convert to a future-duration in milliseconds if it parses. + const isoMatch = errorText.match( + /\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i + ); + if (isoMatch) { + const parsedTs = Date.parse(isoMatch[1]); + if (Number.isFinite(parsedTs)) { + const waitMs = parsedTs - Date.now(); + if (waitMs > 0) return waitMs; + } + } + const match = errorText.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i); if (!match) { // Also try the variant without "reset after": "will reset after XhYmZs" @@ -824,7 +839,17 @@ export function classifyErrorText(errorText: unknown): RateLimitReasonValue { lower.includes("your quota will reset") || lower.includes("quota has been exceeded") || lower.includes("hour quota") || - lower.includes("billing") + lower.includes("billing") || + // Issue #2321: Anthropic OAuth (Claude Code Pro/Team) 429 bodies surface + // the subscription quota with phrases that contain neither "quota" nor + // "billing". Without these patterns the error was classified as a + // transient RATE_LIMIT_EXCEEDED (~5s base cooldown), which cascades all + // Pro accounts into a tight retry loop until the 5h window resets. + lower.includes("usage limit reached") || + lower.includes("usage limit has been") || + lower.includes("claude pro usage limit") || + lower.includes("you've reached your usage limit") || + lower.includes("you have reached your usage limit") ) { return RateLimitReason.QUOTA_EXHAUSTED; } @@ -1105,6 +1130,38 @@ export function checkFallbackError( }; } + // Issue #2321: Anthropic OAuth (Claude Pro/Team) returns 429 with + // "Usage Limit Reached" for the 5-hour subscription quota. The + // pattern-based classifier now flags these as QUOTA_EXHAUSTED, but + // without a dedicated branch the request would still fall through to + // the generic 429 retry path (~5s base cooldown). Honor any + // upstream retry hint (Retry-After header or ISO timestamp in the + // body) when present, otherwise apply a 1h cooldown so all Pro + // accounts on the same subscription tier stop cycling through tight + // retries until the window genuinely resets. (We deliberately do not + // use COOLDOWN_MS.paymentRequired here — that constant is 2 minutes, + // which is shorter than the recovery time of a subscription quota.) + if ( + shouldUseQuotaSignal && + !isCreditsExhausted(errorStr) && + !isDailyQuotaExhausted(errorStr) && + classifyErrorText(errorStr) === RateLimitReason.QUOTA_EXHAUSTED + ) { + // For a quota error the upstream reset hint (Retry-After header or + // ISO timestamp embedded in the body) is the most accurate wait. + // We honor it even when the resilience profile does not opt-in to + // generic upstream retry hints — a subscription quota has a + // definite recovery time, not a best-effort transient backoff. + const hintMs = getUpstreamRetryHintMs() ?? parseRetryFromErrorText(errorStr) ?? null; + const SUBSCRIPTION_QUOTA_COOLDOWN_MS = 60 * 60 * 1000; // 1 hour + return { + shouldFallback: true, + cooldownMs: hintMs ?? SUBSCRIPTION_QUOTA_COOLDOWN_MS, + reason: RateLimitReason.QUOTA_EXHAUSTED, + usedUpstreamRetryHint: Boolean(hintMs), + }; + } + if ( status === HTTP_STATUS.FORBIDDEN && provider && diff --git a/tests/unit/account-fallback-anthropic-quota.test.ts b/tests/unit/account-fallback-anthropic-quota.test.ts new file mode 100644 index 0000000000..c67b980912 --- /dev/null +++ b/tests/unit/account-fallback-anthropic-quota.test.ts @@ -0,0 +1,100 @@ +/** + * Issue #2321 — Anthropic OAuth (Claude Pro/Team) 429 responses with phrases + * like "Usage Limit Reached" must be classified as QUOTA_EXHAUSTED with a + * long cooldown, not as a generic RATE_LIMIT_EXCEEDED with a ~5s base + * cooldown. Without this fix all Pro accounts on the same subscription + * cascade into a tight retry loop until the 5h quota window resets. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +const { classifyErrorText, parseRetryFromErrorText, checkFallbackError } = + await import("../../open-sse/services/accountFallback.ts"); +const { RateLimitReason } = await import("../../open-sse/config/constants.ts"); + +test("#2321 classifyErrorText flags 'Usage Limit Reached' as QUOTA_EXHAUSTED", () => { + const out = classifyErrorText("Usage Limit Reached. Please wait until 10:00 AM"); + assert.equal(out, RateLimitReason.QUOTA_EXHAUSTED); +}); + +test("#2321 classifyErrorText flags 'Claude Pro usage limit reached' as QUOTA_EXHAUSTED", () => { + const out = classifyErrorText("Claude Pro usage limit reached."); + assert.equal(out, RateLimitReason.QUOTA_EXHAUSTED); +}); + +test("#2321 classifyErrorText flags possessive 'you've reached your usage limit'", () => { + const out = classifyErrorText("Sorry — you've reached your usage limit for this model."); + assert.equal(out, RateLimitReason.QUOTA_EXHAUSTED); +}); + +test("#2321 classifyErrorText still returns RATE_LIMIT_EXCEEDED for generic rate_limit", () => { + // Regression guard: short-term rate limits (per-minute TPM) must remain + // transient so we don't lock accounts for an hour after a normal burst. + const out = classifyErrorText("rate_limit_exceeded: too many requests"); + assert.equal(out, RateLimitReason.RATE_LIMIT_EXCEEDED); +}); + +test("#2321 parseRetryFromErrorText extracts an ISO timestamp", () => { + const future = new Date(Date.now() + 60 * 60 * 1000).toISOString(); + const ms = parseRetryFromErrorText(`Usage Limit Reached. Try again at ${future}`); + assert.ok(ms !== null, "expected non-null wait time"); + assert.ok(ms! > 30 * 60 * 1000 && ms! <= 60 * 60 * 1000, `expected ~1h wait, got ${ms}ms`); +}); + +test("#2321 parseRetryFromErrorText ignores past ISO timestamps", () => { + const past = "2020-01-01T00:00:00Z"; + const ms = parseRetryFromErrorText(`Try again at ${past}`); + assert.equal(ms, null); +}); + +test("#2321 parseRetryFromErrorText still handles the 'reset after Xh' format (backward compat)", () => { + const ms = parseRetryFromErrorText("Your quota will reset after 1h30m"); + assert.equal(ms, (60 + 30) * 60 * 1000); +}); + +test("#2321 checkFallbackError returns ~1h cooldown for OAuth 429 + Usage Limit Reached", () => { + const out = checkFallbackError( + 429, + "Usage Limit Reached. Please wait until 5h.", + 0, + null, + "claude" // OAuth provider + ); + assert.equal(out.shouldFallback, true); + assert.equal(out.reason, RateLimitReason.QUOTA_EXHAUSTED); + // Either the 1h default OR an upstream retry hint — both are far above + // the ~5s base cooldown that caused the cascade. + assert.ok( + out.cooldownMs >= 5 * 60 * 1000, + `expected long cooldown (>=5min), got ${out.cooldownMs}ms` + ); +}); + +test("#2321 checkFallbackError honors ISO timestamp from body when present", () => { + const futureMs = 45 * 60 * 1000; + const future = new Date(Date.now() + futureMs).toISOString(); + const out = checkFallbackError( + 429, + `Claude Pro usage limit reached. Try again at ${future}`, + 0, + null, + "claude" + ); + assert.equal(out.reason, RateLimitReason.QUOTA_EXHAUSTED); + // Within ~30s of the requested wait time. + assert.ok( + Math.abs(out.cooldownMs - futureMs) < 30_000, + `expected ~${futureMs}ms cooldown, got ${out.cooldownMs}ms` + ); +}); + +test("#2321 generic 429 without quota keyword still gets the short cooldown path", () => { + // Regression guard: plain rate-limit must NOT get the 1h cooldown. + const out = checkFallbackError(429, "rate_limit_exceeded", 0, null, "claude"); + assert.equal(out.shouldFallback, true); + // Generic 429 keeps the short retryable path (< 5 min). + assert.ok( + out.cooldownMs < 5 * 60 * 1000, + `expected short cooldown (<5min), got ${out.cooldownMs}ms` + ); +});