diff --git a/src/lib/usage/providerLimits.ts b/src/lib/usage/providerLimits.ts index 7e8811a9e9..ae39205674 100644 --- a/src/lib/usage/providerLimits.ts +++ b/src/lib/usage/providerLimits.ts @@ -499,6 +499,25 @@ export async function maybeClearRecoveredQuotaState( // the previous synthetic-cooldown guard. return connection; } + } else if ( + connection.rateLimitedUntil && + new Date(connection.rateLimitedUntil).getTime() > Date.now() + ) { + // Universal fallback guard for every lastErrorType other than + // "quota_exhausted" (which gets the more precise per-window check above, + // and may legitimately release early once the REAL window has reset even + // while a synthetic rateLimitedUntil is still in the future). A future + // rateLimitedUntil is a hard statement made by the 429/error handler that + // persisted it (src/sse/services/auth.ts, src/app/api/providers/[id]/test/ + // route.ts) — no quota poll finding *some* usable window elsewhere should + // be able to overrule it. Before this fix, ANY lastErrorType other than + // "quota_exhausted" skipped straight to hasTransientState/ + // clearRecoveredProviderState() below with no rateLimitedUntil check at + // all, so a multi-day cooldown (observed: 146h, Z.AI weekly quota) got + // cleared on the very next quota sync a few minutes later — a + // self-restart/burn loop that kept burning real upstream calls against a + // known-exhausted connection (#11277). + return connection; } const hasTransientState = diff --git a/tests/unit/provider-limits-recovery.test.ts b/tests/unit/provider-limits-recovery.test.ts index f6f275525b..eb2bcd0ea6 100644 --- a/tests/unit/provider-limits-recovery.test.ts +++ b/tests/unit/provider-limits-recovery.test.ts @@ -83,7 +83,25 @@ test.after(async () => { }); test("successful GLM quota refresh clears transient rate-limit state", async () => { - const connection = await createGlmConnectionWithTransientCooldown(); + // The cooldown must already be EXPIRED for a successful refresh to clear it + // (#11277: a rateLimitedUntil still in the future is a hard statement from + // the error handler that persisted it — no quota poll may overrule it, + // regardless of lastErrorType). Before #11277's fix this test used a + // still-future rateLimitedUntil and asserted it got cleared anyway, which + // was the same defect class as the reported bug, just a shorter window. + const connection = await providersDb.createProviderConnection({ + provider: "glm", + authType: "apikey", + name: `GLM Recovery ${Date.now()}`, + apiKey: "glm-test-key", + testStatus: "unavailable", + rateLimitedUntil: new Date(Date.now() - 60_000).toISOString(), + lastError: "rate limit exceeded", + lastErrorType: "rate_limited", + lastErrorSource: "executor", + errorCode: 429, + backoffLevel: 2, + }); const connectionId = (connection as { id: string }).id; await withMockedFetch((() => glmQuotaResponse()) as typeof fetch, async () => { @@ -101,6 +119,39 @@ test("successful GLM quota refresh clears transient rate-limit state", async () assert.equal(updated.backoffLevel, 0, "backoffLevel should be reset to 0"); }); +test("a still-future rateLimitedUntil is not cleared by a successful quota refresh, regardless of lastErrorType (#11277)", async () => { + const stillFutureRateLimitedUntil = new Date(Date.now() + 60_000).toISOString(); + const connection = await providersDb.createProviderConnection({ + provider: "glm", + authType: "apikey", + name: `GLM Still Cooling ${Date.now()}`, + apiKey: "glm-test-key", + testStatus: "unavailable", + rateLimitedUntil: stillFutureRateLimitedUntil, + lastError: "rate limit exceeded", + lastErrorType: "rate_limited", + lastErrorSource: "executor", + errorCode: 429, + backoffLevel: 2, + }); + const connectionId = (connection as { id: string }).id; + + await withMockedFetch((() => glmQuotaResponse()) as typeof fetch, async () => { + await providerLimits.fetchAndPersistProviderLimits(connectionId, "manual"); + }); + + const updated = (await providersDb.getProviderConnectionById(connectionId)) as Record< + string, + unknown + >; + assert.equal( + updated.testStatus, + "unavailable", + "an active cooldown must stay locked even though the quota fetch succeeded" + ); + assert.equal(updated.rateLimitedUntil, stillFutureRateLimitedUntil); +}); + async function createGlmConnectionWithStatus(status: string) { return providersDb.createProviderConnection({ provider: "glm", @@ -334,6 +385,52 @@ test("Claude subscription quota still exhausted keeps the connection locked (no assert.equal(after.rateLimitedUntil, syntheticRateLimitedUntil); }); +test("rate_limit_exceeded cooldown is not cleared early by an unrelated quota window looking usable (#11277)", async () => { + // Reproduces #11277: a connection-scoped cooldown persisted with + // lastErrorType "rate_limit_exceeded" (RateLimitReason.RATE_LIMIT_EXCEEDED) + // and a long rateLimitedUntil (derived from an upstream reset hint — the + // reported production case was ~146h) must NOT be cleared just because the + // next scheduled quota sync reports hasUsableQuota()===true from some + // unrelated window. Before the fix, only lastErrorType==="quota_exhausted" + // reached the rateLimitedUntil guard, so every other reason (including + // rate_limit_exceeded) skipped straight to clearRecoveredProviderState(), + // producing a self-restart/burn loop on a multi-day cooldown. + const farFutureRateLimitedUntil = new Date(Date.now() + 146 * 60 * 60 * 1000).toISOString(); + const created = await providersDb.createProviderConnection({ + provider: "opencode", + authType: "apikey", + name: `OpenCode RateLimitExceeded ${Date.now()}`, + apiKey: "opencode-test-key", + testStatus: "unavailable", + isActive: true, + lastError: "Account quota exhausted (opencode)", + lastErrorType: "rate_limit_exceeded", + errorCode: 429, + rateLimitedUntil: farFutureRateLimitedUntil, + backoffLevel: 1, + }); + const connectionId = (created as { id: string }).id; + const connection = await providersDb.getProviderConnectionById(connectionId); + + // No `quotas` object at all (degraded/partial fetch shape) — this is the + // exact shape that, pre-fix, fell straight through to hasTransientState + // and cleared the cooldown for any lastErrorType other than quota_exhausted. + const result = await providerLimits.maybeClearRecoveredQuotaState(connection, { + quotas: { unrelated: { unlimited: true } }, + }); + + assert.equal( + result.testStatus, + "unavailable", + "an active rate_limit_exceeded cooldown must stay locked" + ); + + const after = await providersDb.getProviderConnectionById(connectionId); + assert.equal(after.testStatus, "unavailable"); + assert.equal(after.lastErrorType, "rate_limit_exceeded"); + assert.equal(after.rateLimitedUntil, farFutureRateLimitedUntil); +}); + test("CAS primitive clears when expected state matches", async () => { const created = await createGlmConnectionWithTransientCooldown(); const connectionId = (created as { id: string }).id;