From 64f040bdddd7ef207de057be0ab97d029e3f958b Mon Sep 17 00:00:00 2001 From: Oleg Saprykin Date: Fri, 20 Mar 2026 18:07:56 +0300 Subject: [PATCH] fix: drop Bottleneck queue on 429 instead of waiting for reservoir refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a provider returns 429 (rate limit exceeded), the rate limit manager was setting reservoir=0 and waiting for reservoirRefreshInterval before releasing queued requests. For providers with long rate limit windows (e.g. Codex with hours-long resets), this caused all queued requests to hang indefinitely — they never timed out or returned an error. This prevented upstream callers (e.g. LiteLLM) from triggering fallback to alternative providers, effectively making the entire model unavailable until the rate limit window expired. Fix: on 429, call limiter.stop({ dropWaitingJobs: true }) to immediately fail all queued requests, then delete the limiter from the Map so getLimiter() creates a fresh instance for subsequent requests. Co-Authored-By: Claude Opus 4.6 (1M context) --- open-sse/services/rateLimitManager.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index a41612e096..3d8facc53d 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -339,14 +339,19 @@ export function updateFromHeaders(provider, connectionId, headers, status, model // Handle 429 — rate limited if (status === 429) { const retryAfterMs = parseResetTime(retryAfterStr) || 60000; // Default 60s + const counts = limiter.counts(); + const limiterKey = `${provider}:${connectionId}`; console.log( - `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — 429 received, pausing for ${Math.ceil(retryAfterMs / 1000)}s` + `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — 429 received, pausing for ${Math.ceil(retryAfterMs / 1000)}s, dropping ${counts.QUEUED} queued request(s)` ); - limiter.updateSettings({ - reservoir: 0, - reservoirRefreshAmount: limit || 60, - reservoirRefreshInterval: retryAfterMs, + // Stop the limiter and drop all waiting jobs so they fail immediately + // instead of hanging in the queue until reservoir refreshes (which can + // be hours for providers like Codex with long rate limit windows). + // This lets upstream callers (e.g. LiteLLM) trigger fallback to other providers. + // After stop, delete from Map so getLimiter() creates a fresh instance. + limiter.stop({ dropWaitingJobs: true }).then(() => { + limiters.delete(limiterKey); }); return; }