From 64f040bdddd7ef207de057be0ab97d029e3f958b Mon Sep 17 00:00:00 2001 From: Oleg Saprykin Date: Fri, 20 Mar 2026 18:07:56 +0300 Subject: [PATCH 1/3] fix: drop Bottleneck queue on 429 instead of waiting for reservoir refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a provider returns 429 (rate limit exceeded), the rate limit manager was setting reservoir=0 and waiting for reservoirRefreshInterval before releasing queued requests. For providers with long rate limit windows (e.g. Codex with hours-long resets), this caused all queued requests to hang indefinitely — they never timed out or returned an error. This prevented upstream callers (e.g. LiteLLM) from triggering fallback to alternative providers, effectively making the entire model unavailable until the rate limit window expired. Fix: on 429, call limiter.stop({ dropWaitingJobs: true }) to immediately fail all queued requests, then delete the limiter from the Map so getLimiter() creates a fresh instance for subsequent requests. Co-Authored-By: Claude Opus 4.6 (1M context) --- open-sse/services/rateLimitManager.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index a41612e096..3d8facc53d 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -339,14 +339,19 @@ export function updateFromHeaders(provider, connectionId, headers, status, model // Handle 429 — rate limited if (status === 429) { const retryAfterMs = parseResetTime(retryAfterStr) || 60000; // Default 60s + const counts = limiter.counts(); + const limiterKey = `${provider}:${connectionId}`; console.log( - `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — 429 received, pausing for ${Math.ceil(retryAfterMs / 1000)}s` + `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — 429 received, pausing for ${Math.ceil(retryAfterMs / 1000)}s, dropping ${counts.QUEUED} queued request(s)` ); - limiter.updateSettings({ - reservoir: 0, - reservoirRefreshAmount: limit || 60, - reservoirRefreshInterval: retryAfterMs, + // Stop the limiter and drop all waiting jobs so they fail immediately + // instead of hanging in the queue until reservoir refreshes (which can + // be hours for providers like Codex with long rate limit windows). + // This lets upstream callers (e.g. LiteLLM) trigger fallback to other providers. + // After stop, delete from Map so getLimiter() creates a fresh instance. + limiter.stop({ dropWaitingJobs: true }).then(() => { + limiters.delete(limiterKey); }); return; } From b29456c8e5914469ea5404c02487c31427254fa0 Mon Sep 17 00:00:00 2001 From: Oleg Saprykin Date: Fri, 20 Mar 2026 18:27:46 +0300 Subject: [PATCH 2/3] fix: catch stop() already called on concurrent 429s Multiple concurrent requests can receive 429 simultaneously, causing stop() to be called on an already-stopped limiter. Add .catch() to prevent unhandled rejection. Co-Authored-By: Claude Opus 4.6 (1M context) --- open-sse/services/rateLimitManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index 3d8facc53d..d7cebc4935 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -352,7 +352,7 @@ export function updateFromHeaders(provider, connectionId, headers, status, model // After stop, delete from Map so getLimiter() creates a fresh instance. limiter.stop({ dropWaitingJobs: true }).then(() => { limiters.delete(limiterKey); - }); + }).catch(() => {}); return; } From c009c4060629700a99e8554376b63ff1643bea63 Mon Sep 17 00:00:00 2001 From: Oleg Saprykin Date: Fri, 20 Mar 2026 18:31:36 +0300 Subject: [PATCH 3/3] refactor: use .finally() to always delete limiter from Map Address bot review feedback: use .finally() instead of .then()/.catch() so limiters.delete() runs regardless of whether stop() succeeds or throws (e.g. already stopped by concurrent 429). Co-Authored-By: Claude Opus 4.6 (1M context) --- open-sse/services/rateLimitManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index d7cebc4935..70e0946cfb 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -350,9 +350,9 @@ export function updateFromHeaders(provider, connectionId, headers, status, model // be hours for providers like Codex with long rate limit windows). // This lets upstream callers (e.g. LiteLLM) trigger fallback to other providers. // After stop, delete from Map so getLimiter() creates a fresh instance. - limiter.stop({ dropWaitingJobs: true }).then(() => { + limiter.stop({ dropWaitingJobs: true }).finally(() => { limiters.delete(limiterKey); - }).catch(() => {}); + }); return; }