From f8d045c275f56f7eb56c0211943088001e879ae3 Mon Sep 17 00:00:00 2001 From: Chris Staley Date: Thu, 2 Apr 2026 15:13:46 -0600 Subject: [PATCH] =?UTF-8?q?fix(gemini):=20per-model=20quota=20isolation=20?= =?UTF-8?q?=E2=80=94=20429=20on=20one=20model=20keeps=20others=20active?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini AI Studio enforces per-model quotas. Previously a 429 on gemini-2.5-pro would mark the entire connection as credits_exhausted, blocking all models on that API key. Three-layer fix: - chatCore: lock model only (not connection) for RATE_LIMITED and QUOTA_EXHAUSTED errors from Gemini - auth: early-return with model-only lockout before terminal status check, so credits_exhausted is never set on the connection - rateLimitManager: use model-scoped limiter keys for Gemini so the Bottleneck queue pauses only the affected model, not the connection - chat: skip markAccountExhaustedFrom429 for Gemini (per-model quotas) --- open-sse/handlers/chatCore.ts | 33 ++++++++++++++++++--------- open-sse/services/rateLimitManager.ts | 7 +++++- src/sse/services/auth.ts | 17 ++++++++++++++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 7a2887b641..33dd3bba45 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1290,11 +1290,12 @@ export async function handleChatCore({ `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` ); } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) { - // For passthrough providers (e.g. Antigravity), each model has independent - // quota. A 429 on one model must NOT lock out the entire connection — other - // models may still have quota available. Use lockModel() instead. + // For providers with per-model quotas (passthrough providers, Gemini), + // each model has independent quota. A 429 on one model must NOT lock out + // the entire connection — other models may still have quota available. + // Use lockModel() instead. const isPassthrough = provider && getPassthroughProviders().has(provider); - if (isPassthrough) { + if (isPassthrough || provider === "gemini") { const { lockModel } = await import("../services/accountFallback.ts"); const cooldown = retryAfterMs || 120_000; // 2 min default, same as COOLDOWN_MS.rateLimit lockModel(provider, connectionId, model, "rate_limited", cooldown); @@ -1317,13 +1318,23 @@ export async function handleChatCore({ ); } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { - await updateProviderConnection(connectionId, { - testStatus: "credits_exhausted", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - }); - console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`); + // Gemini has per-model quotas — lock the model only, not the connection + if (provider === "gemini" && model) { + const { lockModel } = await import("../services/accountFallback.ts"); + const cooldown = retryAfterMs || 120_000; + lockModel(provider, connectionId, model, "quota_exhausted", cooldown); + console.warn( + `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(cooldown / 1000)}s (connection stays active)` + ); + } else { + await updateProviderConnection(connectionId, { + testStatus: "credits_exhausted", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`); + } } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { await updateProviderConnection(connectionId, { isActive: false, diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index 2d7ba8549c..2b0dd54aa2 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -200,6 +200,11 @@ function getLimiterKey(provider, connectionId, model = null) { if (provider === "codex" && model) { return `${provider}:${getCodexRateLimitKey(connectionId, model)}`; } + // Gemini AI Studio has per-model quotas — use model-scoped limiter keys + // so a 429 on one model doesn't pause requests for other models. + if (provider === "gemini" && model) { + return `${provider}:${connectionId}:${model}`; + } return `${provider}:${connectionId}`; } @@ -570,7 +575,7 @@ export function updateFromResponseBody(provider, connectionId, responseBody, sta const { retryAfterMs, reason } = parseRetryAfterFromBody(responseBody); if (retryAfterMs && retryAfterMs > 0) { - const limiter = getLimiter(provider, connectionId, null); + const limiter = getLimiter(provider, connectionId, model); console.log( `🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — body-parsed retry: ${Math.ceil(retryAfterMs / 1000)}s (${reason})` ); diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index dd57be8069..4347c988aa 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -724,6 +724,23 @@ export async function markAccountUnavailable( try { await currentMutex; + // ── Gemini per-model lockout (must be before terminal status check) ── + // Gemini AI Studio has per-model quotas. A 429 on gemini-2.5-pro must NOT + // lock out gemini-2.5-flash on the same API key. Lock the specific model + // only and return early, before any connection-wide state is modified. + if (provider === "gemini" && model && (status === 429 || status === 404)) { + const reason = status === 404 ? "not_found" : "rate_limited"; + const cooldown = status === 404 + ? COOLDOWN_MS.notFoundLocal + : (COOLDOWN_MS.rateLimit || 60_000); + lockModel(provider, connectionId, model, reason, cooldown); + log.info( + "AUTH", + `Gemini model-only lockout for ${model} — ${status} ${reason} ${Math.ceil(cooldown / 1000)}s (connection stays active)` + ); + return { shouldFallback: true, cooldownMs: cooldown }; + } + // Read current connection to get backoffLevel const connectionsRaw = await getProviderConnections({ provider }); const connections = (Array.isArray(connectionsRaw) ? connectionsRaw : [])