fix(gemini): per-model quota isolation — 429 on one model keeps others active

Gemini AI Studio enforces per-model quotas. Previously a 429 on
gemini-2.5-pro would mark the entire connection as credits_exhausted,
blocking all models on that API key.

Three-layer fix:
- chatCore: lock model only (not connection) for RATE_LIMITED and
  QUOTA_EXHAUSTED errors from Gemini
- auth: early-return with model-only lockout before terminal status
  check, so credits_exhausted is never set on the connection
- rateLimitManager: use model-scoped limiter keys for Gemini so the
  Bottleneck queue pauses only the affected model, not the connection
- chat: skip markAccountExhaustedFrom429 for Gemini (per-model quotas)
This commit is contained in:
Chris Staley
2026-04-02 15:13:46 -06:00
parent 0038fe5ff1
commit f8d045c275
3 changed files with 45 additions and 12 deletions

View File

@@ -1290,11 +1290,12 @@ export async function handleChatCore({
`[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently`
);
} else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) {
// For passthrough providers (e.g. Antigravity), each model has independent
// quota. A 429 on one model must NOT lock out the entire connection — other
// models may still have quota available. Use lockModel() instead.
// For providers with per-model quotas (passthrough providers, Gemini),
// each model has independent quota. A 429 on one model must NOT lock out
// the entire connection — other models may still have quota available.
// Use lockModel() instead.
const isPassthrough = provider && getPassthroughProviders().has(provider);
if (isPassthrough) {
if (isPassthrough || provider === "gemini") {
const { lockModel } = await import("../services/accountFallback.ts");
const cooldown = retryAfterMs || 120_000; // 2 min default, same as COOLDOWN_MS.rateLimit
lockModel(provider, connectionId, model, "rate_limited", cooldown);
@@ -1317,13 +1318,23 @@ export async function handleChatCore({
);
}
} else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
await updateProviderConnection(connectionId, {
testStatus: "credits_exhausted",
lastErrorType: errorType,
lastError: message,
errorCode: statusCode,
});
console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`);
// Gemini has per-model quotas — lock the model only, not the connection
if (provider === "gemini" && model) {
const { lockModel } = await import("../services/accountFallback.ts");
const cooldown = retryAfterMs || 120_000;
lockModel(provider, connectionId, model, "quota_exhausted", cooldown);
console.warn(
`[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil(cooldown / 1000)}s (connection stays active)`
);
} else {
await updateProviderConnection(connectionId, {
testStatus: "credits_exhausted",
lastErrorType: errorType,
lastError: message,
errorCode: statusCode,
});
console.warn(`[provider] Node ${connectionId} exhausted quota (${statusCode})`);
}
} else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) {
await updateProviderConnection(connectionId, {
isActive: false,

View File

@@ -200,6 +200,11 @@ function getLimiterKey(provider, connectionId, model = null) {
if (provider === "codex" && model) {
return `${provider}:${getCodexRateLimitKey(connectionId, model)}`;
}
// Gemini AI Studio has per-model quotas — use model-scoped limiter keys
// so a 429 on one model doesn't pause requests for other models.
if (provider === "gemini" && model) {
return `${provider}:${connectionId}:${model}`;
}
return `${provider}:${connectionId}`;
}
@@ -570,7 +575,7 @@ export function updateFromResponseBody(provider, connectionId, responseBody, sta
const { retryAfterMs, reason } = parseRetryAfterFromBody(responseBody);
if (retryAfterMs && retryAfterMs > 0) {
const limiter = getLimiter(provider, connectionId, null);
const limiter = getLimiter(provider, connectionId, model);
console.log(
`🚫 [RATE-LIMIT] ${provider}:${connectionId.slice(0, 8)} — body-parsed retry: ${Math.ceil(retryAfterMs / 1000)}s (${reason})`
);

View File

@@ -724,6 +724,23 @@ export async function markAccountUnavailable(
try {
await currentMutex;
// ── Gemini per-model lockout (must be before terminal status check) ──
// Gemini AI Studio has per-model quotas. A 429 on gemini-2.5-pro must NOT
// lock out gemini-2.5-flash on the same API key. Lock the specific model
// only and return early, before any connection-wide state is modified.
if (provider === "gemini" && model && (status === 429 || status === 404)) {
const reason = status === 404 ? "not_found" : "rate_limited";
const cooldown = status === 404
? COOLDOWN_MS.notFoundLocal
: (COOLDOWN_MS.rateLimit || 60_000);
lockModel(provider, connectionId, model, reason, cooldown);
log.info(
"AUTH",
`Gemini model-only lockout for ${model}${status} ${reason} ${Math.ceil(cooldown / 1000)}s (connection stays active)`
);
return { shouldFallback: true, cooldownMs: cooldown };
}
// Read current connection to get backoffLevel
const connectionsRaw = await getProviderConnections({ provider });
const connections = (Array.isArray(connectionsRaw) ? connectionsRaw : [])