diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 6994285495..3a521536b0 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -73,7 +73,6 @@ import { getCachedSettings } from "@/lib/db/readCache"; import { parseCodexQuotaHeaders, - getCodexResetTime, getCodexModelScope, getCodexDualWindowCooldownMs, } from "../executors/codex.ts"; @@ -825,7 +824,7 @@ export async function handleChatCore({ }; // T03/T09: on 429, persist exact reset time per scope to avoid global over-blocking. - // Item 3: Use dual-window cooldown to distinguish 5h vs 7d exhaustion. + // Use dual-window cooldown to distinguish short-term and weekly Codex exhaustion. if (status === 429) { const { cooldownMs, window: exhaustedWindow } = getCodexDualWindowCooldownMs(quota); if (cooldownMs > 0) { @@ -2280,7 +2279,7 @@ export async function handleChatCore({ } // T06/T10/T36: classify provider errors and persist terminal account states. - const errorType = classifyProviderError(statusCode, message); + const errorType = classifyProviderError(statusCode, message, provider); if (connectionId && errorType) { try { if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) { @@ -2305,47 +2304,6 @@ export async function handleChatCore({ console.warn( `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` ); - } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) { - // For providers with per-model quotas (passthrough providers, Gemini), - // each model has independent quota. A 429 on one model must NOT lock out - // the entire connection — other models may still have quota available. - const rateLimitCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit; - const accountSemaphoreKey = resolveAccountSemaphoreKey({ - provider, - model: currentModel, - connectionId, - credentials, - }); - if (accountSemaphoreKey) { - markAccountSemaphoreBlocked(accountSemaphoreKey, rateLimitCooldownMs); - } - if ( - lockModelIfPerModelQuota( - provider, - connectionId, - model, - "rate_limited", - rateLimitCooldownMs - ) - ) { - console.warn( - `[provider] Node ${connectionId} model-only rate limited (${statusCode}) for ${model} - ${Math.ceil(rateLimitCooldownMs / 1000)}s (connection stays active)` - ); - } else { - const rateLimitedUntil = new Date(Date.now() + rateLimitCooldownMs).toISOString(); - await updateProviderConnection(connectionId, { - rateLimitedUntil: rateLimitedUntil, - testStatus: "unavailable", - lastErrorType: errorType, - lastError: message, - errorCode: statusCode, - healthCheckInterval: null, - lastHealthCheckAt: null, - }); - console.warn( - `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}` - ); - } } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { // Providers with per-model quotas — lock the model only, not the connection const quotaCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit; diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 252395e65d..d8431067af 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -10,6 +10,7 @@ import { calculateBackoffCooldown, findMatchingErrorRule, matchErrorRuleByText, + matchErrorRuleByStatus, } from "../config/errorConfig.ts"; import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts"; import { @@ -198,6 +199,11 @@ export function getProviderProfile(provider) { return buildProviderProfile(category); } +function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean { + if (!provider) return true; + return getProviderCategory(provider) === "oauth"; +} + export async function getRuntimeProviderProfile(provider: string | null | undefined) { try { const { getCachedSettings } = await import("@/lib/db/readCache"); @@ -929,6 +935,10 @@ export function checkFallbackError( }; } + const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED; + const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider); + const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429; + // Check error message FIRST - specific patterns take priority over status codes if (errorText) { // T06 (sub2api #1037): Permanent account deactivation — do NOT retry, mark as permanent failure @@ -942,7 +952,7 @@ export function checkFallbackError( } // T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit - if (isCreditsExhausted(errorStr)) { + if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) { return { shouldFallback: true, cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown @@ -952,7 +962,7 @@ export function checkFallbackError( } // Daily quota exhausted — lock model until tomorrow - if (isDailyQuotaExhausted(errorStr)) { + if (shouldUseQuotaSignal && isDailyQuotaExhausted(errorStr)) { const msUntilTomorrow = getMsUntilTomorrow(); // Cap at 24 hours to handle timezone edge cases const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000); @@ -965,7 +975,10 @@ export function checkFallbackError( } } - const configuredRule = findMatchingErrorRule(status, errorStr); + const configuredRule = + isRateLimitStatus && !preserveQuota429 + ? matchErrorRuleByStatus(status) + : findMatchingErrorRule(status, errorStr); if (configuredRule) { if (configuredRule.backoff) { return buildRetryableFallback(configuredRule.reason ?? classifyError(status, errorStr)); diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index 4520c2a6e4..da9014b4a6 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -4,6 +4,7 @@ import { isDailyQuotaExhausted, isOAuthInvalidToken, } from "./accountFallback.ts"; +import { getProviderCategory } from "../config/providerRegistry.ts"; export function isEmptyContentResponse(responseBody: unknown): boolean { if (!responseBody || typeof responseBody !== "object") return false; @@ -91,19 +92,35 @@ function responseBodyToString(responseBody: unknown): string { return ""; } -export function classifyProviderError(statusCode: number, responseBody: unknown): string | null { +function shouldPreserveQuotaSignalsFor429(provider?: string | null): boolean { + if (!provider) return true; + return getProviderCategory(provider) === "oauth"; +} + +export function classifyProviderError( + statusCode: number, + responseBody: unknown, + provider?: string | null +): string | null { const bodyStr = responseBodyToString(responseBody); const creditsExhausted = isCreditsExhausted(bodyStr); const accountDeactivated = isAccountDeactivated(bodyStr); const oauthInvalid = isOAuthInvalidToken(bodyStr); + const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider); - if (creditsExhausted && [400, 402, 403, 429].includes(statusCode)) { + if (creditsExhausted && [400, 402, 403].includes(statusCode)) { return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; } - // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit + if (creditsExhausted && statusCode === 429 && preserveQuota429) { + return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + } + + // API-key providers route 429 cooldowns through the resilience-aware fallback layer. + // OAuth providers keep their existing quota semantics because some of them encode + // longer quota windows as 429 responses. if (statusCode === 429) { - if (isDailyQuotaExhausted(bodyStr)) { + if (preserveQuota429 && isDailyQuotaExhausted(bodyStr)) { return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; } return PROVIDER_ERROR_TYPES.RATE_LIMITED; diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 59e64366ad..47eca015d8 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -1267,7 +1267,7 @@ export async function markAccountUnavailable( const result = fallbackResult; const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result; if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 }; - const providerErrorType = classifyProviderError(status, errorText); + const providerErrorType = classifyProviderError(status, errorText, provider); const terminalStatus = resolveTerminalConnectionStatus(status, result, providerErrorType); const cooldownMs = terminalStatus ? 0 : rawCooldownMs; diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.ts index 2095057770..63b473e926 100644 --- a/tests/unit/account-fallback-service.test.ts +++ b/tests/unit/account-fallback-service.test.ts @@ -65,14 +65,58 @@ test("checkFallbackError marks deactivated accounts as permanent auth failures", assert.ok(result.cooldownMs >= 300 * 24 * 60 * 60 * 1000); }); -test("checkFallbackError treats exhausted credits as long quota cooldowns", () => { - const result = checkFallbackError(429, "credit_balance_too_low"); +test("checkFallbackError treats non-429 exhausted credits as long quota cooldowns", () => { + const result = checkFallbackError(402, "credit_balance_too_low"); assert.equal(result.shouldFallback, true); assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.creditsExhausted, true); assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); }); +test("checkFallbackError keeps API-key 429 exhausted-credit text on the resilience cooldown path", () => { + const result = checkFallbackError(429, "credit_balance_too_low", 0, null, "openai", null, { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + }); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); + assert.equal(result.creditsExhausted, undefined); + assert.equal(result.cooldownMs, 125); +}); + +test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => { + const result = checkFallbackError(429, "credit_balance_too_low", 0, null, "codex", null, { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + }); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); + assert.equal(result.creditsExhausted, true); + assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000); +}); + +test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => { + const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + }); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED); + assert.equal(result.cooldownMs, 125); +}); + test("checkFallbackError honors Retry-After header for rate limits", () => { withMockedNow(1_700_000_000_000, () => { const headers = new Headers({ "retry-after": "120" }); @@ -468,9 +512,9 @@ test("getMsUntilTomorrow returns positive value less than 24 hours", () => { assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours"); }); -test("checkFallbackError locks model until tomorrow for daily quota exhaustion", () => { +test("checkFallbackError locks model until tomorrow for non-429 daily quota exhaustion", () => { const result = checkFallbackError( - 429, + 402, "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow" ); assert.equal(result.shouldFallback, true); @@ -480,14 +524,59 @@ test("checkFallbackError locks model until tomorrow for daily quota exhaustion", assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours"); }); -test("checkFallbackError detects daily quota with 'try again tomorrow'", () => { - const result = checkFallbackError(429, "Please try again tomorrow"); +test("checkFallbackError routes API-key 429 'try again tomorrow' through resilience cooldown", () => { + const result = checkFallbackError(429, "Please try again tomorrow", 0, null, "openai", null, { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + }); assert.equal(result.shouldFallback, true); - assert.equal(result.dailyQuotaExhausted, true); + assert.equal(result.dailyQuotaExhausted, undefined); + assert.equal(result.cooldownMs, 125); }); -test("checkFallbackError detects daily quota with 'daily quota'", () => { - const result = checkFallbackError(429, "You have exceeded your daily quota"); +test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => { + const result = checkFallbackError( + 429, + "You have exceeded your daily quota", + 0, + null, + "openai", + null, + { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + } + ); assert.equal(result.shouldFallback, true); + assert.equal(result.dailyQuotaExhausted, undefined); + assert.equal(result.cooldownMs, 125); +}); + +test("checkFallbackError preserves OAuth 429 daily quota semantics", () => { + const result = checkFallbackError( + 429, + "You have exceeded your daily quota", + 0, + null, + "codex", + null, + { + baseCooldownMs: 125, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + failureThreshold: 60, + resetTimeoutMs: 5000, + } + ); + + assert.equal(result.shouldFallback, true); + assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED); assert.equal(result.dailyQuotaExhausted, true); + assert.ok(result.cooldownMs > 0); }); diff --git a/tests/unit/chatcore-translation-paths.test.ts b/tests/unit/chatcore-translation-paths.test.ts index dc7011103a..2ccbb96e7f 100644 --- a/tests/unit/chatcore-translation-paths.test.ts +++ b/tests/unit/chatcore-translation-paths.test.ts @@ -10,6 +10,7 @@ process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const settingsDb = await import("../../src/lib/db/settings.ts"); +const auth = await import("../../src/sse/services/auth.ts"); const upstreamProxyDb = await import("../../src/lib/db/upstreamProxy.ts"); const { invalidateCacheControlSettingsCache } = await import("../../src/lib/cacheControlSettings.ts"); @@ -1496,7 +1497,7 @@ test("chatCore does not inject fallback user for Qwen API key requests", async ( assert.equal("user" in call.body, false); }); -test("chatCore persists Codex quota headers and scope cooldown on 429 responses", async () => { +test("chatCore preserves Codex dual-window scope cooldowns on 429 responses", async () => { const connection = await providersDb.createProviderConnection({ provider: "codex", authType: "oauth", @@ -1547,7 +1548,68 @@ test("chatCore persists Codex quota headers and scope cooldown on 429 responses" typeof (updated as any).providerSpecificData.codexScopeRateLimitedUntil.codex, "string" ); - (assert as any).equal((updated.providerSpecificData as any).codexExhaustedWindow, "5h"); + assert.equal((updated as any).providerSpecificData.codexExhaustedWindow, "5h"); +}); + +test("chatCore 429 lets account fallback apply the configured resilience cooldown", async () => { + await settingsDb.updateSettings({ + resilienceSettings: { + connectionCooldown: { + apikey: { + baseCooldownMs: 1000, + useUpstreamRetryHints: false, + maxBackoffSteps: 3, + }, + }, + }, + }); + + const connection = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "resilience-429", + apiKey: "sk-resilience-429", + isActive: true, + providerSpecificData: {}, + }); + + const { result } = await invokeChatCore({ + provider: "openai", + model: "gpt-4o-mini", + connectionId: connection.id, + body: { + model: "gpt-4o-mini", + stream: false, + messages: [{ role: "user", content: "rate limit me" }], + }, + responseFactory() { + return new Response(JSON.stringify({ error: { message: "too many requests" } }), { + status: 429, + headers: { "Content-Type": "application/json" }, + }); + }, + }); + + const afterCore = await providersDb.getProviderConnectionById((connection as any).id); + assert.equal(result.success, false); + assert.equal(result.status, 429); + assert.equal((afterCore as any).rateLimitedUntil, undefined); + + const fallback = await auth.markAccountUnavailable( + (connection as any).id, + result.status, + result.error, + "openai", + "gpt-4o-mini" + ); + const afterFallback = await providersDb.getProviderConnectionById((connection as any).id); + const cooldownRemaining = + new Date((afterFallback as any).rateLimitedUntil).getTime() - Date.now(); + + assert.equal(fallback.shouldFallback, true); + assert.equal(fallback.cooldownMs, 1000); + assert.equal((afterFallback as any).testStatus, "unavailable"); + assert.ok(cooldownRemaining > 0 && cooldownRemaining <= 2_000); }); test("chatCore falls back to the next family model when the requested model is unavailable", async () => { diff --git a/tests/unit/error-classifier.test.ts b/tests/unit/error-classifier.test.ts index 846f89fb87..2878902d75 100644 --- a/tests/unit/error-classifier.test.ts +++ b/tests/unit/error-classifier.test.ts @@ -34,13 +34,35 @@ test("classifyProviderError: 429 without billing signal => RATE_LIMITED", () => assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED); }); -test("classifyProviderError: 429 with billing signal => QUOTA_EXHAUSTED", () => { +test("classifyProviderError: 429 with billing signal and no provider keeps legacy QUOTA_EXHAUSTED", () => { const result = classifyProviderError(429, { error: { message: "insufficient_quota: exceeded your current quota" }, }); assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED); }); +test("classifyProviderError: API-key provider 429 with billing signal => RATE_LIMITED", () => { + const result = classifyProviderError( + 429, + { + error: { message: "insufficient_quota: exceeded your current quota" }, + }, + "openai" + ); + assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED); +}); + +test("classifyProviderError: OAuth provider 429 with billing signal => QUOTA_EXHAUSTED", () => { + const result = classifyProviderError( + 429, + { + error: { message: "insufficient_quota: exceeded your current quota" }, + }, + "codex" + ); + assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED); +}); + test("classifyProviderError: 403 with 'has not been used in project' => PROJECT_ROUTE_ERROR (transient)", () => { const result = classifyProviderError(403, { error: { @@ -66,20 +88,24 @@ test("classifyProviderError: 403 with project string as plain string body => PRO assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR); }); -test("classifyProviderError: 429 with daily quota signal => QUOTA_EXHAUSTED", () => { +test("classifyProviderError: API-key provider 429 with daily quota signal => RATE_LIMITED", () => { const body = JSON.stringify({ error: { message: "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow", }, }); - const result = classifyProviderError(429, body); - assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED); + const result = classifyProviderError(429, body, "openai"); + assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED); }); -test("classifyProviderError: 429 with 'daily quota' signal => QUOTA_EXHAUSTED", () => { - const result = classifyProviderError(429, { - error: { message: "You have reached your daily quota limit" }, - }); +test("classifyProviderError: OAuth provider 429 with daily quota signal => QUOTA_EXHAUSTED", () => { + const result = classifyProviderError( + 429, + { + error: { message: "You have reached your daily quota limit" }, + }, + "codex" + ); assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED); });