fix(resilience): route 429 cooldowns through settings (#1548)

* fix(resilience): apply configured cooldowns to 429 handling * fix(resilience): preserve oauth quota handling for 429
2026-07-26 09:52:11 +03:00 · 2026-04-24 08:04:07 -04:00
parent 78531fe033
commit a972293151
7 changed files with 236 additions and 71 deletions
--- a/open-sse/handlers/chatCore.ts
+++ b/open-sse/handlers/chatCore.ts
@@ -73,7 +73,6 @@ import { getCachedSettings } from "@/lib/db/readCache";

 import {
  parseCodexQuotaHeaders,
-  getCodexResetTime,
  getCodexModelScope,
  getCodexDualWindowCooldownMs,
 } from "../executors/codex.ts";
@@ -825,7 +824,7 @@ export async function handleChatCore({
      };

      // T03/T09: on 429, persist exact reset time per scope to avoid global over-blocking.
-      // Item 3: Use dual-window cooldown to distinguish 5h vs 7d exhaustion.
+      // Use dual-window cooldown to distinguish short-term and weekly Codex exhaustion.
      if (status === 429) {
        const { cooldownMs, window: exhaustedWindow } = getCodexDualWindowCooldownMs(quota);
        if (cooldownMs > 0) {
@@ -2280,7 +2279,7 @@ export async function handleChatCore({
    }

    // T06/T10/T36: classify provider errors and persist terminal account states.
-    const errorType = classifyProviderError(statusCode, message);
+    const errorType = classifyProviderError(statusCode, message, provider);
    if (connectionId && errorType) {
      try {
        if (errorType === PROVIDER_ERROR_TYPES.FORBIDDEN) {
@@ -2305,47 +2304,6 @@ export async function handleChatCore({
          console.warn(
            `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently`
          );
-        } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) {
-          // For providers with per-model quotas (passthrough providers, Gemini),
-          // each model has independent quota. A 429 on one model must NOT lock out
-          // the entire connection — other models may still have quota available.
-          const rateLimitCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit;
-          const accountSemaphoreKey = resolveAccountSemaphoreKey({
-            provider,
-            model: currentModel,
-            connectionId,
-            credentials,
-          });
-          if (accountSemaphoreKey) {
-            markAccountSemaphoreBlocked(accountSemaphoreKey, rateLimitCooldownMs);
-          }
-          if (
-            lockModelIfPerModelQuota(
-              provider,
-              connectionId,
-              model,
-              "rate_limited",
-              rateLimitCooldownMs
-            )
-          ) {
-            console.warn(
-              `[provider] Node ${connectionId} model-only rate limited (${statusCode}) for ${model} - ${Math.ceil(rateLimitCooldownMs / 1000)}s (connection stays active)`
-            );
-          } else {
-            const rateLimitedUntil = new Date(Date.now() + rateLimitCooldownMs).toISOString();
-            await updateProviderConnection(connectionId, {
-              rateLimitedUntil: rateLimitedUntil,
-              testStatus: "unavailable",
-              lastErrorType: errorType,
-              lastError: message,
-              errorCode: statusCode,
-              healthCheckInterval: null,
-              lastHealthCheckAt: null,
-            });
-            console.warn(
-              `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}`
-            );
-          }
        } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
          // Providers with per-model quotas — lock the model only, not the connection
          const quotaCooldownMs = retryAfterMs || COOLDOWN_MS.rateLimit;
--- a/open-sse/services/accountFallback.ts
+++ b/open-sse/services/accountFallback.ts
@@ -10,6 +10,7 @@ import {
  calculateBackoffCooldown,
  findMatchingErrorRule,
  matchErrorRuleByText,
+  matchErrorRuleByStatus,
 } from "../config/errorConfig.ts";
 import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts";
 import {
@@ -198,6 +199,11 @@ export function getProviderProfile(provider) {
  return buildProviderProfile(category);
 }

+function shouldPreserveQuotaSignalsFor429(provider: string | null | undefined): boolean {
+  if (!provider) return true;
+  return getProviderCategory(provider) === "oauth";
+}
+
 export async function getRuntimeProviderProfile(provider: string | null | undefined) {
  try {
    const { getCachedSettings } = await import("@/lib/db/readCache");
@@ -929,6 +935,10 @@ export function checkFallbackError(
    };
  }

+  const isRateLimitStatus = status === HTTP_STATUS.RATE_LIMITED;
+  const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);
+  const shouldUseQuotaSignal = !isRateLimitStatus || preserveQuota429;
+
  // Check error message FIRST - specific patterns take priority over status codes
  if (errorText) {
    // T06 (sub2api #1037): Permanent account deactivation — do NOT retry, mark as permanent failure
@@ -942,7 +952,7 @@ export function checkFallbackError(
    }

    // T10 (sub2api #1169): Credits/quota exhausted — long cooldown, distinct from rate limit
-    if (isCreditsExhausted(errorStr)) {
+    if (shouldUseQuotaSignal && isCreditsExhausted(errorStr)) {
      return {
        shouldFallback: true,
        cooldownMs: COOLDOWN_MS.paymentRequired ?? 3600 * 1000, // 1h cooldown
@@ -952,7 +962,7 @@ export function checkFallbackError(
    }

    // Daily quota exhausted — lock model until tomorrow
-    if (isDailyQuotaExhausted(errorStr)) {
+    if (shouldUseQuotaSignal && isDailyQuotaExhausted(errorStr)) {
      const msUntilTomorrow = getMsUntilTomorrow();
      // Cap at 24 hours to handle timezone edge cases
      const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);
@@ -965,7 +975,10 @@ export function checkFallbackError(
    }
  }

-  const configuredRule = findMatchingErrorRule(status, errorStr);
+  const configuredRule =
+    isRateLimitStatus && !preserveQuota429
+      ? matchErrorRuleByStatus(status)
+      : findMatchingErrorRule(status, errorStr);
  if (configuredRule) {
    if (configuredRule.backoff) {
      return buildRetryableFallback(configuredRule.reason ?? classifyError(status, errorStr));
--- a/open-sse/services/errorClassifier.ts
+++ b/open-sse/services/errorClassifier.ts
@@ -4,6 +4,7 @@ import {
  isDailyQuotaExhausted,
  isOAuthInvalidToken,
 } from "./accountFallback.ts";
+import { getProviderCategory } from "../config/providerRegistry.ts";

 export function isEmptyContentResponse(responseBody: unknown): boolean {
  if (!responseBody || typeof responseBody !== "object") return false;
@@ -91,19 +92,35 @@ function responseBodyToString(responseBody: unknown): string {
  return "";
 }

-export function classifyProviderError(statusCode: number, responseBody: unknown): string | null {
+function shouldPreserveQuotaSignalsFor429(provider?: string | null): boolean {
+  if (!provider) return true;
+  return getProviderCategory(provider) === "oauth";
+}
+
+export function classifyProviderError(
+  statusCode: number,
+  responseBody: unknown,
+  provider?: string | null
+): string | null {
  const bodyStr = responseBodyToString(responseBody);
  const creditsExhausted = isCreditsExhausted(bodyStr);
  const accountDeactivated = isAccountDeactivated(bodyStr);
  const oauthInvalid = isOAuthInvalidToken(bodyStr);
+  const preserveQuota429 = shouldPreserveQuotaSignalsFor429(provider);

-  if (creditsExhausted && [400, 402, 403, 429].includes(statusCode)) {
+  if (creditsExhausted && [400, 402, 403].includes(statusCode)) {
    return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
  }

-  // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit
+  if (creditsExhausted && statusCode === 429 && preserveQuota429) {
+    return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
+  }
+
+  // API-key providers route 429 cooldowns through the resilience-aware fallback layer.
+  // OAuth providers keep their existing quota semantics because some of them encode
+  // longer quota windows as 429 responses.
  if (statusCode === 429) {
-    if (isDailyQuotaExhausted(bodyStr)) {
+    if (preserveQuota429 && isDailyQuotaExhausted(bodyStr)) {
      return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
    }
    return PROVIDER_ERROR_TYPES.RATE_LIMITED;
--- a/src/sse/services/auth.ts
+++ b/src/sse/services/auth.ts
@@ -1267,7 +1267,7 @@ export async function markAccountUnavailable(
    const result = fallbackResult;
    const { shouldFallback, cooldownMs: rawCooldownMs, newBackoffLevel, reason } = result;
    if (!shouldFallback) return { shouldFallback: false, cooldownMs: 0 };
-    const providerErrorType = classifyProviderError(status, errorText);
+    const providerErrorType = classifyProviderError(status, errorText, provider);
    const terminalStatus = resolveTerminalConnectionStatus(status, result, providerErrorType);
    const cooldownMs = terminalStatus ? 0 : rawCooldownMs;

--- a/tests/unit/account-fallback-service.test.ts
+++ b/tests/unit/account-fallback-service.test.ts
@@ -65,14 +65,58 @@ test("checkFallbackError marks deactivated accounts as permanent auth failures",
  assert.ok(result.cooldownMs >= 300 * 24 * 60 * 60 * 1000);
 });

-test("checkFallbackError treats exhausted credits as long quota cooldowns", () => {
-  const result = checkFallbackError(429, "credit_balance_too_low");
+test("checkFallbackError treats non-429 exhausted credits as long quota cooldowns", () => {
+  const result = checkFallbackError(402, "credit_balance_too_low");
  assert.equal(result.shouldFallback, true);
  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.creditsExhausted, true);
  assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000);
 });

+test("checkFallbackError keeps API-key 429 exhausted-credit text on the resilience cooldown path", () => {
+  const result = checkFallbackError(429, "credit_balance_too_low", 0, null, "openai", null, {
+    baseCooldownMs: 125,
+    useUpstreamRetryHints: false,
+    maxBackoffSteps: 3,
+    failureThreshold: 60,
+    resetTimeoutMs: 5000,
+  });
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
+  assert.equal(result.creditsExhausted, undefined);
+  assert.equal(result.cooldownMs, 125);
+});
+
+test("checkFallbackError preserves OAuth 429 exhausted-credit semantics", () => {
+  const result = checkFallbackError(429, "credit_balance_too_low", 0, null, "codex", null, {
+    baseCooldownMs: 125,
+    useUpstreamRetryHints: false,
+    maxBackoffSteps: 3,
+    failureThreshold: 60,
+    resetTimeoutMs: 5000,
+  });
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
+  assert.equal(result.creditsExhausted, true);
+  assert.equal(result.cooldownMs, COOLDOWN_MS.paymentRequired ?? 3600 * 1000);
+});
+
+test("checkFallbackError keeps API-key 429 quota text on the status-based resilience path", () => {
+  const result = checkFallbackError(429, "quota exceeded", 0, null, "openai", null, {
+    baseCooldownMs: 125,
+    useUpstreamRetryHints: false,
+    maxBackoffSteps: 3,
+    failureThreshold: 60,
+    resetTimeoutMs: 5000,
+  });
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, RateLimitReason.RATE_LIMIT_EXCEEDED);
+  assert.equal(result.cooldownMs, 125);
+});
+
 test("checkFallbackError honors Retry-After header for rate limits", () => {
  withMockedNow(1_700_000_000_000, () => {
    const headers = new Headers({ "retry-after": "120" });
@@ -468,9 +512,9 @@ test("getMsUntilTomorrow returns positive value less than 24 hours", () => {
  assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours");
 });

-test("checkFallbackError locks model until tomorrow for daily quota exhaustion", () => {
+test("checkFallbackError locks model until tomorrow for non-429 daily quota exhaustion", () => {
  const result = checkFallbackError(
-    429,
+    402,
    "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow"
  );
  assert.equal(result.shouldFallback, true);
@@ -480,14 +524,59 @@ test("checkFallbackError locks model until tomorrow for daily quota exhaustion",
  assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours");
 });

-test("checkFallbackError detects daily quota with 'try again tomorrow'", () => {
-  const result = checkFallbackError(429, "Please try again tomorrow");
+test("checkFallbackError routes API-key 429 'try again tomorrow' through resilience cooldown", () => {
+  const result = checkFallbackError(429, "Please try again tomorrow", 0, null, "openai", null, {
+    baseCooldownMs: 125,
+    useUpstreamRetryHints: false,
+    maxBackoffSteps: 3,
+    failureThreshold: 60,
+    resetTimeoutMs: 5000,
+  });
  assert.equal(result.shouldFallback, true);
-  assert.equal(result.dailyQuotaExhausted, true);
+  assert.equal(result.dailyQuotaExhausted, undefined);
+  assert.equal(result.cooldownMs, 125);
 });

-test("checkFallbackError detects daily quota with 'daily quota'", () => {
-  const result = checkFallbackError(429, "You have exceeded your daily quota");
+test("checkFallbackError routes API-key 429 'daily quota' text through resilience cooldown", () => {
+  const result = checkFallbackError(
+    429,
+    "You have exceeded your daily quota",
+    0,
+    null,
+    "openai",
+    null,
+    {
+      baseCooldownMs: 125,
+      useUpstreamRetryHints: false,
+      maxBackoffSteps: 3,
+      failureThreshold: 60,
+      resetTimeoutMs: 5000,
+    }
+  );
  assert.equal(result.shouldFallback, true);
+  assert.equal(result.dailyQuotaExhausted, undefined);
+  assert.equal(result.cooldownMs, 125);
+});
+
+test("checkFallbackError preserves OAuth 429 daily quota semantics", () => {
+  const result = checkFallbackError(
+    429,
+    "You have exceeded your daily quota",
+    0,
+    null,
+    "codex",
+    null,
+    {
+      baseCooldownMs: 125,
+      useUpstreamRetryHints: false,
+      maxBackoffSteps: 3,
+      failureThreshold: 60,
+      resetTimeoutMs: 5000,
+    }
+  );
+
+  assert.equal(result.shouldFallback, true);
+  assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
  assert.equal(result.dailyQuotaExhausted, true);
+  assert.ok(result.cooldownMs > 0);
 });
--- a/tests/unit/chatcore-translation-paths.test.ts
+++ b/tests/unit/chatcore-translation-paths.test.ts
@@ -10,6 +10,7 @@ process.env.DATA_DIR = TEST_DATA_DIR;
 const core = await import("../../src/lib/db/core.ts");
 const providersDb = await import("../../src/lib/db/providers.ts");
 const settingsDb = await import("../../src/lib/db/settings.ts");
+const auth = await import("../../src/sse/services/auth.ts");
 const upstreamProxyDb = await import("../../src/lib/db/upstreamProxy.ts");
 const { invalidateCacheControlSettingsCache } =
  await import("../../src/lib/cacheControlSettings.ts");
@@ -1496,7 +1497,7 @@ test("chatCore does not inject fallback user for Qwen API key requests", async (
  assert.equal("user" in call.body, false);
 });

-test("chatCore persists Codex quota headers and scope cooldown on 429 responses", async () => {
+test("chatCore preserves Codex dual-window scope cooldowns on 429 responses", async () => {
  const connection = await providersDb.createProviderConnection({
    provider: "codex",
    authType: "oauth",
@@ -1547,7 +1548,68 @@ test("chatCore persists Codex quota headers and scope cooldown on 429 responses"
    typeof (updated as any).providerSpecificData.codexScopeRateLimitedUntil.codex,
    "string"
  );
-  (assert as any).equal((updated.providerSpecificData as any).codexExhaustedWindow, "5h");
+  assert.equal((updated as any).providerSpecificData.codexExhaustedWindow, "5h");
+});
+
+test("chatCore 429 lets account fallback apply the configured resilience cooldown", async () => {
+  await settingsDb.updateSettings({
+    resilienceSettings: {
+      connectionCooldown: {
+        apikey: {
+          baseCooldownMs: 1000,
+          useUpstreamRetryHints: false,
+          maxBackoffSteps: 3,
+        },
+      },
+    },
+  });
+
+  const connection = await providersDb.createProviderConnection({
+    provider: "openai",
+    authType: "apikey",
+    name: "resilience-429",
+    apiKey: "sk-resilience-429",
+    isActive: true,
+    providerSpecificData: {},
+  });
+
+  const { result } = await invokeChatCore({
+    provider: "openai",
+    model: "gpt-4o-mini",
+    connectionId: connection.id,
+    body: {
+      model: "gpt-4o-mini",
+      stream: false,
+      messages: [{ role: "user", content: "rate limit me" }],
+    },
+    responseFactory() {
+      return new Response(JSON.stringify({ error: { message: "too many requests" } }), {
+        status: 429,
+        headers: { "Content-Type": "application/json" },
+      });
+    },
+  });
+
+  const afterCore = await providersDb.getProviderConnectionById((connection as any).id);
+  assert.equal(result.success, false);
+  assert.equal(result.status, 429);
+  assert.equal((afterCore as any).rateLimitedUntil, undefined);
+
+  const fallback = await auth.markAccountUnavailable(
+    (connection as any).id,
+    result.status,
+    result.error,
+    "openai",
+    "gpt-4o-mini"
+  );
+  const afterFallback = await providersDb.getProviderConnectionById((connection as any).id);
+  const cooldownRemaining =
+    new Date((afterFallback as any).rateLimitedUntil).getTime() - Date.now();
+
+  assert.equal(fallback.shouldFallback, true);
+  assert.equal(fallback.cooldownMs, 1000);
+  assert.equal((afterFallback as any).testStatus, "unavailable");
+  assert.ok(cooldownRemaining > 0 && cooldownRemaining <= 2_000);
 });

 test("chatCore falls back to the next family model when the requested model is unavailable", async () => {
--- a/tests/unit/error-classifier.test.ts
+++ b/tests/unit/error-classifier.test.ts
@@ -34,13 +34,35 @@ test("classifyProviderError: 429 without billing signal => RATE_LIMITED", () =>
  assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED);
 });

-test("classifyProviderError: 429 with billing signal => QUOTA_EXHAUSTED", () => {
+test("classifyProviderError: 429 with billing signal and no provider keeps legacy QUOTA_EXHAUSTED", () => {
  const result = classifyProviderError(429, {
    error: { message: "insufficient_quota: exceeded your current quota" },
  });
  assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
 });

+test("classifyProviderError: API-key provider 429 with billing signal => RATE_LIMITED", () => {
+  const result = classifyProviderError(
+    429,
+    {
+      error: { message: "insufficient_quota: exceeded your current quota" },
+    },
+    "openai"
+  );
+  assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED);
+});
+
+test("classifyProviderError: OAuth provider 429 with billing signal => QUOTA_EXHAUSTED", () => {
+  const result = classifyProviderError(
+    429,
+    {
+      error: { message: "insufficient_quota: exceeded your current quota" },
+    },
+    "codex"
+  );
+  assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
+});
+
 test("classifyProviderError: 403 with 'has not been used in project' => PROJECT_ROUTE_ERROR (transient)", () => {
  const result = classifyProviderError(403, {
    error: {
@@ -66,20 +88,24 @@ test("classifyProviderError: 403 with project string as plain string body => PRO
  assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);
 });

-test("classifyProviderError: 429 with daily quota signal => QUOTA_EXHAUSTED", () => {
+test("classifyProviderError: API-key provider 429 with daily quota signal => RATE_LIMITED", () => {
  const body = JSON.stringify({
    error: {
      message:
        "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow",
    },
  });
-  const result = classifyProviderError(429, body);
-  assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
+  const result = classifyProviderError(429, body, "openai");
+  assert.equal(result, PROVIDER_ERROR_TYPES.RATE_LIMITED);
 });

-test("classifyProviderError: 429 with 'daily quota' signal => QUOTA_EXHAUSTED", () => {
-  const result = classifyProviderError(429, {
-    error: { message: "You have reached your daily quota limit" },
-  });
+test("classifyProviderError: OAuth provider 429 with daily quota signal => QUOTA_EXHAUSTED", () => {
+  const result = classifyProviderError(
+    429,
+    {
+      error: { message: "You have reached your daily quota limit" },
+    },
+    "codex"
+  );
  assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
 });