From a405f2e81e5e834308c287012efe5ec8cb539f33 Mon Sep 17 00:00:00 2001 From: oyi77 Date: Tue, 31 Mar 2026 17:59:54 +0700 Subject: [PATCH] fix(429): parse long quota reset times from error body - Parse XhYmZs format from antigravity error messages (e.g., 27h41m36s) - Dynamic retry-after threshold (60s default) instead of hardcoded 10s - Add parseRetryFromErrorText() in accountFallback.ts for body parsing - Fix 403 'verify your account' to trigger permanent deactivation - Add keyword matching for 'quota will reset', 'exhausted capacity' - Add unit tests for retry parsing and keyword matching Fixes #858 (Antigravity 429 handling) Fixes #832 (Qwen quota 429 - same underlying bug) --- open-sse/executors/antigravity.ts | 42 ++++++++++++++-- open-sse/handlers/chatCore.ts | 25 ++++++++++ open-sse/services/accountFallback.ts | 45 +++++++++++++++++ open-sse/services/errorClassifier.ts | 3 ++ tests/unit/error-classification.test.mjs | 63 +++++++++++++++++++++++- 5 files changed, 172 insertions(+), 6 deletions(-) diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index 80f1e6105e..a0e402892c 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -2,7 +2,8 @@ import crypto from "crypto"; import { BaseExecutor, mergeUpstreamExtraHeaders } from "./base.ts"; import { PROVIDERS, OAUTH_ENDPOINTS, HTTP_STATUS } from "../config/constants.ts"; -const MAX_RETRY_AFTER_MS = 10000; +const MAX_RETRY_AFTER_MS = 60_000; +const LONG_RETRY_THRESHOLD_MS = 60_000; /** * Strip provider prefixes (e.g. "antigravity/model" → "model"). @@ -224,12 +225,15 @@ export class AntigravityExecutor extends BaseExecutor { signal, }); + // Parse retry time for 429/503 responses + let retryMs = null; + if ( response.status === HTTP_STATUS.RATE_LIMITED || response.status === HTTP_STATUS.SERVICE_UNAVAILABLE ) { // Try to get retry time from headers first - let retryMs = this.parseRetryHeaders(response.headers); + retryMs = this.parseRetryHeaders(response.headers); // If no retry time in headers, try to parse from error message body if (!retryMs) { @@ -243,12 +247,13 @@ export class AntigravityExecutor extends BaseExecutor { } } - if (retryMs && retryMs <= MAX_RETRY_AFTER_MS) { + if (retryMs && retryMs <= LONG_RETRY_THRESHOLD_MS) { + const effectiveRetryMs = Math.min(retryMs, MAX_RETRY_AFTER_MS); log?.debug?.( "RETRY", - `${response.status} with Retry-After: ${Math.ceil(retryMs / 1000)}s, waiting...` + `${response.status} with Retry-After: ${Math.ceil(effectiveRetryMs / 1000)}s, waiting...` ); - await new Promise((resolve) => setTimeout(resolve, retryMs)); + await new Promise((resolve) => setTimeout(resolve, effectiveRetryMs)); urlIndex--; continue; } @@ -291,6 +296,33 @@ export class AntigravityExecutor extends BaseExecutor { continue; } + // If we have a 429 with long retry time, embed it in response body + if ( + response.status === HTTP_STATUS.RATE_LIMITED && + retryMs && + retryMs > LONG_RETRY_THRESHOLD_MS + ) { + try { + const respBody = await response.clone().text(); + let obj; + try { + obj = JSON.parse(respBody); + } catch { + obj = {}; + } + obj.retryAfterMs = retryMs; + const modifiedBody = JSON.stringify(obj); + const modifiedResponse = new Response(modifiedBody, { + status: response.status, + headers: response.headers, + }); + return { response: modifiedResponse, url, headers, transformedBody }; + } catch (err) { + log?.warn?.("RETRY", `Failed to embed retryAfterMs: ${err}`); + // Fall back to original response + } + } + return { response, url, headers, transformedBody }; } catch (error) { lastError = error; diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 5c3312ab87..69c5f7d476 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1168,6 +1168,31 @@ export async function handleChatCore({ console.warn( `[provider] Node ${connectionId} banned (${statusCode}) — disabling permanently` ); + } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) { + await updateProviderConnection(connectionId, { + isActive: false, + testStatus: "deactivated", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + }); + console.warn( + `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently` + ); + } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) { + const rateLimitedUntil = new Date(Date.now() + retryAfterMs).toISOString(); + await updateProviderConnection(connectionId, { + rateLimitedUntil: rateLimitedUntil, + testStatus: "credits_exhausted", + lastErrorType: errorType, + lastError: message, + errorCode: statusCode, + healthCheckInterval: null, + lastHealthCheckAt: null, + }); + console.warn( + `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}` + ); } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) { await updateProviderConnection(connectionId, { testStatus: "credits_exhausted", diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index be2be3e8c1..f19a531309 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -229,6 +229,39 @@ function parseDelayString(value) { return isNaN(num) ? null : num * 1000; } +/** + * T07: Parse retry time from error text body with combined "XhYmZs" format. + * Examples: "Your quota will reset after 2h30m14s", "reset after 45m", "reset after 30s" + * Returns milliseconds or null if not parseable. + * + * @param {string} errorText - Error message text from response body + * @returns {number|null} Retry duration in milliseconds + */ +export function parseRetryFromErrorText(errorText) { + if (!errorText || typeof errorText !== "string") return null; + + const match = errorText.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i); + if (!match) { + // Also try the variant without "reset after": "will reset after XhYmZs" + const altMatch = errorText.match(/will reset after (\d+h)?(\d+m)?(\d+s)?/i); + if (!altMatch) return null; + return computeDurationMs(altMatch); + } + + return computeDurationMs(match); +} + +/** + * Compute total milliseconds from regex match groups (Xh)(Ym)(Zs) + */ +function computeDurationMs(match) { + let totalMs = 0; + if (match[1]) totalMs += parseInt(match[1], 10) * 3600 * 1000; // hours + if (match[2]) totalMs += parseInt(match[2], 10) * 60 * 1000; // minutes + if (match[3]) totalMs += parseInt(match[3], 10) * 1000; // seconds + return totalMs > 0 ? totalMs : null; +} + // ─── Error Classification ─────────────────────────────────────────────────── /** @@ -428,6 +461,9 @@ export function checkFallbackError( lowerError.includes("rate limit") || lowerError.includes("too many requests") || lowerError.includes("quota exceeded") || + lowerError.includes("quota will reset") || + lowerError.includes("exhausted your capacity") || + lowerError.includes("quota exhausted") || lowerError.includes("capacity") || lowerError.includes("overloaded") ) { @@ -443,6 +479,15 @@ export function checkFallbackError( }; } } + const retryFromBody = parseRetryFromErrorText(errorStr); + if (retryFromBody && retryFromBody > 60_000) { + return { + shouldFallback: true, + cooldownMs: retryFromBody, + newBackoffLevel: 0, + reason: RateLimitReason.RATE_LIMIT_EXCEEDED, + }; + } const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel); const reason = classifyErrorText(errorStr); return { diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts index dc61b11390..00e9f368ac 100644 --- a/open-sse/services/errorClassifier.ts +++ b/open-sse/services/errorClassifier.ts @@ -46,6 +46,9 @@ export function classifyProviderError(statusCode: number, responseBody: unknown) } if (statusCode === 402) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED; + if (statusCode === 403 && accountDeactivated) { + return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED; + } if (statusCode === 403) return PROVIDER_ERROR_TYPES.FORBIDDEN; if (statusCode >= 500) return PROVIDER_ERROR_TYPES.SERVER_ERROR; diff --git a/tests/unit/error-classification.test.mjs b/tests/unit/error-classification.test.mjs index 78136f07e6..9fb24d7181 100644 --- a/tests/unit/error-classification.test.mjs +++ b/tests/unit/error-classification.test.mjs @@ -1,7 +1,7 @@ import test from "node:test"; import assert from "node:assert/strict"; -const { checkFallbackError, getProviderProfile } = +const { checkFallbackError, getProviderProfile, parseRetryFromErrorText } = await import("../../open-sse/services/accountFallback.ts"); const { getProviderCategory } = await import("../../open-sse/config/providerRegistry.ts"); @@ -127,3 +127,64 @@ test("400 bad request: still returns shouldFallback false", () => { const result = checkFallbackError(400, "", 0, null, "groq"); assert.equal(result.shouldFallback, false); }); + +// ─── T07: Retry Time Parsing from Error Text ───────────────────────────────── + +test("parseRetryFromErrorText: parses 27h41m36s format", () => { + const result = parseRetryFromErrorText("Your quota will reset after 27h41m36s"); + assert.equal(result, 27 * 3600 * 1000 + 41 * 60 * 1000 + 36 * 1000); +}); + +test("parseRetryFromErrorText: parses 2h30m format", () => { + const result = parseRetryFromErrorText("quota will reset after 2h30m"); + assert.equal(result, 2 * 3600 * 1000 + 30 * 60 * 1000); +}); + +test("parseRetryFromErrorText: parses 45m format", () => { + const result = parseRetryFromErrorText("reset after 45m"); + assert.equal(result, 45 * 60 * 1000); +}); + +test("parseRetryFromErrorText: parses 30s format", () => { + const result = parseRetryFromErrorText("reset after 30s"); + assert.equal(result, 30 * 1000); +}); + +test("parseRetryFromErrorText: returns null for invalid format", () => { + const result = parseRetryFromErrorText("invalid error message"); + assert.equal(result, null); +}); + +test("parseRetryFromErrorText: parses will reset after variant", () => { + const result = parseRetryFromErrorText("quota will reset after 5h"); + assert.equal(result, 5 * 3600 * 1000); +}); + +// ─── T06: Keyword Matching for Long Cooldowns ──────────────────────────────── + +test("quota will reset keyword triggers long cooldown from body", () => { + const result = checkFallbackError( + 429, + "Your quota will reset after 27h41m36s", + 0, + null, + "antigravity", + null + ); + assert.equal(result.shouldFallback, true); + assert.ok(result.cooldownMs > 60_000, "cooldownMs should be > 60s"); + assert.equal(result.newBackoffLevel, 0, "backoffLevel should reset to 0"); +}); + +test("exhausted your capacity keyword triggers long cooldown", () => { + const result = checkFallbackError( + 429, + "You have exhausted your capacity. Your quota will reset after 2h", + 0, + null, + "antigravity", + null + ); + assert.equal(result.shouldFallback, true); + assert.ok(result.cooldownMs > 60_000); +});