fix(429): parse long quota reset times from error body

- Parse XhYmZs format from antigravity error messages (e.g., 27h41m36s)
- Dynamic retry-after threshold (60s default) instead of hardcoded 10s
- Add parseRetryFromErrorText() in accountFallback.ts for body parsing
- Fix 403 'verify your account' to trigger permanent deactivation
- Add keyword matching for 'quota will reset', 'exhausted capacity'
- Add unit tests for retry parsing and keyword matching

Fixes #858 (Antigravity 429 handling)
Fixes #832 (Qwen quota 429 - same underlying bug)

open-sse/executors/antigravity.ts

@@ -2,7 +2,8 @@ import crypto from "crypto";
 import { BaseExecutor, mergeUpstreamExtraHeaders } from "./base.ts";
 import { PROVIDERS, OAUTH_ENDPOINTS, HTTP_STATUS } from "../config/constants.ts";
 
-const MAX_RETRY_AFTER_MS = 10000;
+const MAX_RETRY_AFTER_MS = 60_000;
+const LONG_RETRY_THRESHOLD_MS = 60_000;
 
 /**
  * Strip provider prefixes (e.g. "antigravity/model" → "model").
@@ -224,12 +225,15 @@ export class AntigravityExecutor extends BaseExecutor {
           signal,
         });
 
+        // Parse retry time for 429/503 responses
+        let retryMs = null;
+
         if (
           response.status === HTTP_STATUS.RATE_LIMITED ||
           response.status === HTTP_STATUS.SERVICE_UNAVAILABLE
         ) {
           // Try to get retry time from headers first
-          let retryMs = this.parseRetryHeaders(response.headers);
+          retryMs = this.parseRetryHeaders(response.headers);
 
           // If no retry time in headers, try to parse from error message body
           if (!retryMs) {
@@ -243,12 +247,13 @@ export class AntigravityExecutor extends BaseExecutor {
             }
           }
 
-          if (retryMs && retryMs <= MAX_RETRY_AFTER_MS) {
+          if (retryMs && retryMs <= LONG_RETRY_THRESHOLD_MS) {
+            const effectiveRetryMs = Math.min(retryMs, MAX_RETRY_AFTER_MS);
             log?.debug?.(
               "RETRY",
-              `${response.status} with Retry-After: ${Math.ceil(retryMs / 1000)}s, waiting...`
+              `${response.status} with Retry-After: ${Math.ceil(effectiveRetryMs / 1000)}s, waiting...`
             );
-            await new Promise((resolve) => setTimeout(resolve, retryMs));
+            await new Promise((resolve) => setTimeout(resolve, effectiveRetryMs));
             urlIndex--;
             continue;
           }
@@ -291,6 +296,33 @@ export class AntigravityExecutor extends BaseExecutor {
           continue;
         }
 
+        // If we have a 429 with long retry time, embed it in response body
+        if (
+          response.status === HTTP_STATUS.RATE_LIMITED &&
+          retryMs &&
+          retryMs > LONG_RETRY_THRESHOLD_MS
+        ) {
+          try {
+            const respBody = await response.clone().text();
+            let obj;
+            try {
+              obj = JSON.parse(respBody);
+            } catch {
+              obj = {};
+            }
+            obj.retryAfterMs = retryMs;
+            const modifiedBody = JSON.stringify(obj);
+            const modifiedResponse = new Response(modifiedBody, {
+              status: response.status,
+              headers: response.headers,
+            });
+            return { response: modifiedResponse, url, headers, transformedBody };
+          } catch (err) {
+            log?.warn?.("RETRY", `Failed to embed retryAfterMs: ${err}`);
+            // Fall back to original response
+          }
+        }
+
         return { response, url, headers, transformedBody };
       } catch (error) {
         lastError = error;

open-sse/handlers/chatCore.ts

@@ -1168,6 +1168,31 @@ export async function handleChatCore({
           console.warn(
             `[provider] Node ${connectionId} banned (${statusCode}) — disabling permanently`
           );
+        } else if (errorType === PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED) {
+          await updateProviderConnection(connectionId, {
+            isActive: false,
+            testStatus: "deactivated",
+            lastErrorType: errorType,
+            lastError: message,
+            errorCode: statusCode,
+          });
+          console.warn(
+            `[provider] Node ${connectionId} account deactivated (${statusCode}) — disabling permanently`
+          );
+        } else if (errorType === PROVIDER_ERROR_TYPES.RATE_LIMITED) {
+          const rateLimitedUntil = new Date(Date.now() + retryAfterMs).toISOString();
+          await updateProviderConnection(connectionId, {
+            rateLimitedUntil: rateLimitedUntil,
+            testStatus: "credits_exhausted",
+            lastErrorType: errorType,
+            lastError: message,
+            errorCode: statusCode,
+            healthCheckInterval: null,
+            lastHealthCheckAt: null,
+          });
+          console.warn(
+            `[provider] Node ${connectionId} rate limited (${statusCode}) - Next available at ${rateLimitedUntil}`
+          );
         } else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
           await updateProviderConnection(connectionId, {
             testStatus: "credits_exhausted",

open-sse/services/accountFallback.ts

@@ -229,6 +229,39 @@ function parseDelayString(value) {
   return isNaN(num) ? null : num * 1000;
 }
 
+/**
+ * T07: Parse retry time from error text body with combined "XhYmZs" format.
+ * Examples: "Your quota will reset after 2h30m14s", "reset after 45m", "reset after 30s"
+ * Returns milliseconds or null if not parseable.
+ *
+ * @param {string} errorText - Error message text from response body
+ * @returns {number|null} Retry duration in milliseconds
+ */
+export function parseRetryFromErrorText(errorText) {
+  if (!errorText || typeof errorText !== "string") return null;
+
+  const match = errorText.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i);
+  if (!match) {
+    // Also try the variant without "reset after": "will reset after XhYmZs"
+    const altMatch = errorText.match(/will reset after (\d+h)?(\d+m)?(\d+s)?/i);
+    if (!altMatch) return null;
+    return computeDurationMs(altMatch);
+  }
+
+  return computeDurationMs(match);
+}
+
+/**
+ * Compute total milliseconds from regex match groups (Xh)(Ym)(Zs)
+ */
+function computeDurationMs(match) {
+  let totalMs = 0;
+  if (match[1]) totalMs += parseInt(match[1], 10) * 3600 * 1000; // hours
+  if (match[2]) totalMs += parseInt(match[2], 10) * 60 * 1000; // minutes
+  if (match[3]) totalMs += parseInt(match[3], 10) * 1000; // seconds
+  return totalMs > 0 ? totalMs : null;
+}
+
 // ─── Error Classification ───────────────────────────────────────────────────
 
 /**
@@ -428,6 +461,9 @@ export function checkFallbackError(
       lowerError.includes("rate limit") ||
       lowerError.includes("too many requests") ||
       lowerError.includes("quota exceeded") ||
+      lowerError.includes("quota will reset") ||
+      lowerError.includes("exhausted your capacity") ||
+      lowerError.includes("quota exhausted") ||
       lowerError.includes("capacity") ||
       lowerError.includes("overloaded")
     ) {
@@ -443,6 +479,15 @@ export function checkFallbackError(
           };
         }
       }
+      const retryFromBody = parseRetryFromErrorText(errorStr);
+      if (retryFromBody && retryFromBody > 60_000) {
+        return {
+          shouldFallback: true,
+          cooldownMs: retryFromBody,
+          newBackoffLevel: 0,
+          reason: RateLimitReason.RATE_LIMIT_EXCEEDED,
+        };
+      }
       const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel);
       const reason = classifyErrorText(errorStr);
       return {

open-sse/services/errorClassifier.ts

@@ -46,6 +46,9 @@ export function classifyProviderError(statusCode: number, responseBody: unknown)
   }
 
   if (statusCode === 402) return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
+  if (statusCode === 403 && accountDeactivated) {
+    return PROVIDER_ERROR_TYPES.ACCOUNT_DEACTIVATED;
+  }
   if (statusCode === 403) return PROVIDER_ERROR_TYPES.FORBIDDEN;
   if (statusCode >= 500) return PROVIDER_ERROR_TYPES.SERVER_ERROR;
 

tests/unit/error-classification.test.mjs

@@ -1,7 +1,7 @@
 import test from "node:test";
 import assert from "node:assert/strict";
 
-const { checkFallbackError, getProviderProfile } =
+const { checkFallbackError, getProviderProfile, parseRetryFromErrorText } =
   await import("../../open-sse/services/accountFallback.ts");
 
 const { getProviderCategory } = await import("../../open-sse/config/providerRegistry.ts");
@@ -127,3 +127,64 @@ test("400 bad request: still returns shouldFallback false", () => {
   const result = checkFallbackError(400, "", 0, null, "groq");
   assert.equal(result.shouldFallback, false);
 });
+
+// ─── T07: Retry Time Parsing from Error Text ─────────────────────────────────
+
+test("parseRetryFromErrorText: parses 27h41m36s format", () => {
+  const result = parseRetryFromErrorText("Your quota will reset after 27h41m36s");
+  assert.equal(result, 27 * 3600 * 1000 + 41 * 60 * 1000 + 36 * 1000);
+});
+
+test("parseRetryFromErrorText: parses 2h30m format", () => {
+  const result = parseRetryFromErrorText("quota will reset after 2h30m");
+  assert.equal(result, 2 * 3600 * 1000 + 30 * 60 * 1000);
+});
+
+test("parseRetryFromErrorText: parses 45m format", () => {
+  const result = parseRetryFromErrorText("reset after 45m");
+  assert.equal(result, 45 * 60 * 1000);
+});
+
+test("parseRetryFromErrorText: parses 30s format", () => {
+  const result = parseRetryFromErrorText("reset after 30s");
+  assert.equal(result, 30 * 1000);
+});
+
+test("parseRetryFromErrorText: returns null for invalid format", () => {
+  const result = parseRetryFromErrorText("invalid error message");
+  assert.equal(result, null);
+});
+
+test("parseRetryFromErrorText: parses will reset after variant", () => {
+  const result = parseRetryFromErrorText("quota will reset after 5h");
+  assert.equal(result, 5 * 3600 * 1000);
+});
+
+// ─── T06: Keyword Matching for Long Cooldowns ────────────────────────────────
+
+test("quota will reset keyword triggers long cooldown from body", () => {
+  const result = checkFallbackError(
+    429,
+    "Your quota will reset after 27h41m36s",
+    0,
+    null,
+    "antigravity",
+    null
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.ok(result.cooldownMs > 60_000, "cooldownMs should be > 60s");
+  assert.equal(result.newBackoffLevel, 0, "backoffLevel should reset to 0");
+});
+
+test("exhausted your capacity keyword triggers long cooldown", () => {
+  const result = checkFallbackError(
+    429,
+    "You have exhausted your capacity. Your quota will reset after 2h",
+    0,
+    null,
+    "antigravity",
+    null
+  );
+  assert.equal(result.shouldFallback, true);
+  assert.ok(result.cooldownMs > 60_000);
+});