fix(antigravity): skip credits retry on full_quota_exhausted, persist 24h cooldown to DB (#3707) (#3754)

Three bugs fixed together:
1. decide429() was called AFTER the Google One AI credits retry — a quota-exhausted
   account could hang ~41s on the credits HTTP call before the full_quota_exhausted
   verdict was ever computed. Now decide429() runs first; credits retry is skipped
   when kind === full_quota_exhausted.
2. setConnectionRateLimitUntil() was never called from the antigravity executor —
   the 24h cooldown state lived only in memory and was lost on restart, causing
   post-restart requests to re-learn exhaustion the hard way (7,412 upstream 429s
   documented in #3707). markConnectionQuotaExhausted() now persists it to the DB.
3. antigravity429Engine classify429() did not recognise the real Antigravity quota
   message ("Individual quota reached. Contact your administrator to enable overages.")
   — it fell through to the unknown category and never triggered the quota_exhausted
   path. Added "quota reached", "enable overages", "individual quota" to the keywords.

CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### 🐛 Fixed
 
+- fix(antigravity): skip Google One AI credits retry on `full_quota_exhausted` verdict — antigravity executor now calls `decide429()` before attempting the credits retry so that a quota-exhausted account (24h cooldown) bypasses the extra upstream HTTP call instead of hanging for up to ~41s. Also persists the cooldown in the DB via `setConnectionRateLimitUntil` so post-restart routing skips exhausted connections without re-learning the hard way. Bonus: `antigravity429Engine` now recognises the real Antigravity "Individual quota reached. Contact your administrator to enable overages." error message as `quota_exhausted`. ([#3707](https://github.com/diegosouzapw/OmniRoute/issues/3707) — thanks @andrea-kingautomation)
+
 - fix(cli): `ServerSupervisor.handleExit` now coerces the exit code to a number before calling `process.exit()` — Node.js v24 throws `TypeError [ERR_INVALID_ARG_TYPE]` when `process.exit()` receives a string (e.g. `'ENOENT'` from a spawn `error` event's `err.code`). The `error` callback also now passes `-1` instead of the raw `err.code`, which is an OS error string rather than a meaningful exit code. ([#3748](https://github.com/diegosouzapw/OmniRoute/issues/3748))
 
 ---

open-sse/executors/antigravity.ts

@@ -30,6 +30,7 @@ import {
   handleCreditsFailure,
 } from "../services/antigravityCredits.ts";
 import { persistCreditBalance, getAllPersistedCreditBalances } from "@/lib/db/creditBalance";
+import { setConnectionRateLimitUntil } from "@/lib/db/providers";
 import { getMitmAlias } from "@/lib/db/models";
 import { obfuscateSensitiveWords } from "../services/antigravityObfuscation.ts";
 import { resolveAntigravityVersion } from "../services/antigravityVersion.ts";
@@ -328,6 +329,19 @@ function markCreditsExhausted(accountId: string): void {
   creditsExhaustedUntil.set(accountId, Date.now() + CREDITS_EXHAUSTED_TTL_MS);
 }
 
+/**
+ * Persist a quota-exhausted cooldown to the DB for `connectionId` so that
+ * cross-request and post-restart routing skips this connection until the
+ * cooldown expires. Exported for unit testing. @internal
+ */
+export function markConnectionQuotaExhausted(connectionId: string, retryAfterMs: number): void {
+  try {
+    setConnectionRateLimitUntil(connectionId, Date.now() + retryAfterMs);
+  } catch {
+    // DB write failure must never crash the request path
+  }
+}
+
 /**
  * Accumulate one Antigravity SSE `data:` payload into `collected`. Exported for unit
  * tests (the markdown / candidate-parts extraction branches). @internal
@@ -1183,10 +1197,21 @@ export class AntigravityExecutor extends BaseExecutor {
               const effectiveRetryHintMs = retryMs ?? parsedRetryMs ?? null;
               const category = classify429(errorMessage);
 
-              // 3. For quota_exhausted, attempt Google One AI credits retry FIRST!
-              //    Skip if credits were already injected on the first call
-              //    (creditsMode === "always") — no point re-running with the
-              //    same body. Record the failure so the 5h breaker kicks in.
+              // 3. Decide final retry time BEFORE the credits retry so that
+              //    full_quota_exhausted can skip the credits attempt entirely
+              //    (avoids ~41s hold on an already-exhausted account) and
+              //    persist the cooldown to DB for post-restart routing.
+              const decision: Decision = decide429(category, parsedRetryMs);
+              retryMs = decision.retryAfterMs;
+              log?.debug?.(
+                "AG_429",
+                `Category: ${category}, Decision: ${decision.kind} — ${decision.reason}`
+              );
+
+              if (decision.kind === "full_quota_exhausted" && retryMs) {
+                markConnectionQuotaExhausted(accountId, retryMs);
+              }
+
               const creditsAlreadyInjected =
                 (transformedBody as { enabledCreditTypes?: unknown }).enabledCreditTypes != null;
 
@@ -1198,6 +1223,7 @@ export class AntigravityExecutor extends BaseExecutor {
 
               if (
                 category === "quota_exhausted" &&
+                decision.kind !== "full_quota_exhausted" &&
                 !creditsAlreadyInjected &&
                 shouldRetryWithCredits(credentials?.accessToken || "", creditsMode !== "off")
               ) {
@@ -1269,13 +1295,6 @@ export class AntigravityExecutor extends BaseExecutor {
                 }
               }
 
-              // 4. Decide final retry time (apply 4-tier engine)
-              const decision: Decision = decide429(category, parsedRetryMs);
-              retryMs = decision.retryAfterMs;
-              log?.debug?.(
-                "AG_429",
-                `Category: ${category}, Decision: ${decision.kind} — ${decision.reason}`
-              );
             } catch (e) {
               // Ignore parse errors, will fall back to exponential backoff
             }

open-sse/services/antigravity429Engine.ts

@@ -32,7 +32,14 @@ export interface Decision {
   reason: string;
 }
 
-const QUOTA_EXHAUSTED_KEYWORDS = ["quota_exhausted", "quota exhausted"];
+const QUOTA_EXHAUSTED_KEYWORDS = [
+  "quota_exhausted",
+  "quota exhausted",
+  // Antigravity native message: "Individual quota reached. Contact your administrator to enable overages."
+  "quota reached",
+  "enable overages",
+  "individual quota",
+];
 
 const CREDITS_EXHAUSTED_KEYWORDS = [
   "google_one_ai",

tests/unit/antigravity-429-quota-cooldown.test.ts

@@ -0,0 +1,98 @@
+/**
+ * TDD regression tests for #3707:
+ * 1. `decide429("quota_exhausted")` → `full_quota_exhausted` verdict (engine contract)
+ * 2. `markConnectionQuotaExhausted` persists the 24h cooldown in the DB so that
+ *    cross-request and post-restart routing skips exhausted connections.
+ *
+ * Bug: before the fix the executor never called `setConnectionRateLimitUntil`,
+ * so `isConnectionRateLimited` always returned false for AG connections that
+ * had their daily quota exhausted — learned state was lost on restart.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ag-quota-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+
+import {
+  classify429,
+  decide429,
+  FULL_QUOTA_COOLDOWN_MS,
+} from "../../open-sse/services/antigravity429Engine.ts";
+import { markConnectionQuotaExhausted } from "../../open-sse/executors/antigravity.ts";
+
+test.after(() => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+// ── Engine contract (regression guard) ───────────────────────────────────────
+
+test("decide429: quota_exhausted category → full_quota_exhausted kind with 24h cooldown", () => {
+  const decision = decide429("quota_exhausted", null);
+  assert.equal(decision.kind, "full_quota_exhausted");
+  assert.equal(decision.retryAfterMs, FULL_QUOTA_COOLDOWN_MS);
+  assert.equal(FULL_QUOTA_COOLDOWN_MS, 24 * 60 * 60 * 1000, "cooldown must be 24h");
+});
+
+test("decide429: quota_exhausted with explicit retryAfterMs preserves the provided value", () => {
+  const twoDaysMs = 2 * 24 * 60 * 60 * 1000;
+  const decision = decide429("quota_exhausted", twoDaysMs);
+  assert.equal(decision.kind, "full_quota_exhausted");
+  assert.equal(decision.retryAfterMs, twoDaysMs);
+});
+
+test("classify429: AG 'Individual quota reached' message → quota_exhausted", () => {
+  const msg =
+    "Individual quota reached. Contact your administrator to enable overages. Resets in 14h22m.";
+  assert.equal(classify429(msg), "quota_exhausted");
+});
+
+// ── DB persistence (the missing wire — Bug #2) ───────────────────────────────
+
+test("markConnectionQuotaExhausted persists 24h cooldown; isConnectionRateLimited returns true", async () => {
+  const conn = await providersDb.createProviderConnection({
+    provider: "antigravity",
+    authType: "oauth",
+    name: "AG Test Quota",
+  });
+  const connId = (conn as any).id;
+
+  assert.equal(
+    providersDb.isConnectionRateLimited(connId),
+    false,
+    "should start as not rate-limited"
+  );
+
+  markConnectionQuotaExhausted(connId, FULL_QUOTA_COOLDOWN_MS);
+
+  assert.equal(
+    providersDb.isConnectionRateLimited(connId),
+    true,
+    "should be rate-limited after marking quota exhausted"
+  );
+});
+
+test("markConnectionQuotaExhausted: expired cooldown does not block the connection", async () => {
+  const conn = await providersDb.createProviderConnection({
+    provider: "antigravity",
+    authType: "oauth",
+    name: "AG Test Expired",
+  });
+  const connId = (conn as any).id;
+
+  // Set cooldown in the past — simulates expired cooldown
+  providersDb.setConnectionRateLimitUntil(connId, Date.now() - 1);
+  assert.equal(
+    providersDb.isConnectionRateLimited(connId),
+    false,
+    "expired cooldown should not block"
+  );
+});