From 37116fd4d6fef68340b6293150d11012463a49d0 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Sat, 13 Jun 2026 03:01:09 -0300 Subject: [PATCH] fix(antigravity): skip credits retry on full_quota_exhausted, persist 24h cooldown to DB (#3707) (#3754) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs fixed together: 1. decide429() was called AFTER the Google One AI credits retry — a quota-exhausted account could hang ~41s on the credits HTTP call before the full_quota_exhausted verdict was ever computed. Now decide429() runs first; credits retry is skipped when kind === full_quota_exhausted. 2. setConnectionRateLimitUntil() was never called from the antigravity executor — the 24h cooldown state lived only in memory and was lost on restart, causing post-restart requests to re-learn exhaustion the hard way (7,412 upstream 429s documented in #3707). markConnectionQuotaExhausted() now persists it to the DB. 3. antigravity429Engine classify429() did not recognise the real Antigravity quota message ("Individual quota reached. Contact your administrator to enable overages.") — it fell through to the unknown category and never triggered the quota_exhausted path. Added "quota reached", "enable overages", "individual quota" to the keywords. --- CHANGELOG.md | 2 + open-sse/executors/antigravity.ts | 41 +++++--- open-sse/services/antigravity429Engine.ts | 9 +- .../antigravity-429-quota-cooldown.test.ts | 98 +++++++++++++++++++ 4 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 tests/unit/antigravity-429-quota-cooldown.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 808fd81e6f..09349db1c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### 🐛 Fixed +- fix(antigravity): skip Google One AI credits retry on `full_quota_exhausted` verdict — antigravity executor now calls `decide429()` before attempting the credits retry so that a quota-exhausted account (24h cooldown) bypasses the extra upstream HTTP call instead of hanging for up to ~41s. Also persists the cooldown in the DB via `setConnectionRateLimitUntil` so post-restart routing skips exhausted connections without re-learning the hard way. Bonus: `antigravity429Engine` now recognises the real Antigravity "Individual quota reached. Contact your administrator to enable overages." error message as `quota_exhausted`. ([#3707](https://github.com/diegosouzapw/OmniRoute/issues/3707) — thanks @andrea-kingautomation) + - fix(cli): `ServerSupervisor.handleExit` now coerces the exit code to a number before calling `process.exit()` — Node.js v24 throws `TypeError [ERR_INVALID_ARG_TYPE]` when `process.exit()` receives a string (e.g. `'ENOENT'` from a spawn `error` event's `err.code`). The `error` callback also now passes `-1` instead of the raw `err.code`, which is an OS error string rather than a meaningful exit code. ([#3748](https://github.com/diegosouzapw/OmniRoute/issues/3748)) --- diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index cd0f50fe50..d55a9162b9 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -30,6 +30,7 @@ import { handleCreditsFailure, } from "../services/antigravityCredits.ts"; import { persistCreditBalance, getAllPersistedCreditBalances } from "@/lib/db/creditBalance"; +import { setConnectionRateLimitUntil } from "@/lib/db/providers"; import { getMitmAlias } from "@/lib/db/models"; import { obfuscateSensitiveWords } from "../services/antigravityObfuscation.ts"; import { resolveAntigravityVersion } from "../services/antigravityVersion.ts"; @@ -328,6 +329,19 @@ function markCreditsExhausted(accountId: string): void { creditsExhaustedUntil.set(accountId, Date.now() + CREDITS_EXHAUSTED_TTL_MS); } +/** + * Persist a quota-exhausted cooldown to the DB for `connectionId` so that + * cross-request and post-restart routing skips this connection until the + * cooldown expires. Exported for unit testing. @internal + */ +export function markConnectionQuotaExhausted(connectionId: string, retryAfterMs: number): void { + try { + setConnectionRateLimitUntil(connectionId, Date.now() + retryAfterMs); + } catch { + // DB write failure must never crash the request path + } +} + /** * Accumulate one Antigravity SSE `data:` payload into `collected`. Exported for unit * tests (the markdown / candidate-parts extraction branches). @internal @@ -1183,10 +1197,21 @@ export class AntigravityExecutor extends BaseExecutor { const effectiveRetryHintMs = retryMs ?? parsedRetryMs ?? null; const category = classify429(errorMessage); - // 3. For quota_exhausted, attempt Google One AI credits retry FIRST! - // Skip if credits were already injected on the first call - // (creditsMode === "always") — no point re-running with the - // same body. Record the failure so the 5h breaker kicks in. + // 3. Decide final retry time BEFORE the credits retry so that + // full_quota_exhausted can skip the credits attempt entirely + // (avoids ~41s hold on an already-exhausted account) and + // persist the cooldown to DB for post-restart routing. + const decision: Decision = decide429(category, parsedRetryMs); + retryMs = decision.retryAfterMs; + log?.debug?.( + "AG_429", + `Category: ${category}, Decision: ${decision.kind} — ${decision.reason}` + ); + + if (decision.kind === "full_quota_exhausted" && retryMs) { + markConnectionQuotaExhausted(accountId, retryMs); + } + const creditsAlreadyInjected = (transformedBody as { enabledCreditTypes?: unknown }).enabledCreditTypes != null; @@ -1198,6 +1223,7 @@ export class AntigravityExecutor extends BaseExecutor { if ( category === "quota_exhausted" && + decision.kind !== "full_quota_exhausted" && !creditsAlreadyInjected && shouldRetryWithCredits(credentials?.accessToken || "", creditsMode !== "off") ) { @@ -1269,13 +1295,6 @@ export class AntigravityExecutor extends BaseExecutor { } } - // 4. Decide final retry time (apply 4-tier engine) - const decision: Decision = decide429(category, parsedRetryMs); - retryMs = decision.retryAfterMs; - log?.debug?.( - "AG_429", - `Category: ${category}, Decision: ${decision.kind} — ${decision.reason}` - ); } catch (e) { // Ignore parse errors, will fall back to exponential backoff } diff --git a/open-sse/services/antigravity429Engine.ts b/open-sse/services/antigravity429Engine.ts index f1b58d7bc0..36364a9e6c 100644 --- a/open-sse/services/antigravity429Engine.ts +++ b/open-sse/services/antigravity429Engine.ts @@ -32,7 +32,14 @@ export interface Decision { reason: string; } -const QUOTA_EXHAUSTED_KEYWORDS = ["quota_exhausted", "quota exhausted"]; +const QUOTA_EXHAUSTED_KEYWORDS = [ + "quota_exhausted", + "quota exhausted", + // Antigravity native message: "Individual quota reached. Contact your administrator to enable overages." + "quota reached", + "enable overages", + "individual quota", +]; const CREDITS_EXHAUSTED_KEYWORDS = [ "google_one_ai", diff --git a/tests/unit/antigravity-429-quota-cooldown.test.ts b/tests/unit/antigravity-429-quota-cooldown.test.ts new file mode 100644 index 0000000000..3791e4ff2a --- /dev/null +++ b/tests/unit/antigravity-429-quota-cooldown.test.ts @@ -0,0 +1,98 @@ +/** + * TDD regression tests for #3707: + * 1. `decide429("quota_exhausted")` → `full_quota_exhausted` verdict (engine contract) + * 2. `markConnectionQuotaExhausted` persists the 24h cooldown in the DB so that + * cross-request and post-restart routing skips exhausted connections. + * + * Bug: before the fix the executor never called `setConnectionRateLimitUntil`, + * so `isConnectionRateLimited` always returned false for AG connections that + * had their daily quota exhausted — learned state was lost on restart. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ag-quota-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); + +import { + classify429, + decide429, + FULL_QUOTA_COOLDOWN_MS, +} from "../../open-sse/services/antigravity429Engine.ts"; +import { markConnectionQuotaExhausted } from "../../open-sse/executors/antigravity.ts"; + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +// ── Engine contract (regression guard) ─────────────────────────────────────── + +test("decide429: quota_exhausted category → full_quota_exhausted kind with 24h cooldown", () => { + const decision = decide429("quota_exhausted", null); + assert.equal(decision.kind, "full_quota_exhausted"); + assert.equal(decision.retryAfterMs, FULL_QUOTA_COOLDOWN_MS); + assert.equal(FULL_QUOTA_COOLDOWN_MS, 24 * 60 * 60 * 1000, "cooldown must be 24h"); +}); + +test("decide429: quota_exhausted with explicit retryAfterMs preserves the provided value", () => { + const twoDaysMs = 2 * 24 * 60 * 60 * 1000; + const decision = decide429("quota_exhausted", twoDaysMs); + assert.equal(decision.kind, "full_quota_exhausted"); + assert.equal(decision.retryAfterMs, twoDaysMs); +}); + +test("classify429: AG 'Individual quota reached' message → quota_exhausted", () => { + const msg = + "Individual quota reached. Contact your administrator to enable overages. Resets in 14h22m."; + assert.equal(classify429(msg), "quota_exhausted"); +}); + +// ── DB persistence (the missing wire — Bug #2) ─────────────────────────────── + +test("markConnectionQuotaExhausted persists 24h cooldown; isConnectionRateLimited returns true", async () => { + const conn = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "AG Test Quota", + }); + const connId = (conn as any).id; + + assert.equal( + providersDb.isConnectionRateLimited(connId), + false, + "should start as not rate-limited" + ); + + markConnectionQuotaExhausted(connId, FULL_QUOTA_COOLDOWN_MS); + + assert.equal( + providersDb.isConnectionRateLimited(connId), + true, + "should be rate-limited after marking quota exhausted" + ); +}); + +test("markConnectionQuotaExhausted: expired cooldown does not block the connection", async () => { + const conn = await providersDb.createProviderConnection({ + provider: "antigravity", + authType: "oauth", + name: "AG Test Expired", + }); + const connId = (conn as any).id; + + // Set cooldown in the past — simulates expired cooldown + providersDb.setConnectionRateLimitUntil(connId, Date.now() - 1); + assert.equal( + providersDb.isConnectionRateLimited(connId), + false, + "expired cooldown should not block" + ); +});