From 808992a717d75372e55198981feb92bd482d12b4 Mon Sep 17 00:00:00 2001 From: Paco Cartones <253313177+pacocartones@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:14:24 +0200 Subject: [PATCH] fix(antigravity): cap unverified body retry hints (#11823) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserves whether a 429 retry hint came from transport headers, structured google.rpc.RetryInfo, or unverified response-body text, and caps body-derived cooldowns at the operator's configured maxCooldownMs so an unverified upstream hint can no longer force an arbitrarily long model/semaphore lockout — authoritative header/structured resets stay intact across combo, chat, and Responses paths. Closes #11695. 29/29 focused tests passing. Thanks! --- .../11695-antigravity-cooldown-provenance.md | 1 + config/quality/eslint-suppressions.json | 4 +- open-sse/executors/antigravity.ts | 30 +++- open-sse/handlers/chatCore.ts | 54 ++++++- open-sse/services/accountFallback.ts | 141 ++++++++++++++---- open-sse/services/combo.ts | 24 ++- open-sse/services/retryAfterJson.ts | 25 +++- src/sse/handlers/chat.ts | 1 + src/sse/services/auth.ts | 16 +- stryker.conf.json | 1 + .../antigravity-429-quota-cooldown.test.ts | 117 ++++++++++++++- .../combo-lockout-quota-reset-6863.test.ts | 17 +-- ...bo-model-lockout-honors-reset-1308.test.ts | 92 ++++++++++++ 13 files changed, 445 insertions(+), 78 deletions(-) create mode 100644 changelog.d/fixes/11695-antigravity-cooldown-provenance.md diff --git a/changelog.d/fixes/11695-antigravity-cooldown-provenance.md b/changelog.d/fixes/11695-antigravity-cooldown-provenance.md new file mode 100644 index 0000000000..7638d9dc81 --- /dev/null +++ b/changelog.d/fixes/11695-antigravity-cooldown-provenance.md @@ -0,0 +1 @@ +- Cap prose-derived Antigravity quota resets at the configured model cooldown maximum while preserving authoritative Retry-After headers and Google RetryInfo hints. (#11695) Thanks @pacocartones. diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index 67e564098f..a23cbbcfa6 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -512,7 +512,7 @@ }, "open-sse/services/combo.ts": { "@typescript-eslint/no-unused-vars": { - "count": 22 + "count": 21 } }, "open-sse/services/combo/concurrencyCaps.ts": { @@ -6542,4 +6542,4 @@ "count": 2 } } -} +} \ No newline at end of file diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts index d239f6d7d1..c16d626812 100644 --- a/open-sse/executors/antigravity.ts +++ b/open-sse/executors/antigravity.ts @@ -13,7 +13,11 @@ import { getAntigravityOAuthUserAgent, } from "../services/antigravityHeaders.ts"; import { classify429, decide429, type Decision } from "../services/antigravity429Engine.ts"; -import { lockExactModel } from "../services/accountFallback.ts"; +import { + parseRetryFromErrorText, + type RetryHintProvenance, +} from "../services/accountFallback.ts"; +import { parseDetailedRetryHintFromJsonBody } from "../services/retryAfterJson.ts"; import { shouldRetryWithCredits, shouldUseCreditsFirst, @@ -89,6 +93,21 @@ const ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS = 15_000; // the no-Retry-After transient/429 backoff loop in executeOnce(). const MAX_AUTO_RETRIES = 3; +export function resolveAntigravityBodyRetryHint( + body: string, + errorMessage: string +): { retryMs: number; source: RetryHintProvenance } | null { + const structured = parseDetailedRetryHintFromJsonBody(body, Number.MAX_SAFE_INTEGER); + if (structured) { + return { + retryMs: structured.retryAfterMs, + source: structured.provenance, + }; + } + const retryMs = parseRetryFromErrorText(errorMessage); + return retryMs ? { retryMs, source: "body" } : null; +} + const ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS: RegExp[] = [ /high\s+traffic/i, /agent\s+(execution\s+)?terminated\s+due\s+to\s+error/i, @@ -1544,7 +1563,6 @@ export class AntigravityExecutor extends BaseExecutor { const { response, url, - model, headers, transformedBody, credentials, @@ -1562,7 +1580,8 @@ export class AntigravityExecutor extends BaseExecutor { const errorMessage = buildAntigravity429ErrorMessage(errorJson); // 1. Try to parse explicit retry time from message - const parsedRetryMs = this.parseRetryFromErrorMessage(errorMessage); + const bodyRetryHint = resolveAntigravityBodyRetryHint(errorBody, errorMessage); + const parsedRetryMs = bodyRetryHint?.retryMs ?? null; // 2. Classify 429, then decide the final retry time BEFORE the credits retry so // full_quota_exhausted can skip the credits attempt entirely (avoids ~41s hold @@ -1580,11 +1599,6 @@ export class AntigravityExecutor extends BaseExecutor { !creditsRetryState.attempted && shouldRetryWithCredits(credentials?.accessToken || "", creditsMode); - // Retry mode gets one credits attempt before the exact-model lock is persisted. - if (decision.kind === "full_quota_exhausted" && retryMs && !creditsRetryEligible) { - lockExactModel(this.provider, accountId, model, "quota_exhausted", retryMs); - } - if (category === "quota_exhausted" && creditsAlreadyInjected) { handleCreditsFailure(credentials?.accessToken || ""); log.warn("AG_CREDITS", "Credits-first request 429'd — credits likely exhausted"); diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index e0616946e5..374c8e3b6f 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -399,8 +399,12 @@ import { acquireMany as acquireConcurrencyGates, markBlocked as markAccountSemaphoreBlocked, } from "../services/accountSemaphore.ts"; -import { lockModel, lockModelIfPerModelQuota } from "../services/accountFallback.ts"; -import { lockExactModel } from "../services/accountFallback.ts"; +import { + lockModel, + lockModelIfPerModelQuota, + recordCoreOwnedAntigravityQuotaState, + shouldDeferAntigravityQuotaStateToCaller, +} from "../services/accountFallback.ts"; import { generateSignature, getCachedResponse, @@ -4280,19 +4284,56 @@ export async function handleChatCore({ } // Providers with per-model quotas — lock the model only, not the connection - const quotaCooldownMs = kimiRateLimitResetAt + let quotaCooldownMs = kimiRateLimitResetAt ? Math.max(new Date(kimiRateLimitResetAt).getTime() - Date.now(), 0) : retryAfterMs || COOLDOWN_MS.rateLimit; + const deferAntigravityQuotaStateToCaller = + shouldDeferAntigravityQuotaStateToCaller( + provider, + typeof onStreamFailure === "function" + ); + const isAntigravityQuotaFamily = + shouldDeferAntigravityQuotaStateToCaller(provider, true); + let coreOwnedAntigravityLockout: { + cooldownMs: number; + failureCount: number; + } | null = null; + if (isAntigravityQuotaFamily && !deferAntigravityQuotaStateToCaller) { + const quotaErrorText = + typeof upstreamErrorBody === "string" + ? upstreamErrorBody + : upstreamErrorBody == null + ? message + : JSON.stringify(upstreamErrorBody); + coreOwnedAntigravityLockout = await recordCoreOwnedAntigravityQuotaState({ + provider, + connectionId: errorConnectionId, + model, + status: statusCode, + errorText: quotaErrorText, + headers: providerResponse.headers, + }); + quotaCooldownMs = coreOwnedAntigravityLockout.cooldownMs; + } const accountSemaphoreKey = resolveAccountSemaphoreKey({ provider, model: currentModel, connectionId: errorConnectionId, credentials, }); - if (accountSemaphoreKey) { + if (accountSemaphoreKey && !deferAntigravityQuotaStateToCaller) { markAccountSemaphoreBlocked(accountSemaphoreKey, quotaCooldownMs); } - if (kimiRateLimitResetAt) { + if (deferAntigravityQuotaStateToCaller) { + // Defer both model and account-semaphore cooldowns to + // markAccountUnavailable, where header/body provenance and the + // configured maxCooldownMs are available. Direct consumers such + // as Responses pass no owner callback and retain core ownership. + } else if (coreOwnedAntigravityLockout) { + console.warn( + `[provider] Node ${errorConnectionId} Antigravity model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(coreOwnedAntigravityLockout.cooldownMs / 1000)}s (failureCount=${coreOwnedAntigravityLockout.failureCount}, owner=core)` + ); + } else if (kimiRateLimitResetAt) { await updateProviderConnection(errorConnectionId, { testStatus: "unavailable", rateLimitedUntil: kimiRateLimitResetAt, @@ -4305,8 +4346,7 @@ export async function handleChatCore({ `[provider] Node ${errorConnectionId} Kimi request window exhausted (${statusCode}) — retrying after ${kimiRateLimitResetAt}` ); } else if (isModelScope() && errorConnectionId) { - const lockFn = provider === "antigravity" ? lockExactModel : lockModel; - lockFn(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); + lockModel(provider, errorConnectionId, model, "quota_exhausted", quotaCooldownMs); console.warn( `[provider] Node ${errorConnectionId} ModelScope model quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)` ); diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 63407a38fc..8d1c3a88a1 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -59,9 +59,19 @@ import { import { setConnectionRateLimitUntil } from "@/lib/db/providers"; import { parseRetryHintFromJsonBody, + parseDetailedRetryHintFromJsonBody, parseDelayString, MAX_SHORT_RETRY_HINT_MS, } from "./retryAfterJson.ts"; + +export type RetryHintProvenance = "header" | "google_rpc_retry_info" | "body"; + +export function retryHintBypassesMaxCooldownMs( + provenance: RetryHintProvenance | undefined +): boolean { + return provenance === "header" || provenance === "google_rpc_retry_info"; +} + import { isSubscriptionQuotaText, buildSubscriptionQuotaFallback, @@ -503,6 +513,66 @@ function getCanonicalLockProvider(provider: string): string { return canonical; } +export function shouldDeferAntigravityQuotaStateToCaller( + provider: string, + hasCallerOwner: boolean +): boolean { + const canonicalProvider = getCanonicalLockProvider(provider); + return ( + hasCallerOwner && (canonicalProvider === "antigravity" || canonicalProvider === "agy") + ); +} + +export async function recordCoreOwnedAntigravityQuotaState({ + provider, + connectionId, + model, + status, + errorText, + headers, + profileOverride = null, +}: { + provider: string; + connectionId: string; + model: string; + status: number; + errorText: string; + headers: Headers | Record | null; + profileOverride?: ProviderProfile | null; +}) { + const profile = profileOverride ?? (await getRuntimeProviderProfile(provider)); + const fallback = checkFallbackError( + status, + errorText, + 0, + model, + provider, + headers, + profile + ); + const lockout = recordModelLockoutFailure( + provider, + connectionId, + model, + "quota_exhausted", + status, + fallback.baseCooldownMs ?? profile.baseCooldownMs ?? COOLDOWN_MS.rateLimit, + profile, + { + exactCooldownMs: + fallback.usedUpstreamRetryHint === true + ? fallback.cooldownMs + : (fallback.quotaResetHintMs ?? null), + maxCooldownMs: profile.maxCooldownMs, + scope: "exact", + exactCooldownIsUpstreamReset: retryHintBypassesMaxCooldownMs( + fallback.retryHintSource + ), + } + ); + return { cooldownMs: lockout.cooldownMs, failureCount: lockout.failureCount }; +} + function getModelLockKey( provider: string, connectionId: string, @@ -654,13 +724,9 @@ export const lockExactModel = exactModelLock.createLockExactModel( /** * Pick the `exactCooldownMs` to apply to a model lockout (#1308). * - * When the upstream response carried an explicit reset longer than the base - * cooldown — e.g. Antigravity "Resets in 160h", a `Retry-After` header, or a - * parseable reset text already extracted by `checkFallbackError`/`parseRetryFromErrorText` - * into `parsedCooldownMs` — honor it exactly so an exhausted model is not retried - * again within minutes. Otherwise preserve the previous behavior: return `0` to let - * `recordModelLockoutFailure` apply its exponential backoff, or the base cooldown when - * backoff is disabled. + * Prefer a parsed reset longer than the base cooldown so a precise body hint + * still beats exponential backoff. Whether it may bypass maxCooldownMs is a + * separate provenance decision made by retryHintBypassesMaxCooldownMs. */ export function selectLockoutCooldownMs( parsedCooldownMs: number, @@ -686,14 +752,11 @@ export function recordModelLockoutFailure( scope?: "exact" | "quota_family"; /** * #6863 vs #7940: set true only when `exactCooldownMs` came from an actual - * upstream signal (Retry-After header, X-RateLimit-Reset, or a reset parsed - * from the error body — i.e. `usedUpstreamRetryHint`/`quotaResetHintMs` from - * `checkFallbackError`). Such a reset is honored exactly, even past - * `maxCooldownMs` — a real "Resets in 92h" must not be clamped down to - * minutes, or the router hammers 429 against quota that is known not to be - * back yet. Leave false/omitted for SYNTHETIC estimates (the quota_exhausted - * until-midnight default below, plain exponential backoff) — those stay - * capped, per #7940. + * authoritative upstream signal: Retry-After/X-RateLimit-Reset headers or + * google.rpc.RetryInfo. Generic JSON and prose-derived reset text are useful + * exact hints but remain bounded by maxCooldownMs. Leave false/omitted for + * those body hints and for synthetic estimates (the quota_exhausted + * until-midnight default below, plain exponential backoff). */ exactCooldownIsUpstreamReset?: boolean; } = {} @@ -1289,7 +1352,7 @@ export function parseRetryFromErrorText(errorText: unknown): number | null { } } - const match = /reset after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg); + const match = /resets? after (\d+h)?(\d+m)?(\d+s)?/i.exec(msg); if (match?.[1] || match?.[2] || match?.[3]) return computeDurationMs(match); // Variant without "reset after": "will reset after XhYmZs" @@ -1514,6 +1577,7 @@ export function checkFallbackError( baseCooldownMs?: number; newBackoffLevel?: number; usedUpstreamRetryHint?: boolean; + retryHintSource?: RetryHintProvenance; reason?: string; permanent?: boolean; creditsExhausted?: boolean; @@ -1597,22 +1661,39 @@ export function checkFallbackError( return null; } - function getUpstreamRetryHintMs() { - if (!profile?.useUpstreamRetryHints) return null; + function detectRetryHint(): { + retryAfterMs: number; + provenance: RetryHintProvenance; + } | null { const resetTime = parseResetFromHeaders(headers); if (resetTime) { const waitMs = Math.max(resetTime - Date.now(), 0); - if (waitMs > 0) return waitMs; + if (waitMs > 0) return { retryAfterMs: waitMs, provenance: "header" }; + } + + const detailedJsonHint = parseDetailedRetryHintFromJsonBody( + errorStr, + MAX_PROVIDER_COOLDOWN_MS + ); + if (detailedJsonHint) { + return { + retryAfterMs: detailedJsonHint.retryAfterMs, + provenance: detailedJsonHint.provenance, + }; } const retryFromErrorText = parseRetryFromErrorText(errorStr); if (retryFromErrorText && retryFromErrorText > 0) { - return retryFromErrorText; + return { retryAfterMs: retryFromErrorText, provenance: "body" }; } return null; } + function getUpstreamRetryHint() { + return profile?.useUpstreamRetryHints ? detectRetryHint() : null; + } + function getScaledBaseCooldown(reason: RateLimitReasonValue, level = backoffLevel) { void reason; const baseCooldownMs = @@ -1632,14 +1713,15 @@ export function checkFallbackError( } function buildRetryableFallback(reason: RateLimitReasonValue) { - const upstreamRetryHintMs = getUpstreamRetryHintMs(); - if (typeof upstreamRetryHintMs === "number" && upstreamRetryHintMs > 0) { + const upstreamRetryHint = getUpstreamRetryHint(); + if (upstreamRetryHint && upstreamRetryHint.retryAfterMs > 0) { return { shouldFallback: true, - cooldownMs: upstreamRetryHintMs, - baseCooldownMs: upstreamRetryHintMs, + cooldownMs: upstreamRetryHint.retryAfterMs, + baseCooldownMs: upstreamRetryHint.retryAfterMs, newBackoffLevel: 0, usedUpstreamRetryHint: true, + retryHintSource: upstreamRetryHint.provenance, reason, }; } @@ -1745,7 +1827,7 @@ export function checkFallbackError( if (shouldUseQuotaSignal && !isCreditsExhausted(errorStr) && !isDailyQuotaExhausted(errorStr)) { const subResult = buildSubscriptionQuotaFallback( errorStr, - getUpstreamRetryHintMs, + () => getUpstreamRetryHint()?.retryAfterMs ?? null, parseRetryFromErrorText, provider ); @@ -1760,7 +1842,13 @@ export function checkFallbackError( const sessionResult = buildSessionQuotaFallback(errorStr); if (sessionResult) return sessionResult; - const quotaResetHintMs = parseRetryFromErrorText(errorStr); + const detectedRetryHint = detectRetryHint(); + const quotaResetHintMs = detectedRetryHint?.retryAfterMs ?? parseRetryFromErrorText(errorStr); + const quotaResetHintSource: RetryHintProvenance | undefined = detectedRetryHint + ? detectedRetryHint.provenance + : quotaResetHintMs + ? "body" + : undefined; if ( shouldUseQuotaSignal && quotaResetHintMs && @@ -1770,6 +1858,7 @@ export function checkFallbackError( return { ...fallbackResult, quotaResetHintMs, + retryHintSource: fallbackResult.retryHintSource ?? quotaResetHintSource, }; } diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index 749b30376a..1f73cbcc9f 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -20,6 +20,7 @@ import { recordModelLockoutFailure, recordProviderFailure, recordProviderSuccess, + retryHintBypassesMaxCooldownMs, selectLockoutCooldownMs, } from "./accountFallback.ts"; import { @@ -2142,12 +2143,12 @@ async function handleComboChatInner({ fallbackResult.usedUpstreamRetryHint === true ? cooldownMs : (fallbackResult.quotaResetHintMs ?? 0); - // #6863 vs #7940: lockoutHintMs is only ever nonzero when it traces back to - // a genuine upstream signal (usedUpstreamRetryHint or a parsed quotaResetHintMs) - // — never a synthetic estimate. Tell recordModelLockoutFailure to honor it - // exactly instead of clamping it to maxCooldownMs (#7940's cap still applies - // to the exponential-backoff / synthetic-default paths). - const lockoutHintVerified = lockoutHintMs > 0; + // Only a transport header or google.rpc.RetryInfo is authoritative enough + // to bypass maxCooldownMs. Prose and generic JSON remain useful exact hints, + // but the operator cap still bounds them. + const lockoutHintVerified = retryHintBypassesMaxCooldownMs( + fallbackResult.retryHintSource + ); const selectedConnectionId = result.headers?.get("X-OmniRoute-Selected-Connection-Id") || result.headers?.get("x-omniroute-selected-connection-id") || @@ -2342,10 +2343,8 @@ async function handleComboChatInner({ // upstream reset (lockoutHintVerified) bypasses it. exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings), maxCooldownMs: mlSettings.maxCooldownMs, - // #6863: a parsed upstream quota reset is authoritative — the upstream - // told us exactly when it resets, so honor it in full instead of - // clamping to maxCooldownMs (which only bounds computed backoff). - exactCooldownIsUpstreamReset: lockoutHintMs > mlSettings.baseCooldownMs, + // Preserve authoritative structured/header resets; clamp body prose. + exactCooldownIsUpstreamReset: lockoutHintVerified, } ); lockoutRecorded = true; @@ -2434,9 +2433,8 @@ async function handleComboChatInner({ // upstream reset (lockoutHintVerified) bypasses it. exactCooldownMs: selectLockoutCooldownMs(lockoutHintMs, mlSettings), maxCooldownMs: mlSettings.maxCooldownMs, - // #6863: an authoritative parsed upstream reset must be honored in full, - // never clamped to maxCooldownMs (which only bounds computed backoff). - exactCooldownIsUpstreamReset: lockoutHintMs > mlSettings.baseCooldownMs, + // Preserve authoritative structured/header resets; clamp body prose. + exactCooldownIsUpstreamReset: lockoutHintVerified, } ); } diff --git a/open-sse/services/retryAfterJson.ts b/open-sse/services/retryAfterJson.ts index 1c4b89f30a..c0e7cbce6d 100644 --- a/open-sse/services/retryAfterJson.ts +++ b/open-sse/services/retryAfterJson.ts @@ -1,5 +1,12 @@ type JsonRecord = Record; +export type JsonRetryHintProvenance = "google_rpc_retry_info" | "body"; + +export type DetailedJsonRetryHint = { + retryAfterMs: number; + provenance: JsonRetryHintProvenance; +}; + function objectRecord(value: unknown): JsonRecord { return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; } @@ -63,7 +70,10 @@ function retryInfoDetailsMs(details: unknown): number | null { * Parse Retry-After hints from a 429 JSON response body. Providers use both * top-level and nested `error` fields for ISO timestamps and millisecond values. */ -export function parseRetryHintFromJsonBody(body: string, maxMs: number): number | null { +export function parseDetailedRetryHintFromJsonBody( + body: string, + maxMs: number +): DetailedJsonRetryHint | null { let parsed: unknown; try { parsed = JSON.parse(body); @@ -76,13 +86,20 @@ export function parseRetryHintFromJsonBody(body: string, maxMs: number): number const errorObj = objectRecord(root.error); const retryInfoMs = retryInfoDetailsMs(errorObj.details ?? root.details); - if (retryInfoMs !== null) return retryInfoMs; + if (retryInfoMs !== null) { + return { retryAfterMs: retryInfoMs, provenance: "google_rpc_retry_info" }; + } const isoHint = futureTimestampMs(errorObj.retryAfter ?? root.retryAfter, maxMs); - if (isoHint !== null) return isoHint; + if (isoHint !== null) return { retryAfterMs: isoHint, provenance: "body" }; - return positiveCappedMs( + const numericHint = positiveCappedMs( errorObj.retry_after_ms ?? root.retry_after_ms ?? errorObj.retryAfterMs ?? root.retryAfterMs, maxMs ); + return numericHint === null ? null : { retryAfterMs: numericHint, provenance: "body" }; +} + +export function parseRetryHintFromJsonBody(body: string, maxMs: number): number | null { + return parseDetailedRetryHintFromJsonBody(body, maxMs)?.retryAfterMs ?? null; } diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 065b39af6d..f3b65fa085 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -2304,6 +2304,7 @@ async function handleSingleModelChat( (failureKind === "rate_limit" || failureKind === "transient") ), isCombo, + headers: result.response.headers, } ); diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index a5d93bd39b..dea088ba3c 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -63,6 +63,7 @@ import { hasPerModelQuota, getRuntimeProviderProfile, recordModelLockoutFailure, + retryHintBypassesMaxCooldownMs, isProviderModelUnsupported400, } from "@omniroute/open-sse/services/accountFallback.ts"; import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts"; @@ -2548,6 +2549,7 @@ export async function markAccountUnavailable( persistUnavailableState?: boolean; /** Caller is the combo engine — it records its own model-level lockouts. */ isCombo?: boolean; + headers?: Headers | Record | null; } = {} ) { const currentMutex = markMutexes.get(connectionId) || Promise.resolve(); @@ -2663,7 +2665,7 @@ export async function markAccountUnavailable( backoffLevel, model, provider, - null, + options.headers ?? null, effectiveProviderProfile ); @@ -2901,13 +2903,11 @@ export async function markAccountUnavailable( : (fallbackResult.quotaResetHintMs ?? null), maxCooldownMs: mlSettings.maxCooldownMs, scope: usesExactAntigravityLock ? "exact" : undefined, - // #6863 vs #7940: exactCooldownMs above is only ever set from a genuine - // upstream signal (Retry-After/reset header or a parsed quotaResetHintMs) — - // never a synthetic estimate — so it must bypass maxCooldownMs instead of - // being clamped down to a window the upstream already told us is wrong. - exactCooldownIsUpstreamReset: - fallbackResult.usedUpstreamRetryHint === true || - typeof fallbackResult.quotaResetHintMs === "number", + // Only a transport header or google.rpc.RetryInfo can bypass maxCooldownMs. + // Prose and generic JSON hints remain exact but operator-capped. + exactCooldownIsUpstreamReset: retryHintBypassesMaxCooldownMs( + fallbackResult.retryHintSource + ), } ); // Update last error for observability (without changing terminal status) diff --git a/stryker.conf.json b/stryker.conf.json index a6a8d282ed..8784091f12 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -68,6 +68,7 @@ "tests/unit/aihorde-optional-api-key.test.ts", "tests/unit/alibaba-free-tier-exhaustion.test.ts", "tests/unit/anthropic-thinking-signature-recovery.test.ts", + "tests/unit/antigravity-429-quota-cooldown.test.ts", "tests/unit/antigravity-429-quota-tdd.test.ts", "tests/unit/antigravity-prefer-stored-project.test.ts", "tests/unit/api-key-policy-noauth-allowed-connections.test.ts", diff --git a/tests/unit/antigravity-429-quota-cooldown.test.ts b/tests/unit/antigravity-429-quota-cooldown.test.ts index eaf0a6e09b..c420781f0d 100644 --- a/tests/unit/antigravity-429-quota-cooldown.test.ts +++ b/tests/unit/antigravity-429-quota-cooldown.test.ts @@ -20,15 +20,27 @@ process.env.DATA_DIR = TEST_DATA_DIR; const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); +const { + clearAllModelLockouts, + getModelLockoutInfo, + recordModelLockoutFailure, + recordCoreOwnedAntigravityQuotaState, + getProviderProfile, + shouldDeferAntigravityQuotaStateToCaller, +} = await import("../../open-sse/services/accountFallback.ts"); import { classify429, decide429, FULL_QUOTA_COOLDOWN_MS, } from "../../open-sse/services/antigravity429Engine.ts"; -import { markConnectionQuotaExhausted } from "../../open-sse/executors/antigravity.ts"; +import { + markConnectionQuotaExhausted, + resolveAntigravityBodyRetryHint, +} from "../../open-sse/executors/antigravity.ts"; test.after(() => { + clearAllModelLockouts(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); @@ -121,3 +133,106 @@ test("markConnectionQuotaExhausted: expired cooldown does not block the connecti "expired cooldown should not block" ); }); + +test("direct Antigravity body prose preserves non-authoritative provenance", () => { + const body = JSON.stringify({ + error: { message: "Individual quota reached. Resets in 131h." }, + }); + + assert.deepEqual( + resolveAntigravityBodyRetryHint(body, "Individual quota reached. Resets in 131h."), + { retryMs: 131 * 60 * 60_000, source: "body" } + ); +}); + +test("direct Antigravity structured reset remains authoritative", () => { + const body = JSON.stringify({ + error: { + message: "Individual quota reached.", + details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "2h" }], + }, + }); + + assert.deepEqual(resolveAntigravityBodyRetryHint(body, "Individual quota reached."), { + retryMs: 2 * 60 * 60_000, + source: "google_rpc_retry_info", + }); +}); + +test("direct Antigravity has one downstream model-lock owner and clamps body prose", () => { + const chatCoreSource = fs.readFileSync( + path.resolve(import.meta.dirname, "../../open-sse/handlers/chatCore.ts"), + "utf8" + ); + assert.match( + chatCoreSource, + /accountSemaphoreKey && !deferAntigravityQuotaStateToCaller/, + "chatCore must not apply a prose-derived Antigravity semaphore TTL" + ); + assert.match( + chatCoreSource, + /if \(deferAntigravityQuotaStateToCaller\)[\s\S]{0,2000}else if \(kimiRateLimitResetAt\)/ + ); + assert.doesNotMatch(chatCoreSource, /lockExactModel/); + + clearAllModelLockouts(); + const maxCooldownMs = 30 * 60_000; + recordModelLockoutFailure( + "antigravity", + "direct-connection", + "direct-model", + "quota_exhausted", + 429, + 3_000, + null, + { + exactCooldownMs: 131 * 60 * 60_000, + maxCooldownMs, + scope: "exact", + exactCooldownIsUpstreamReset: false, + } + ); + const info = getModelLockoutInfo("antigravity", "direct-connection", "direct-model"); + assert.ok(info); + assert.equal(info.failureCount, 1); + assert.ok(info.remainingMs > maxCooldownMs - 5_000 && info.remainingMs <= maxCooldownMs); +}); + +test("Antigravity quota state is deferred only when a caller owner exists", () => { + assert.equal(shouldDeferAntigravityQuotaStateToCaller("antigravity", true), true); + assert.equal(shouldDeferAntigravityQuotaStateToCaller("agy", true), true); + assert.equal(shouldDeferAntigravityQuotaStateToCaller("antigravity", false), false); + assert.equal(shouldDeferAntigravityQuotaStateToCaller("agy", false), false); + assert.equal(shouldDeferAntigravityQuotaStateToCaller("gemini", true), false); +}); + +test("core-owned Antigravity quota state applies the same provenance-aware cap", async () => { + clearAllModelLockouts(); + const maxCooldownMs = 30 * 60_000; + const profile = { ...getProviderProfile("antigravity"), maxCooldownMs }; + const bodyResult = await recordCoreOwnedAntigravityQuotaState({ + provider: "agy", + connectionId: "responses-body", + model: "direct-model", + status: 429, + errorText: "Individual quota reached. Resets in 131h.", + headers: null, + profileOverride: profile, + }); + assert.equal(bodyResult.failureCount, 1); + assert.ok( + bodyResult.cooldownMs > maxCooldownMs - 5_000 && bodyResult.cooldownMs <= maxCooldownMs + ); + + const headerResult = await recordCoreOwnedAntigravityQuotaState({ + provider: "antigravity", + connectionId: "responses-header", + model: "direct-model", + status: 429, + errorText: "Individual quota reached.", + headers: new Headers({ "Retry-After": "7200" }), + profileOverride: profile, + }); + assert.equal(headerResult.failureCount, 1); + assert.ok(headerResult.cooldownMs > 2 * 60 * 60_000 - 5_000); +}); diff --git a/tests/unit/combo-lockout-quota-reset-6863.test.ts b/tests/unit/combo-lockout-quota-reset-6863.test.ts index 84eb7ed70a..463cd361d2 100644 --- a/tests/unit/combo-lockout-quota-reset-6863.test.ts +++ b/tests/unit/combo-lockout-quota-reset-6863.test.ts @@ -1,6 +1,5 @@ -// #6863: combo path model lockout must honor a parsed upstream quota reset -// ("Resets in 92h27m28s") instead of the base cooldown ladder, mirroring the -// single-model path (src/sse/services/auth.ts usedUpstreamRetryHint/quotaResetHintMs). +// #6863: combo path model lockout must prefer a parsed reset over the base +// cooldown ladder while still respecting the operator's max for body prose. import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; @@ -35,7 +34,7 @@ test.after(() => { } catch {} }); -test("combo 429 lockout honors parsed upstream quota reset over base cooldown (#6863)", async () => { +test("combo 429 body reset beats base cooldown but is capped by maxCooldownMs (#6863)", async () => { const provider = "antigravity"; // OAuth category → quota signals preserved on 429 const model = "claude-sonnet-4.6"; @@ -77,12 +76,12 @@ test("combo 429 lockout honors parsed upstream quota reset over base cooldown (# const info = getModelLockoutInfo(provider, "", model); assert.ok(info, "combo 429 must record a model lockout"); - // Bug #6863: lockout was baseCooldownMs (~seconds) while upstream said 92.5h. - // The lockout must equal the parsed reset minus elapsed test runtime (bounded slack), - // so a hardcoded long cooldown (e.g. a fixed 1h) cannot pass. + // Preserve #6863 (do not fall back to ~seconds), but prose is not an + // authoritative reset and must not bypass the operator's 30m maximum. assert.ok( - info!.remainingMs > parsedResetMs! - 5_000 && info!.remainingMs <= parsedResetMs!, - `lockout must equal the parsed upstream reset (~${parsedResetMs}ms); got ${info!.remainingMs}ms (~${Math.round(info!.remainingMs / 1000)}s)` + info!.remainingMs > settings.modelLockout.maxCooldownMs - 5_000 && + info!.remainingMs <= settings.modelLockout.maxCooldownMs, + `body reset must clamp to maxCooldownMs (${settings.modelLockout.maxCooldownMs}ms); got ${info!.remainingMs}ms` ); }); diff --git a/tests/unit/combo-model-lockout-honors-reset-1308.test.ts b/tests/unit/combo-model-lockout-honors-reset-1308.test.ts index 7681faab85..c75d66ccb3 100644 --- a/tests/unit/combo-model-lockout-honors-reset-1308.test.ts +++ b/tests/unit/combo-model-lockout-honors-reset-1308.test.ts @@ -7,8 +7,14 @@ import { getModelLockoutInfo, clearAllModelLockouts, parseRetryFromErrorText, + checkFallbackError, + retryHintBypassesMaxCooldownMs, } from "../../open-sse/services/accountFallback.ts"; import { AntigravityExecutor } from "../../open-sse/executors/antigravity.ts"; +import { + parseDetailedRetryHintFromJsonBody, + parseRetryHintFromJsonBody, +} from "../../open-sse/services/retryAfterJson.ts"; // Regression for #1308: a combo model-lockout was capped at the short base cooldown // (~minutes) and discarded the long upstream quota reset that the central parser had @@ -70,3 +76,89 @@ test("antigravity executor parseRetryFromErrorMessage matches plural 'Resets in' const ms = executor.parseRetryFromErrorMessage("Individual quota reached. Resets in 160h27m24s."); assert.ok(ms && ms > 150 * HOUR, `expected ~160h, got ${ms}`); }); + +test("prose reset above max is identified as text and capped", () => { + const maxCooldownMs = 30 * 60_000; + const result = checkFallbackError( + 429, + "Individual quota reached. Resets in 131h.", + 0, + "claude-sonnet-4-6", + "antigravity", + null, + { + baseCooldownMs: 5 * 60_000, + maxCooldownMs, + maxBackoffSteps: 3, + useExponentialBackoff: true, + useUpstreamRetryHints: true, + } + ); + + assert.equal(result.retryHintSource, "body"); + assert.equal(retryHintBypassesMaxCooldownMs(result.retryHintSource), false); +}); + +test("Retry-After remains authoritative for model locks when connection hints are disabled", () => { + const maxCooldownMs = 30 * 60_000; + const result = checkFallbackError( + 429, + "Individual quota reached.", + 0, + "claude-sonnet-4-6", + "antigravity", + new Headers({ "retry-after": String(131 * 60 * 60) }), + { + baseCooldownMs: 5 * 60_000, + maxCooldownMs, + maxBackoffSteps: 3, + useExponentialBackoff: true, + useUpstreamRetryHints: false, + } + ); + + assert.equal(result.retryHintSource, "header"); + assert.equal(result.quotaResetHintMs, 131 * HOUR); + assert.equal(retryHintBypassesMaxCooldownMs(result.retryHintSource), true); +}); + +test("structured RetryInfo remains authoritative when connection hints are disabled", () => { + const maxCooldownMs = 30 * 60_000; + const body = JSON.stringify({ + error: { + message: "Individual quota reached.", + details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "2h" }], + }, + }); + const result = checkFallbackError(429, body, 0, "claude-sonnet-4-6", "antigravity", null, { + baseCooldownMs: 5 * 60_000, + maxCooldownMs, + maxBackoffSteps: 3, + useExponentialBackoff: true, + useUpstreamRetryHints: false, + }); + + assert.equal(result.retryHintSource, "google_rpc_retry_info"); + assert.equal(result.quotaResetHintMs, 2 * HOUR); + assert.equal(retryHintBypassesMaxCooldownMs(result.retryHintSource), true); +}); + +test("detailed JSON parsing preserves provenance without breaking the numeric wrapper", () => { + const genericBody = JSON.stringify({ error: { retry_after_ms: 2 * HOUR } }); + assert.deepEqual(parseDetailedRetryHintFromJsonBody(genericBody, 3 * HOUR), { + retryAfterMs: 2 * HOUR, + provenance: "body", + }); + assert.equal(parseRetryHintFromJsonBody(genericBody, 3 * HOUR), 2 * HOUR); + + const retryInfoBody = JSON.stringify({ + error: { + details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "26s" }], + }, + }); + assert.deepEqual(parseDetailedRetryHintFromJsonBody(retryInfoBody, 10_000), { + retryAfterMs: 26_000, + provenance: "google_rpc_retry_info", + }); + assert.equal(parseRetryHintFromJsonBody(retryInfoBody, 10_000), 26_000); +});