mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-13 10:43:43 +03:00
fix(antigravity): propagate switchAuth signal from 429 engine to retry guard (#9351)
When Google returns a 429 with no parseable retry hint, decide429 correctly classifies it as short_cooldown_switch_auth (switch accounts). But the executor discarded that decision, keeping only retryMs=60000. The retry guard then slept 60s against the same URL/account up to 3 times because 60000 <= LONG_RETRY_THRESHOLD_MS (inclusive boundary). Plumb a switchAuth boolean through tryResolveRetryFromErrorBody so the retry guard can decline the sleep branch and fall through to URL/account fallback immediately. Signed-off-by: Minxi Hou <houminxi@gmail.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"_rebaseline_2026_08_09_8984_api_key_cache_mode": "PR #8984 own growth during the 2026-08-09 rebase: src/lib/db/apiKeys.ts 1529->1545 (+16 = the per-key apiKeys.cacheDefaultMode column + its row parsers and cascade wiring; additive at the existing connection write/read chokepoints). Covered by tests/unit/chatcore-semantic-cache.test.ts. (chatCore.ts stays at the pre-existing base-red ceiling — upstream tip already exceeds the frozen 5042, this PR only adds +2 on top; not re-bumped per the no-inherit-ratchet rule.)",
|
||||
"_rebaseline_2026_08_09_9207_breaker_halfopen_recovery": "PR #9207 own growth during the 2026-08-09 rebase: open-sse/services/accountFallback.ts 1978->2020 (+42 = recordProviderSuccess now also transitions the provider circuit breaker from HALF_OPEN to CLOSED when a request succeeds, so the breaker is not stuck half-open after repeated failures; the transition and its reset wiring grow the existing provider-success path, not extractable). Covered by tests/unit/provider-breaker-halfopen-recovery.test.ts.",
|
||||
"_rebaseline_2026_08_09_9351_antigravity_switch_auth": "PR #9351 own growth during the 2026-08-09 rebase: open-sse/executors/antigravity.ts 1528->1536 (+8 = switchAuth threaded out of tryResolveRetryFromErrorBody into handleAntigravityRateLimit's short-retry guard, so a decide429 switch decision beats the 60s same-account sleep; cohesive at the existing resolve chokepoint, not extractable). Covered by tests/unit/antigravity-429-switch-auth.test.ts.",
|
||||
"_rebaseline_2026_08_09_9296_adobe_media_capabilities": "PR #9296 (artickc, fix/adobe-firefly-model-capabilities) own growth: src/app/api/v1/models/catalog.ts 1590->1597 (+7). The image and video catalog serializers now expose the already-normalized Adobe Firefly discovery capability data (media_capabilities, plus the existing video modality/size fields) at their only response-emission chokepoints. The discovery parser and capability normalization remain in open-sse/services/adobeFireflyModels.ts; extracting these seven serialization fields would obscure the catalog contract. Covered by tests/unit/adobe-firefly.test.ts and tests/unit/image-upscale.test.ts.",
|
||||
"_rebaseline_2026_08_08_v3850_base_drift_batch_9757": "Base drift on release/v3.8.50, not own growth: the 08-06..08-08 merge batches grew 12 already-frozen (or newly-landed) files without carrying their rebaselines \u2014 the dedicated rebaseline PR #9616 was closed as 'superseded' but its file-size entries never actually reached the base, and later merges (#8894 combos page, #9539 EditConnectionModal, #8895 models route, #9294/#9293 catalog, #9541 db/core, #8970 tokenHealthCheck, #8925 mcp schemas+server, #8890 accountFallback, #9467 chat.ts, #8931 openai-to-kiro, ProxyRegistryManager) kept growing them. All 12 values re-measured on THIS branch's tree (= pure tip + this PR's 1-line chat.ts fix, which adds zero lines). This PR's own source changes (chat.ts identifier restore, stream.ts format carve-out) do not grow any frozen file past these values.",
|
||||
"_rebaseline_2026_08_08_migration_135_collision": "fix(db): resolve migration version 135 numbering collision \u2014 #9449's 135_connection_runtime_state.sql and #8908's 135_migrate_model_capability_max_token.sql both claimed version 135 (#9449 branched before #8908 merged and never got renumbered before landing on release/v3.8.50), which threw 'Migration version collision detected' the moment ANY code touched the database \u2014 a fresh install/deploy from this tip cannot even boot. Renumbered the later-landing file to 140 (next free slot) and added the matching isSchemaAlreadyApplied('140') retroactive guard, matching the established pattern already used for the prior 135/136 -> 137/138 renumber in the same file. Own growth: src/lib/db/migrationRunner.ts 1084->1094 (+10, the new case block) \u2014 irreducible, matches the existing per-case guard pattern exactly. Covered by tests/unit/migration-135-numbering-collision.test.ts (2/2), confirmed failing (reproducing the exact live crash) against the pre-fix colliding filenames, passing after.",
|
||||
|
||||
@@ -1342,7 +1342,7 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
* the last url with no more retries left) fall through with the resolved retryMs
|
||||
* so the caller can still embed a long Retry-After in the final response body.
|
||||
*/
|
||||
private async handleAntigravityRateLimit(
|
||||
async handleAntigravityRateLimit(
|
||||
ctx: AntigravityRateLimitContext
|
||||
): Promise<AntigravityRateLimitOutcome> {
|
||||
const { response, log, urlIndex, retryAttemptsByUrl, fallbackCount } = ctx;
|
||||
@@ -1351,10 +1351,12 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
let retryMs: number | null = this.parseRetryHeaders(response.headers);
|
||||
|
||||
// If no retry time in headers, try to parse from error message body
|
||||
let switchAuth = false;
|
||||
if (!retryMs) {
|
||||
const resolved = await this.tryResolveRetryFromErrorBody(ctx);
|
||||
if (resolved.kind === "return") return { action: "return", result: resolved.result };
|
||||
retryMs = resolved.retryMs;
|
||||
switchAuth = resolved.switchAuth;
|
||||
}
|
||||
|
||||
// Bounded short-retry: a non-null retryAfterMs ≤ 60s covers nearly every
|
||||
@@ -1365,6 +1367,7 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
if (
|
||||
retryMs &&
|
||||
retryMs <= LONG_RETRY_THRESHOLD_MS &&
|
||||
!switchAuth &&
|
||||
retryAttemptsByUrl[urlIndex] < MAX_AUTO_RETRIES
|
||||
) {
|
||||
retryAttemptsByUrl[urlIndex]++;
|
||||
@@ -1420,7 +1423,8 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
private async tryResolveRetryFromErrorBody(
|
||||
ctx: AntigravityRateLimitContext
|
||||
): Promise<
|
||||
{ kind: "return"; result: SsePassthroughResult } | { kind: "resolved"; retryMs: number | null }
|
||||
| { kind: "return"; result: SsePassthroughResult }
|
||||
| { kind: "resolved"; retryMs: number | null; switchAuth: boolean }
|
||||
> {
|
||||
const {
|
||||
response,
|
||||
@@ -1490,13 +1494,17 @@ export class AntigravityExecutor extends BaseExecutor {
|
||||
if (retryMs) markConnectionQuotaExhausted(accountId, retryMs);
|
||||
}
|
||||
|
||||
return { kind: "resolved", retryMs };
|
||||
return {
|
||||
kind: "resolved",
|
||||
retryMs,
|
||||
switchAuth: decision.kind === "short_cooldown_switch_auth",
|
||||
};
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isAbortError(error)) {
|
||||
throw signal?.reason ?? error;
|
||||
}
|
||||
// Ignore parse errors, will fall back to exponential backoff
|
||||
return { kind: "resolved", retryMs: null };
|
||||
return { kind: "resolved", retryMs: null, switchAuth: false };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
169
tests/unit/antigravity-429-switch-auth.test.ts
Normal file
169
tests/unit/antigravity-429-switch-auth.test.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Regression tests for the switchAuth signal propagation fix.
|
||||
*
|
||||
* When Google returns a 429 with no parseable retry hint, decide429 classifies
|
||||
* it as short_cooldown_switch_auth. The executor must NOT sleep on the same
|
||||
* URL/account -- it should fall through to URL/account fallback immediately.
|
||||
*
|
||||
* Row table from WO3 section 2.1:
|
||||
* parsed hint decide429 kind retryMs guard behavior
|
||||
* none (null) short_cooldown_switch_auth 60_000 default skip sleep (THE FIX)
|
||||
* "reset after 0s" soft_retry 2_000 floor sleep 2s
|
||||
* <= 60s soft_retry as parsed sleep that
|
||||
* 60s .. 5min soft_retry as parsed over threshold, skip
|
||||
* > 5min short_cooldown_switch_auth as parsed over threshold, skip
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { classify429, decide429 } from "../../open-sse/services/antigravity429Engine.ts";
|
||||
import { AntigravityExecutor } from "../../open-sse/executors/antigravity.ts";
|
||||
|
||||
// -- Helpers -----------------------------------------------------------------
|
||||
|
||||
function noopLog() {
|
||||
return { debug() {}, info() {}, warn() {}, error() {} };
|
||||
}
|
||||
|
||||
function make429Response(body: unknown): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 429,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
// Rate_limited body: "Too many requests" triggers classify429 -> rate_limited
|
||||
const RATE_LIMITED_BODY = {
|
||||
error: {
|
||||
code: 429,
|
||||
message: "Too many requests",
|
||||
status: "RESOURCE_EXHAUSTED",
|
||||
},
|
||||
};
|
||||
|
||||
function makeCtx(response: Response) {
|
||||
return {
|
||||
response,
|
||||
log: noopLog(),
|
||||
urlIndex: 0,
|
||||
retryAttemptsByUrl: { 0: 0 } as Record<number, number>,
|
||||
fallbackCount: 2,
|
||||
url: "https://example.com/test",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
transformedBody: {},
|
||||
credentials: { accessToken: "test-token" },
|
||||
stream: true,
|
||||
signal: undefined as AbortSignal | undefined,
|
||||
finalHeaders: { "Content-Type": "application/json" },
|
||||
accountId: "test-account",
|
||||
creditsMode: "off" as const,
|
||||
creditsRetryState: { attempted: false },
|
||||
};
|
||||
}
|
||||
|
||||
// -- Engine contract: decide429 returns correct kind -------------------------
|
||||
|
||||
test("Row 1: rate_limited with no hint -> short_cooldown_switch_auth, retryAfterMs=60000", () => {
|
||||
const decision = decide429("rate_limited", null);
|
||||
assert.equal(decision.kind, "short_cooldown_switch_auth");
|
||||
assert.equal(decision.retryAfterMs, 60_000);
|
||||
});
|
||||
|
||||
test("Row 2: rate_limited with 0s hint -> soft_retry (floor at 2000ms)", () => {
|
||||
const decision = decide429("rate_limited", 2000);
|
||||
assert.equal(decision.kind, "soft_retry");
|
||||
assert.equal(decision.retryAfterMs, 2000);
|
||||
});
|
||||
|
||||
test("Row 3: rate_limited with <=60s hint -> soft_retry", () => {
|
||||
const decision = decide429("rate_limited", 30_000);
|
||||
assert.equal(decision.kind, "soft_retry");
|
||||
assert.equal(decision.retryAfterMs, 30_000);
|
||||
});
|
||||
|
||||
test("Row 4: rate_limited with 60s..5min hint -> soft_retry", () => {
|
||||
const decision = decide429("rate_limited", 240_000);
|
||||
assert.equal(decision.kind, "soft_retry");
|
||||
assert.equal(decision.retryAfterMs, 240_000);
|
||||
});
|
||||
|
||||
test("Row 5: rate_limited with >5min hint -> short_cooldown_switch_auth", () => {
|
||||
const decision = decide429("rate_limited", 360_000);
|
||||
assert.equal(decision.kind, "short_cooldown_switch_auth");
|
||||
assert.equal(decision.retryAfterMs, 360_000);
|
||||
});
|
||||
|
||||
// -- THE FIX: real production path via handleAntigravityRateLimit ------------
|
||||
|
||||
test("THE FIX: 429 rate_limited with no hint -> no sleep, falls through to fallback", async () => {
|
||||
const executor = new AntigravityExecutor();
|
||||
const response = make429Response(RATE_LIMITED_BODY);
|
||||
const ctx = makeCtx(response);
|
||||
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
let setTimeoutCalled = false;
|
||||
// @ts-expect-error -- mock replacement for spy
|
||||
globalThis.setTimeout = (...args: unknown[]) => {
|
||||
setTimeoutCalled = true;
|
||||
return originalSetTimeout(...(args as [() => void, number]));
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await executor.handleAntigravityRateLimit(ctx);
|
||||
|
||||
assert.equal(setTimeoutCalled, false, "must not sleep when switchAuth=true");
|
||||
assert.equal(result.action, "retryNextUrl", "must fall through to next URL");
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
test("Regression: 429 with 30s hint -> sleeps and retries same URL", async () => {
|
||||
const executor = new AntigravityExecutor();
|
||||
const response = make429Response({
|
||||
error: {
|
||||
code: 429,
|
||||
message: "Too many requests. Resets after 30s",
|
||||
status: "RESOURCE_EXHAUSTED",
|
||||
},
|
||||
});
|
||||
const ctx = makeCtx(response);
|
||||
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
let sleepMs = 0;
|
||||
let setTimeoutCallCount = 0;
|
||||
// @ts-expect-error -- mock replacement for spy
|
||||
globalThis.setTimeout = (fn: () => void, ms?: number) => {
|
||||
setTimeoutCallCount++;
|
||||
sleepMs = ms ?? 0;
|
||||
return originalSetTimeout(fn, 0); // resolve immediately for test
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await executor.handleAntigravityRateLimit(ctx);
|
||||
|
||||
assert.ok(setTimeoutCallCount > 0, `setTimeout must be called (was ${setTimeoutCallCount})`);
|
||||
assert.ok(sleepMs > 0, `must sleep for parsed retry hint (sleepMs=${sleepMs})`);
|
||||
assert.equal(result.action, "retrySameUrl", "must retry same URL");
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
// -- classify429: rate_limited detection -------------------------------------
|
||||
|
||||
test("classify429: 'queries per minute limit was reached' -> rate_limited", () => {
|
||||
assert.equal(
|
||||
classify429("RESOURCE_EXHAUSTED: queries per minute limit was reached"),
|
||||
"rate_limited"
|
||||
);
|
||||
});
|
||||
|
||||
test("classify429: 'too many requests' -> rate_limited", () => {
|
||||
assert.equal(classify429("Too many requests"), "rate_limited");
|
||||
});
|
||||
|
||||
test("classify429: 'RPM' -> rate_limited", () => {
|
||||
assert.equal(classify429("RPM limit exceeded"), "rate_limited");
|
||||
});
|
||||
Reference in New Issue
Block a user