From 1fcd5e4d1f5f73f968b339708f4e8c9c38c54cc6 Mon Sep 17 00:00:00 2001 From: Markus Hartung Date: Sat, 29 Aug 2026 05:32:22 -0300 Subject: [PATCH] fix(db): rate-limit Arena ELO fetch-failure warnings on repeated timeouts (#11500) --- .../fixes/11500-arena-elo-log-dedup.md | 1 + src/lib/arenaEloSync.ts | 30 +++++- tests/unit/arena-elo-sync.test.ts | 93 +++++++++++++++++++ 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixes/11500-arena-elo-log-dedup.md diff --git a/changelog.d/fixes/11500-arena-elo-log-dedup.md b/changelog.d/fixes/11500-arena-elo-log-dedup.md new file mode 100644 index 0000000000..cead9e4b6a --- /dev/null +++ b/changelog.d/fixes/11500-arena-elo-log-dedup.md @@ -0,0 +1 @@ +- fix(db): rate-limit repeated Arena ELO leaderboard fetch-failure warnings instead of logging one per sync attempt (#11500) diff --git a/src/lib/arenaEloSync.ts b/src/lib/arenaEloSync.ts index 427763b4a8..15ed583df6 100644 --- a/src/lib/arenaEloSync.ts +++ b/src/lib/arenaEloSync.ts @@ -212,6 +212,25 @@ export function normalizeModelName(rawName: string): string { // ─── Core: Fetch ───────────────────────────────────────── +/** + * How many consecutive per-category fetch failures to let through to + * `console.warn` before going quiet again. Timeouts against the Arena API + * repeat every sync cycle (see `startPeriodicSync()`), so logging every + * single one turns into log spam within a few hours (#11500). Mirrors the + * once-per-label dedup pattern used by `warnEmptyAutoPoolOnce()` + * (`open-sse/services/autoCombo/virtualFactory.ts`), except here the streak + * resets on the next successful fetch so a genuinely new outage warns again. + */ +const ARENA_ELO_FETCH_WARN_STREAK_INTERVAL = 10; + +/** Consecutive fetch-failure count per leaderboard category, since the last success. */ +const categoryFetchFailureStreak = new Map(); + +/** Test-only: reset the per-category fetch-failure streak dedup state. */ +export function resetArenaEloFetchFailureStreaksForTests(): void { + categoryFetchFailureStreak.clear(); +} + /** * Fetch leaderboards from the Arena AI API for all configured categories. * @@ -244,9 +263,18 @@ export async function fetchArenaLeaderboards(): Promise { `Arena API returned invalid JSON for "${category}" (${text.slice(0, 100)}...)` ); } + // Recovered: let the next failure streak warn from scratch again. + categoryFetchFailureStreak.delete(category); } catch (err) { const message = err instanceof Error ? err.message : String(err); - console.warn(`[ARENA_ELO_SYNC] Failed to fetch "${category}" leaderboard: ${message}`); + const streak = (categoryFetchFailureStreak.get(category) ?? 0) + 1; + categoryFetchFailureStreak.set(category, streak); + if (streak === 1 || streak % ARENA_ELO_FETCH_WARN_STREAK_INTERVAL === 0) { + console.warn( + `[ARENA_ELO_SYNC] Failed to fetch "${category}" leaderboard: ${message}` + + (streak > 1 ? ` (${streak} consecutive failures; further warnings rate-limited)` : "") + ); + } errors.push(message); } }); diff --git a/tests/unit/arena-elo-sync.test.ts b/tests/unit/arena-elo-sync.test.ts index 0fc7b07ae9..d498ff0b5a 100644 --- a/tests/unit/arena-elo-sync.test.ts +++ b/tests/unit/arena-elo-sync.test.ts @@ -37,6 +37,7 @@ const { getArenaEloSyncStatus, initArenaEloSync, stopArenaEloSync, + resetArenaEloFetchFailureStreaksForTests, } = await import("../../src/lib/arenaEloSync.ts"); const { setFeatureFlagOverride, removeFeatureFlagOverride } = await import("../../src/lib/db/featureFlags.ts"); @@ -630,6 +631,98 @@ describe("fetchArenaLeaderboards()", () => { } ); }); + + // #11500 sub-issue (3): repeated consecutive timeouts must not spam one + // console.warn per category per attempt — the per-category fetch-failure + // warning is rate-limited the same way warnEmptyAutoPoolOnce dedupes. + it("rate-limits the per-category fetch-failure warning across many consecutive timeouts", async () => { + resetArenaEloFetchFailureStreaksForTests(); + const originalWarn = console.warn; + const warnCalls: string[] = []; + console.warn = ((...args: unknown[]) => { + warnCalls.push(args.map(String).join(" ")); + }) as typeof console.warn; + + try { + mockFetch(async () => { + throw new Error("The operation was aborted due to timeout"); + }); + + const ATTEMPTS = 25; + for (let i = 0; i < ATTEMPTS; i++) { + await assert.rejects(() => fetchArenaLeaderboards()); + } + + const fetchFailureWarnings = warnCalls.filter((line) => + line.includes('Failed to fetch "text" leaderboard') + ); + + // One category × 25 consecutive-failure attempts would be 25 raw warns — + // the rate limiter must keep the emitted count far below that. + assert.ok( + fetchFailureWarnings.length < ATTEMPTS, + `expected fewer than ${ATTEMPTS} warnings, got ${fetchFailureWarnings.length}` + ); + assert.ok( + fetchFailureWarnings.length <= 5, + `expected the streak-gated warning to stay tightly bounded, got ${fetchFailureWarnings.length}` + ); + assert.ok(fetchFailureWarnings.length >= 1, "the first failure must still be logged"); + } finally { + console.warn = originalWarn; + resetArenaEloFetchFailureStreaksForTests(); + } + }); + + it("resets the fetch-failure streak after a successful fetch so the next outage warns again", async () => { + resetArenaEloFetchFailureStreaksForTests(); + const originalWarn = console.warn; + const warnCalls: string[] = []; + console.warn = ((...args: unknown[]) => { + warnCalls.push(args.map(String).join(" ")); + }) as typeof console.warn; + + try { + mockFetch(async () => { + throw new Error("timeout"); + }); + await assert.rejects(() => fetchArenaLeaderboards()); + await assert.rejects(() => fetchArenaLeaderboards()); + + const textData = makeLeaderboardData( + [makeModelEntry({ model: "recovered-model", score: 1200, votes: 5000, rank: 1 })], + "text" + ); + const codeData = makeLeaderboardData( + [makeModelEntry({ model: "recovered-code", score: 1200, votes: 5000, rank: 1 })], + "code" + ); + mockFetch(async (url: string) => { + if (url.includes("name=text")) return jsonResponse(textData); + if (url.includes("name=code")) return jsonResponse(codeData); + return new Response("Not found", { status: 404 }); + }); + await fetchArenaLeaderboards(); + + mockFetch(async () => { + throw new Error("timeout again"); + }); + warnCalls.length = 0; + await assert.rejects(() => fetchArenaLeaderboards()); + + const fetchFailureWarnings = warnCalls.filter((line) => + line.includes('Failed to fetch "text" leaderboard') + ); + assert.strictEqual( + fetchFailureWarnings.length, + 1, + "streak reset by the success must re-arm the first-failure warning" + ); + } finally { + console.warn = originalWarn; + resetArenaEloFetchFailureStreaksForTests(); + } + }); }); // ═══════════════════════════════════════════════════════════