fix(db): rate-limit Arena ELO fetch-failure warnings on repeated timeouts (#11500)

changelog.d/fixes/11500-arena-elo-log-dedup.md

@@ -0,0 +1 @@
+- fix(db): rate-limit repeated Arena ELO leaderboard fetch-failure warnings instead of logging one per sync attempt (#11500)

src/lib/arenaEloSync.ts

@@ -212,6 +212,25 @@ export function normalizeModelName(rawName: string): string {
 
 // ─── Core: Fetch ─────────────────────────────────────────
 
+/**
+ * How many consecutive per-category fetch failures to let through to
+ * `console.warn` before going quiet again. Timeouts against the Arena API
+ * repeat every sync cycle (see `startPeriodicSync()`), so logging every
+ * single one turns into log spam within a few hours (#11500). Mirrors the
+ * once-per-label dedup pattern used by `warnEmptyAutoPoolOnce()`
+ * (`open-sse/services/autoCombo/virtualFactory.ts`), except here the streak
+ * resets on the next successful fetch so a genuinely new outage warns again.
+ */
+const ARENA_ELO_FETCH_WARN_STREAK_INTERVAL = 10;
+
+/** Consecutive fetch-failure count per leaderboard category, since the last success. */
+const categoryFetchFailureStreak = new Map<string, number>();
+
+/** Test-only: reset the per-category fetch-failure streak dedup state. */
+export function resetArenaEloFetchFailureStreaksForTests(): void {
+  categoryFetchFailureStreak.clear();
+}
+
 /**
  * Fetch leaderboards from the Arena AI API for all configured categories.
  *
@@ -244,9 +263,18 @@ export async function fetchArenaLeaderboards(): Promise<ArenaLeaderboardMap> {
           `Arena API returned invalid JSON for "${category}" (${text.slice(0, 100)}...)`
         );
       }
+      // Recovered: let the next failure streak warn from scratch again.
+      categoryFetchFailureStreak.delete(category);
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
-      console.warn(`[ARENA_ELO_SYNC] Failed to fetch "${category}" leaderboard: ${message}`);
+      const streak = (categoryFetchFailureStreak.get(category) ?? 0) + 1;
+      categoryFetchFailureStreak.set(category, streak);
+      if (streak === 1 || streak % ARENA_ELO_FETCH_WARN_STREAK_INTERVAL === 0) {
+        console.warn(
+          `[ARENA_ELO_SYNC] Failed to fetch "${category}" leaderboard: ${message}` +
+            (streak > 1 ? ` (${streak} consecutive failures; further warnings rate-limited)` : "")
+        );
+      }
       errors.push(message);
     }
   });

tests/unit/arena-elo-sync.test.ts

@@ -37,6 +37,7 @@ const {
   getArenaEloSyncStatus,
   initArenaEloSync,
   stopArenaEloSync,
+  resetArenaEloFetchFailureStreaksForTests,
 } = await import("../../src/lib/arenaEloSync.ts");
 const { setFeatureFlagOverride, removeFeatureFlagOverride } =
   await import("../../src/lib/db/featureFlags.ts");
@@ -630,6 +631,98 @@ describe("fetchArenaLeaderboards()", () => {
       }
     );
   });
+
+  // #11500 sub-issue (3): repeated consecutive timeouts must not spam one
+  // console.warn per category per attempt — the per-category fetch-failure
+  // warning is rate-limited the same way warnEmptyAutoPoolOnce dedupes.
+  it("rate-limits the per-category fetch-failure warning across many consecutive timeouts", async () => {
+    resetArenaEloFetchFailureStreaksForTests();
+    const originalWarn = console.warn;
+    const warnCalls: string[] = [];
+    console.warn = ((...args: unknown[]) => {
+      warnCalls.push(args.map(String).join(" "));
+    }) as typeof console.warn;
+
+    try {
+      mockFetch(async () => {
+        throw new Error("The operation was aborted due to timeout");
+      });
+
+      const ATTEMPTS = 25;
+      for (let i = 0; i < ATTEMPTS; i++) {
+        await assert.rejects(() => fetchArenaLeaderboards());
+      }
+
+      const fetchFailureWarnings = warnCalls.filter((line) =>
+        line.includes('Failed to fetch "text" leaderboard')
+      );
+
+      // One category × 25 consecutive-failure attempts would be 25 raw warns —
+      // the rate limiter must keep the emitted count far below that.
+      assert.ok(
+        fetchFailureWarnings.length < ATTEMPTS,
+        `expected fewer than ${ATTEMPTS} warnings, got ${fetchFailureWarnings.length}`
+      );
+      assert.ok(
+        fetchFailureWarnings.length <= 5,
+        `expected the streak-gated warning to stay tightly bounded, got ${fetchFailureWarnings.length}`
+      );
+      assert.ok(fetchFailureWarnings.length >= 1, "the first failure must still be logged");
+    } finally {
+      console.warn = originalWarn;
+      resetArenaEloFetchFailureStreaksForTests();
+    }
+  });
+
+  it("resets the fetch-failure streak after a successful fetch so the next outage warns again", async () => {
+    resetArenaEloFetchFailureStreaksForTests();
+    const originalWarn = console.warn;
+    const warnCalls: string[] = [];
+    console.warn = ((...args: unknown[]) => {
+      warnCalls.push(args.map(String).join(" "));
+    }) as typeof console.warn;
+
+    try {
+      mockFetch(async () => {
+        throw new Error("timeout");
+      });
+      await assert.rejects(() => fetchArenaLeaderboards());
+      await assert.rejects(() => fetchArenaLeaderboards());
+
+      const textData = makeLeaderboardData(
+        [makeModelEntry({ model: "recovered-model", score: 1200, votes: 5000, rank: 1 })],
+        "text"
+      );
+      const codeData = makeLeaderboardData(
+        [makeModelEntry({ model: "recovered-code", score: 1200, votes: 5000, rank: 1 })],
+        "code"
+      );
+      mockFetch(async (url: string) => {
+        if (url.includes("name=text")) return jsonResponse(textData);
+        if (url.includes("name=code")) return jsonResponse(codeData);
+        return new Response("Not found", { status: 404 });
+      });
+      await fetchArenaLeaderboards();
+
+      mockFetch(async () => {
+        throw new Error("timeout again");
+      });
+      warnCalls.length = 0;
+      await assert.rejects(() => fetchArenaLeaderboards());
+
+      const fetchFailureWarnings = warnCalls.filter((line) =>
+        line.includes('Failed to fetch "text" leaderboard')
+      );
+      assert.strictEqual(
+        fetchFailureWarnings.length,
+        1,
+        "streak reset by the success must re-arm the first-failure warning"
+      );
+    } finally {
+      console.warn = originalWarn;
+      resetArenaEloFetchFailureStreaksForTests();
+    }
+  });
 });
 
 // ═══════════════════════════════════════════════════════════