fix(api): evict stale in-memory rate-limit windows to stop slow heap leak (#4041) (#4957)

Integrated into release/v3.8.36 (fixes #4041)

src/shared/utils/rateLimiter.ts

@@ -87,6 +87,43 @@ const TEST_MEMORY_STORE = new Map<string, number>();
 const FALLBACK_MEMORY_STORE = new Map<string, number>();
 let explicitTestMode = false;
 
+// Minimum store size before we bother sweeping (avoids O(n) cost on tiny stores)
+const EVICTION_THRESHOLD = 50;
+
+/**
+ * Evict all in-memory rate-limit window keys whose window has already ended.
+ *
+ * Key format: `rl:api_key:{id}:{windowSize}:{windowNumber}`
+ * A key expires at epoch-second `(windowNumber + 1) * windowSize`.
+ *
+ * Exported so tests can exercise it directly and so callers can invoke it
+ * with any store (TEST_MEMORY_STORE or FALLBACK_MEMORY_STORE).
+ *
+ * Fixes: #4041 — FALLBACK_MEMORY_STORE accumulated indefinitely → OOM (#4771).
+ */
+export function evictStaleRateLimitWindows(store: Map<string, number>, nowSeconds: number): void {
+  for (const key of store.keys()) {
+    // Format: rl:api_key:{id}:{windowSize}:{windowNumber}
+    // Split only on the last two colons to handle ids that contain colons.
+    const lastColon = key.lastIndexOf(":");
+    if (lastColon === -1) continue;
+    const secondLastColon = key.lastIndexOf(":", lastColon - 1);
+    if (secondLastColon === -1) continue;
+
+    const windowNumber = Number(key.slice(lastColon + 1));
+    const windowSize = Number(key.slice(secondLastColon + 1, lastColon));
+
+    if (!Number.isFinite(windowNumber) || !Number.isFinite(windowSize) || windowSize <= 0) {
+      continue;
+    }
+
+    const windowEnd = (windowNumber + 1) * windowSize;
+    if (windowEnd <= nowSeconds) {
+      store.delete(key);
+    }
+  }
+}
+
 export function setRateLimiterTestMode(enabled: boolean) {
   explicitTestMode = enabled;
   if (enabled) TEST_MEMORY_STORE.clear();
@@ -98,6 +135,12 @@ function checkInMemoryRateLimit(
   rules: RateLimitRule[]
 ): RateLimitResult {
   const now = Math.floor(Date.now() / 1000);
+
+  // Opportunistic eviction: sweep stale windows when the store has grown past
+  // the threshold. Bounded O(n) sweep — no timer, no background work.
+  if (store.size > EVICTION_THRESHOLD) {
+    evictStaleRateLimitWindows(store, now);
+  }
   for (const rule of rules) {
     const currentWindow = Math.floor(now / rule.window);
     const windowKey = `rl:api_key:${keyId}:${rule.window}:${currentWindow}`;

tests/unit/rate-limiter-eviction-4041.test.ts

@@ -0,0 +1,116 @@
+/**
+ * Regression test for #4041 / #4771: FALLBACK_MEMORY_STORE accumulates
+ * stale window keys and never evicts them, leading to a slow heap leak
+ * (~500 MB → OOM over ~2 days idle in the self-hosted/no-Redis case).
+ *
+ * The fix adds `evictStaleRateLimitWindows(store, nowSeconds)` which
+ * must be exported from rateLimiter.ts.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+// Key format: rl:api_key:{id}:{windowSize}:{windowNumber}
+// A key's window ends at (windowNumber + 1) * windowSize (epoch-seconds).
+// Keys whose window ended in the past must be deleted; the current window must survive.
+
+function makeKey(id: string, windowSize: number, windowNumber: number): string {
+  return `rl:api_key:${id}:${windowSize}:${windowNumber}`;
+}
+
+test("evictStaleRateLimitWindows is exported from rateLimiter", async () => {
+  const { evictStaleRateLimitWindows } = await import("@/shared/utils/rateLimiter.js");
+  assert.equal(
+    typeof evictStaleRateLimitWindows,
+    "function",
+    "evictStaleRateLimitWindows must be exported"
+  );
+});
+
+test("evictStaleRateLimitWindows deletes keys whose window has ended and keeps current keys", async () => {
+  const { evictStaleRateLimitWindows } = await import("@/shared/utils/rateLimiter.js");
+
+  const nowSeconds = 1_000_000; // arbitrary fixed "now"
+  const windowSize = 60; // 60-second window
+  const currentWindow = Math.floor(nowSeconds / windowSize); // window that contains nowSeconds
+  const pastWindow1 = currentWindow - 1; // ended at currentWindow * windowSize — already past
+  const pastWindow2 = currentWindow - 5; // even older
+
+  const store = new Map<string, number>([
+    [makeKey("user-a", windowSize, pastWindow1), 3],
+    [makeKey("user-a", windowSize, pastWindow2), 7],
+    [makeKey("user-b", windowSize, pastWindow1), 1],
+    [makeKey("user-a", windowSize, currentWindow), 2], // LIVE — must survive
+  ]);
+
+  assert.equal(store.size, 4, "should start with 4 keys");
+
+  evictStaleRateLimitWindows(store, nowSeconds);
+
+  // Stale keys must be gone
+  assert.equal(
+    store.has(makeKey("user-a", windowSize, pastWindow1)),
+    false,
+    "past window -1 for user-a must be evicted"
+  );
+  assert.equal(
+    store.has(makeKey("user-a", windowSize, pastWindow2)),
+    false,
+    "past window -5 for user-a must be evicted"
+  );
+  assert.equal(
+    store.has(makeKey("user-b", windowSize, pastWindow1)),
+    false,
+    "past window -1 for user-b must be evicted"
+  );
+
+  // Current key must survive
+  assert.equal(
+    store.has(makeKey("user-a", windowSize, currentWindow)),
+    true,
+    "current window for user-a must survive"
+  );
+  assert.equal(
+    store.get(makeKey("user-a", windowSize, currentWindow)),
+    2,
+    "current window count must be unchanged"
+  );
+
+  assert.equal(store.size, 1, "only 1 key should remain after eviction");
+});
+
+test("evictStaleRateLimitWindows leaves store untouched when all keys are current", async () => {
+  const { evictStaleRateLimitWindows } = await import("@/shared/utils/rateLimiter.js");
+
+  const nowSeconds = 2_000_000;
+  const windowSize = 3600;
+  const currentWindow = Math.floor(nowSeconds / windowSize);
+
+  const store = new Map<string, number>([
+    [makeKey("x", windowSize, currentWindow), 5],
+    [makeKey("y", windowSize, currentWindow), 9],
+  ]);
+
+  evictStaleRateLimitWindows(store, nowSeconds);
+
+  assert.equal(store.size, 2, "no keys should be evicted when all are current");
+});
+
+test("evictStaleRateLimitWindows is a no-op on an empty store", async () => {
+  const { evictStaleRateLimitWindows } = await import("@/shared/utils/rateLimiter.js");
+
+  const store = new Map<string, number>();
+  evictStaleRateLimitWindows(store, 1_000_000);
+  assert.equal(store.size, 0);
+});
+
+test("evictStaleRateLimitWindows ignores keys with unexpected formats (does not throw)", async () => {
+  const { evictStaleRateLimitWindows } = await import("@/shared/utils/rateLimiter.js");
+
+  const store = new Map<string, number>([
+    ["not-a-rate-limit-key", 1],
+    ["rl:api_key:only-four-segments", 2],
+  ]);
+
+  // Must not throw
+  assert.doesNotThrow(() => evictStaleRateLimitWindows(store, 1_000_000));
+});