fix: honor PROVIDER_LIMITS_SYNC_SPACING_MS for local/API-key connections (#6916) (#7214)

changelog.d/fixes/6916-provider-limits-spacing-local.md

@@ -0,0 +1 @@
+- fix(providers): `PROVIDER_LIMITS_SYNC_SPACING_MS` now also throttles local / API-key (Ollama) connections, not just OAuth — spaced between concurrency chunks so a local endpoint isn't hit by a simultaneous refresh burst (#6916)

src/lib/usage/providerLimits.ts

@@ -35,6 +35,7 @@ import {
   normalizeUsageQuotasForProvider,
   sanitizeUsageQuotasForProvider,
 } from "./providerLimits/quotaNormalize";
+import { syncInChunksWithSpacing } from "./providerLimits/chunkedSpacingSync";
 
 type JsonRecord = Record<string, unknown>;
 type SyncSource = "manual" | "scheduled";
@@ -616,15 +617,18 @@ export function getProviderLimitsSyncIntervalMs(): number {
 const DEFAULT_PROVIDER_LIMITS_SYNC_SPACING_MS = 1500;
 
 /**
- * Spacing (ms) between consecutive OAuth provider-limits fetches in a bulk sync.
+ * Spacing (ms) applied between consecutive provider-limits fetch batches in a
+ * bulk sync, for BOTH the OAuth and local/API-key paths.
  *
  * OAuth providers (Codex/Claude/Kimi-coding/…) are fetched ONE AT A TIME with
  * this gap so a single host never bursts several simultaneous usage/refresh
  * requests to the same upstream — bursts read as automated traffic and
  * contribute to session termination / anomaly flags (and, for rotating-token
- * providers, to the Auth0 family-revocation race). Stateless API-key providers
- * keep the fast concurrent path. Tunable via `PROVIDER_LIMITS_SYNC_SPACING_MS`;
- * set to `"0"` to opt out.
+ * providers, to the Auth0 family-revocation race). Local/API-key connections
+ * (e.g. Ollama) keep their fast in-chunk concurrent path, but the gap is now
+ * also applied BETWEEN concurrency chunks so a local endpoint isn't hit by a
+ * simultaneous refresh burst either (#6916). Tunable via
+ * `PROVIDER_LIMITS_SYNC_SPACING_MS`; set to `"0"` to opt out on either path.
  */
 export function getProviderLimitsSyncSpacingMs(): number {
   const rawEnv = process.env.PROVIDER_LIMITS_SYNC_SPACING_MS;
@@ -633,8 +637,6 @@ export function getProviderLimitsSyncSpacingMs(): number {
   return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_PROVIDER_LIMITS_SYNC_SPACING_MS;
 }
 
-const syncDelay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
-
 export async function getLastProviderLimitsAutoSyncTime(): Promise<string | null> {
   try {
     const settings = await getSettings();
@@ -955,31 +957,27 @@ export async function syncAllProviderLimits(
     return { connectionId: connection.id, cache };
   };
 
-  // OAuth connections are processed STRICTLY SEQUENTIALLY with a spacing gap so a
-  // single host never bursts simultaneous usage/refresh requests to the same
-  // upstream (anomaly/session-termination guard; see getProviderLimitsSyncSpacingMs).
-  // Stateless API-key connections keep the fast chunked-concurrent path.
+  // OAuth connections are processed STRICTLY SEQUENTIALLY (chunk size 1) with a
+  // spacing gap so a single host never bursts simultaneous usage/refresh
+  // requests to the same upstream (anomaly/session-termination guard; see
+  // getProviderLimitsSyncSpacingMs). Local/API-key connections keep their fast
+  // in-chunk concurrent path, spaced BETWEEN chunks (#6916).
   const oauthConnections = connections.filter((c) => c.authType === "oauth");
   const otherConnections = connections.filter((c) => c.authType !== "oauth");
   const spacingMs = getProviderLimitsSyncSpacingMs();
 
-  for (let i = 0; i < otherConnections.length; i += concurrency) {
-    const chunk = otherConnections.slice(i, i + concurrency);
-    const results = await Promise.allSettled(chunk.map(fetchOne));
+  const recordChunk = (
+    chunk: ProviderConnectionLike[],
+    results: PromiseSettledResult<{ connectionId: string; cache: ProviderLimitsCacheEntry }>[]
+  ) => {
     results.forEach((result, index) => {
       const connectionId = chunk[index]?.id;
       if (connectionId) recordResult(connectionId, result);
     });
-  }
+  };
 
-  for (let i = 0; i < oauthConnections.length; i++) {
-    const connection = oauthConnections[i];
-    const [result] = await Promise.allSettled([fetchOne(connection)]);
-    recordResult(connection.id, result);
-    if (spacingMs > 0 && i < oauthConnections.length - 1) {
-      await syncDelay(spacingMs);
-    }
-  }
+  await syncInChunksWithSpacing(otherConnections, concurrency, spacingMs, fetchOne, recordChunk);
+  await syncInChunksWithSpacing(oauthConnections, 1, spacingMs, fetchOne, recordChunk);
 
   if (cacheEntries.length > 0) {
     setProviderLimitsCacheBatch(cacheEntries);

src/lib/usage/providerLimits/chunkedSpacingSync.ts

@@ -0,0 +1,30 @@
+/**
+ * Pure, DB-free chunked sync helper shared by both the OAuth and non-OAuth
+ * (local/API-key) paths in `syncAllProviderLimits()`.
+ *
+ * Processes `items` in chunks of `chunkSize`, running each chunk's fetchers
+ * concurrently (`Promise.allSettled`) but waiting `spacingMs` between chunks
+ * (never after the last one). `chunkSize=1` reproduces the strictly-sequential
+ * OAuth behavior; `chunkSize=concurrency` reproduces the previous fast
+ * chunked-concurrent behavior for local/API-key connections, now with the
+ * spacing gap applied between chunks so `PROVIDER_LIMITS_SYNC_SPACING_MS` is
+ * honored on both paths (see #6916).
+ */
+export async function syncInChunksWithSpacing<T, R>(
+  items: T[],
+  chunkSize: number,
+  spacingMs: number,
+  fetcher: (item: T) => Promise<R>,
+  onChunkResults: (chunk: T[], results: PromiseSettledResult<R>[]) => void
+): Promise<void> {
+  const size = chunkSize > 0 ? chunkSize : 1;
+  for (let i = 0; i < items.length; i += size) {
+    const chunk = items.slice(i, i + size);
+    const results = await Promise.allSettled(chunk.map(fetcher));
+    onChunkResults(chunk, results);
+    const isLastChunk = i + size >= items.length;
+    if (spacingMs > 0 && !isLastChunk) {
+      await new Promise<void>((resolve) => setTimeout(resolve, spacingMs));
+    }
+  }
+}

tests/unit/provider-limits-chunked-spacing-sync.test.ts

@@ -0,0 +1,108 @@
+/**
+ * Unit tests for the pure `syncInChunksWithSpacing` helper (#6916).
+ *
+ * Proves the chunking/spacing contract in isolation — no DB, no network —
+ * before it is wired into `syncAllProviderLimits()`'s OAuth and non-OAuth
+ * paths.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { syncInChunksWithSpacing } from "../../src/lib/usage/providerLimits/chunkedSpacingSync.ts";
+
+test("waits between chunks but not after the last chunk when spacingMs > 0", async () => {
+  const items = [1, 2, 3, 4];
+  const chunkStarts: number[] = [];
+
+  const start = Date.now();
+  await syncInChunksWithSpacing(
+    items,
+    2,
+    40,
+    async (item) => {
+      chunkStarts.push(Date.now() - start);
+      return item;
+    },
+    () => {}
+  );
+
+  // 2 chunks of 2 → chunkStarts has 4 entries (2 per chunk, same start time).
+  assert.equal(chunkStarts.length, 4);
+  const chunk1Start = Math.min(chunkStarts[0], chunkStarts[1]);
+  const chunk2Start = Math.min(chunkStarts[2], chunkStarts[3]);
+  assert.ok(
+    chunk2Start - chunk1Start >= 35,
+    `expected >=35ms gap between chunks, got ${chunk2Start - chunk1Start}`
+  );
+});
+
+test("never waits when spacingMs === 0 (opt-out, preserves fast path)", async () => {
+  const items = [1, 2, 3, 4];
+  const start = Date.now();
+
+  await syncInChunksWithSpacing(items, 2, 0, async (item) => item, () => {});
+
+  const elapsed = Date.now() - start;
+  assert.ok(elapsed < 30, `expected near-instant run with spacingMs=0, took ${elapsed}ms`);
+});
+
+test("chunkSize=1 processes items strictly one at a time (reproduces OAuth semantics)", async () => {
+  const items = ["a", "b", "c"];
+  const chunks: string[][] = [];
+
+  await syncInChunksWithSpacing(
+    items,
+    1,
+    0,
+    async (item) => item,
+    (chunk) => chunks.push([...chunk])
+  );
+
+  assert.deepEqual(chunks, [["a"], ["b"], ["c"]]);
+});
+
+test("preserves in-chunk concurrency — all items in a chunk start before any resolves", async () => {
+  const items = [1, 2, 3];
+  let inFlight = 0;
+  let maxInFlight = 0;
+
+  await syncInChunksWithSpacing(
+    items,
+    3,
+    0,
+    async (item) => {
+      inFlight++;
+      maxInFlight = Math.max(maxInFlight, inFlight);
+      await new Promise((resolve) => setTimeout(resolve, 20));
+      inFlight--;
+      return item;
+    },
+    () => {}
+  );
+
+  assert.equal(maxInFlight, 3, "all 3 items in the single chunk should overlap");
+});
+
+test("delivers chunk + results to onChunkResults, including rejections", async () => {
+  const items = [1, 2, 3];
+  const seen: Array<{ chunk: number[]; statuses: string[] }> = [];
+
+  await syncInChunksWithSpacing(
+    items,
+    2,
+    0,
+    async (item) => {
+      if (item === 2) throw new Error("boom");
+      return item * 10;
+    },
+    (chunk, results) => {
+      seen.push({ chunk: [...chunk], statuses: results.map((r) => r.status) });
+    }
+  );
+
+  assert.equal(seen.length, 2);
+  assert.deepEqual(seen[0].chunk, [1, 2]);
+  assert.deepEqual(seen[0].statuses, ["fulfilled", "rejected"]);
+  assert.deepEqual(seen[1].chunk, [3]);
+  assert.deepEqual(seen[1].statuses, ["fulfilled"]);
+});

tests/unit/provider-limits-local-apikey-sync-spacing.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Local/API-key provider-limits sync must honor PROVIDER_LIMITS_SYNC_SPACING_MS
+ * too, not just the OAuth path (#6916).
+ *
+ * `syncAllProviderLimits` previously ran non-OAuth (local/API-key, e.g. Ollama)
+ * connections in `concurrency`-sized chunks with NO spacing at all between
+ * chunks, so setting `PROVIDER_LIMITS_SYNC_SPACING_MS` had no effect on that
+ * path. This is the direct regression guard: forces >1 chunk (concurrency=1)
+ * and asserts a measured gap >= spacingMs between chunk start times.
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-apikey-spacing-sync-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.API_KEY_SECRET = "test-apikey-spacing-sync-secret";
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const providerLimits = await import("../../src/lib/usage/providerLimits.ts");
+
+const originalFetch = globalThis.fetch;
+
+test.beforeEach(() => {
+  globalThis.fetch = originalFetch;
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+  delete process.env.PROVIDER_LIMITS_SYNC_SPACING_MS;
+});
+
+test.after(() => {
+  globalThis.fetch = originalFetch;
+  delete process.env.PROVIDER_LIMITS_SYNC_SPACING_MS;
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+async function createGlmApiKeyConnection(i: number) {
+  return providersDb.createProviderConnection({
+    provider: "glm",
+    authType: "apikey",
+    name: `GLM Spacing ${i}`,
+    apiKey: `glm-spacing-key-${i}`,
+  });
+}
+
+function glmQuotaResponse() {
+  return new Response(
+    JSON.stringify({
+      code: 200,
+      success: true,
+      data: {
+        planName: "max",
+        limits: [
+          {
+            type: "TOKENS_LIMIT",
+            unit: 3,
+            number: 5,
+            percentage: 13,
+            nextResetTime: Math.floor(Date.now() / 1000) + 3 * 3600,
+            models: [],
+          },
+        ],
+      },
+    }),
+    { status: 200, headers: { "content-type": "application/json" } }
+  );
+}
+
+test("syncAllProviderLimits spaces chunks for local/API-key connections when spacingMs is set", async () => {
+  process.env.PROVIDER_LIMITS_SYNC_SPACING_MS = "60";
+  for (let i = 0; i < 3; i++) await createGlmApiKeyConnection(i);
+
+  const chunkStarts: number[] = [];
+  const start = Date.now();
+
+  globalThis.fetch = (async () => {
+    chunkStarts.push(Date.now() - start);
+    return glmQuotaResponse();
+  }) as typeof fetch;
+
+  // concurrency: 1 forces 3 chunks of size 1 → 2 gaps must be >= spacingMs.
+  await providerLimits.syncAllProviderLimits({ source: "scheduled", concurrency: 1 });
+
+  assert.equal(chunkStarts.length, 3, "expected 3 fetches, one per connection");
+  const gaps: number[] = [];
+  for (let i = 1; i < chunkStarts.length; i++) gaps.push(chunkStarts[i] - chunkStarts[i - 1]);
+  assert.ok(
+    gaps.every((g) => g >= 50),
+    `every chunk gap must be >= configured spacing (~60ms), gaps=${gaps.join(",")}`
+  );
+});
+
+test("syncAllProviderLimits does not space local/API-key chunks when spacingMs=0 (opt-out)", async () => {
+  process.env.PROVIDER_LIMITS_SYNC_SPACING_MS = "0";
+  for (let i = 0; i < 3; i++) await createGlmApiKeyConnection(i);
+
+  const chunkStarts: number[] = [];
+  const start = Date.now();
+
+  globalThis.fetch = (async () => {
+    chunkStarts.push(Date.now() - start);
+    return glmQuotaResponse();
+  }) as typeof fetch;
+
+  await providerLimits.syncAllProviderLimits({ source: "scheduled", concurrency: 1 });
+
+  assert.equal(chunkStarts.length, 3);
+  const gaps: number[] = [];
+  for (let i = 1; i < chunkStarts.length; i++) gaps.push(chunkStarts[i] - chunkStarts[i - 1]);
+  assert.ok(
+    gaps.every((g) => g < 40),
+    `spacingMs=0 must not introduce a forced gap, gaps=${gaps.join(",")}`
+  );
+});