From a2df195d5e4f0855672298279f4c88d4034f36fa Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:39:54 -0300 Subject: [PATCH] fix: honor PROVIDER_LIMITS_SYNC_SPACING_MS for local/API-key connections (#6916) (#7214) --- .../6916-provider-limits-spacing-local.md | 1 + src/lib/usage/providerLimits.ts | 42 +++---- .../providerLimits/chunkedSpacingSync.ts | 30 +++++ ...ovider-limits-chunked-spacing-sync.test.ts | 108 ++++++++++++++++ ...r-limits-local-apikey-sync-spacing.test.ts | 118 ++++++++++++++++++ 5 files changed, 277 insertions(+), 22 deletions(-) create mode 100644 changelog.d/fixes/6916-provider-limits-spacing-local.md create mode 100644 src/lib/usage/providerLimits/chunkedSpacingSync.ts create mode 100644 tests/unit/provider-limits-chunked-spacing-sync.test.ts create mode 100644 tests/unit/provider-limits-local-apikey-sync-spacing.test.ts diff --git a/changelog.d/fixes/6916-provider-limits-spacing-local.md b/changelog.d/fixes/6916-provider-limits-spacing-local.md new file mode 100644 index 0000000000..2bfb988ee4 --- /dev/null +++ b/changelog.d/fixes/6916-provider-limits-spacing-local.md @@ -0,0 +1 @@ +- fix(providers): `PROVIDER_LIMITS_SYNC_SPACING_MS` now also throttles local / API-key (Ollama) connections, not just OAuth — spaced between concurrency chunks so a local endpoint isn't hit by a simultaneous refresh burst (#6916) diff --git a/src/lib/usage/providerLimits.ts b/src/lib/usage/providerLimits.ts index 6dcafb051f..f8ebe5d7e3 100644 --- a/src/lib/usage/providerLimits.ts +++ b/src/lib/usage/providerLimits.ts @@ -35,6 +35,7 @@ import { normalizeUsageQuotasForProvider, sanitizeUsageQuotasForProvider, } from "./providerLimits/quotaNormalize"; +import { syncInChunksWithSpacing } from "./providerLimits/chunkedSpacingSync"; type JsonRecord = Record; type SyncSource = "manual" | "scheduled"; @@ -616,15 +617,18 @@ export function getProviderLimitsSyncIntervalMs(): number { const DEFAULT_PROVIDER_LIMITS_SYNC_SPACING_MS = 1500; /** - * Spacing (ms) between consecutive OAuth provider-limits fetches in a bulk sync. + * Spacing (ms) applied between consecutive provider-limits fetch batches in a + * bulk sync, for BOTH the OAuth and local/API-key paths. * * OAuth providers (Codex/Claude/Kimi-coding/…) are fetched ONE AT A TIME with * this gap so a single host never bursts several simultaneous usage/refresh * requests to the same upstream — bursts read as automated traffic and * contribute to session termination / anomaly flags (and, for rotating-token - * providers, to the Auth0 family-revocation race). Stateless API-key providers - * keep the fast concurrent path. Tunable via `PROVIDER_LIMITS_SYNC_SPACING_MS`; - * set to `"0"` to opt out. + * providers, to the Auth0 family-revocation race). Local/API-key connections + * (e.g. Ollama) keep their fast in-chunk concurrent path, but the gap is now + * also applied BETWEEN concurrency chunks so a local endpoint isn't hit by a + * simultaneous refresh burst either (#6916). Tunable via + * `PROVIDER_LIMITS_SYNC_SPACING_MS`; set to `"0"` to opt out on either path. */ export function getProviderLimitsSyncSpacingMs(): number { const rawEnv = process.env.PROVIDER_LIMITS_SYNC_SPACING_MS; @@ -633,8 +637,6 @@ export function getProviderLimitsSyncSpacingMs(): number { return Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_PROVIDER_LIMITS_SYNC_SPACING_MS; } -const syncDelay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - export async function getLastProviderLimitsAutoSyncTime(): Promise { try { const settings = await getSettings(); @@ -955,31 +957,27 @@ export async function syncAllProviderLimits( return { connectionId: connection.id, cache }; }; - // OAuth connections are processed STRICTLY SEQUENTIALLY with a spacing gap so a - // single host never bursts simultaneous usage/refresh requests to the same - // upstream (anomaly/session-termination guard; see getProviderLimitsSyncSpacingMs). - // Stateless API-key connections keep the fast chunked-concurrent path. + // OAuth connections are processed STRICTLY SEQUENTIALLY (chunk size 1) with a + // spacing gap so a single host never bursts simultaneous usage/refresh + // requests to the same upstream (anomaly/session-termination guard; see + // getProviderLimitsSyncSpacingMs). Local/API-key connections keep their fast + // in-chunk concurrent path, spaced BETWEEN chunks (#6916). const oauthConnections = connections.filter((c) => c.authType === "oauth"); const otherConnections = connections.filter((c) => c.authType !== "oauth"); const spacingMs = getProviderLimitsSyncSpacingMs(); - for (let i = 0; i < otherConnections.length; i += concurrency) { - const chunk = otherConnections.slice(i, i + concurrency); - const results = await Promise.allSettled(chunk.map(fetchOne)); + const recordChunk = ( + chunk: ProviderConnectionLike[], + results: PromiseSettledResult<{ connectionId: string; cache: ProviderLimitsCacheEntry }>[] + ) => { results.forEach((result, index) => { const connectionId = chunk[index]?.id; if (connectionId) recordResult(connectionId, result); }); - } + }; - for (let i = 0; i < oauthConnections.length; i++) { - const connection = oauthConnections[i]; - const [result] = await Promise.allSettled([fetchOne(connection)]); - recordResult(connection.id, result); - if (spacingMs > 0 && i < oauthConnections.length - 1) { - await syncDelay(spacingMs); - } - } + await syncInChunksWithSpacing(otherConnections, concurrency, spacingMs, fetchOne, recordChunk); + await syncInChunksWithSpacing(oauthConnections, 1, spacingMs, fetchOne, recordChunk); if (cacheEntries.length > 0) { setProviderLimitsCacheBatch(cacheEntries); diff --git a/src/lib/usage/providerLimits/chunkedSpacingSync.ts b/src/lib/usage/providerLimits/chunkedSpacingSync.ts new file mode 100644 index 0000000000..ad3374d557 --- /dev/null +++ b/src/lib/usage/providerLimits/chunkedSpacingSync.ts @@ -0,0 +1,30 @@ +/** + * Pure, DB-free chunked sync helper shared by both the OAuth and non-OAuth + * (local/API-key) paths in `syncAllProviderLimits()`. + * + * Processes `items` in chunks of `chunkSize`, running each chunk's fetchers + * concurrently (`Promise.allSettled`) but waiting `spacingMs` between chunks + * (never after the last one). `chunkSize=1` reproduces the strictly-sequential + * OAuth behavior; `chunkSize=concurrency` reproduces the previous fast + * chunked-concurrent behavior for local/API-key connections, now with the + * spacing gap applied between chunks so `PROVIDER_LIMITS_SYNC_SPACING_MS` is + * honored on both paths (see #6916). + */ +export async function syncInChunksWithSpacing( + items: T[], + chunkSize: number, + spacingMs: number, + fetcher: (item: T) => Promise, + onChunkResults: (chunk: T[], results: PromiseSettledResult[]) => void +): Promise { + const size = chunkSize > 0 ? chunkSize : 1; + for (let i = 0; i < items.length; i += size) { + const chunk = items.slice(i, i + size); + const results = await Promise.allSettled(chunk.map(fetcher)); + onChunkResults(chunk, results); + const isLastChunk = i + size >= items.length; + if (spacingMs > 0 && !isLastChunk) { + await new Promise((resolve) => setTimeout(resolve, spacingMs)); + } + } +} diff --git a/tests/unit/provider-limits-chunked-spacing-sync.test.ts b/tests/unit/provider-limits-chunked-spacing-sync.test.ts new file mode 100644 index 0000000000..f61b410114 --- /dev/null +++ b/tests/unit/provider-limits-chunked-spacing-sync.test.ts @@ -0,0 +1,108 @@ +/** + * Unit tests for the pure `syncInChunksWithSpacing` helper (#6916). + * + * Proves the chunking/spacing contract in isolation — no DB, no network — + * before it is wired into `syncAllProviderLimits()`'s OAuth and non-OAuth + * paths. + */ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { syncInChunksWithSpacing } from "../../src/lib/usage/providerLimits/chunkedSpacingSync.ts"; + +test("waits between chunks but not after the last chunk when spacingMs > 0", async () => { + const items = [1, 2, 3, 4]; + const chunkStarts: number[] = []; + + const start = Date.now(); + await syncInChunksWithSpacing( + items, + 2, + 40, + async (item) => { + chunkStarts.push(Date.now() - start); + return item; + }, + () => {} + ); + + // 2 chunks of 2 → chunkStarts has 4 entries (2 per chunk, same start time). + assert.equal(chunkStarts.length, 4); + const chunk1Start = Math.min(chunkStarts[0], chunkStarts[1]); + const chunk2Start = Math.min(chunkStarts[2], chunkStarts[3]); + assert.ok( + chunk2Start - chunk1Start >= 35, + `expected >=35ms gap between chunks, got ${chunk2Start - chunk1Start}` + ); +}); + +test("never waits when spacingMs === 0 (opt-out, preserves fast path)", async () => { + const items = [1, 2, 3, 4]; + const start = Date.now(); + + await syncInChunksWithSpacing(items, 2, 0, async (item) => item, () => {}); + + const elapsed = Date.now() - start; + assert.ok(elapsed < 30, `expected near-instant run with spacingMs=0, took ${elapsed}ms`); +}); + +test("chunkSize=1 processes items strictly one at a time (reproduces OAuth semantics)", async () => { + const items = ["a", "b", "c"]; + const chunks: string[][] = []; + + await syncInChunksWithSpacing( + items, + 1, + 0, + async (item) => item, + (chunk) => chunks.push([...chunk]) + ); + + assert.deepEqual(chunks, [["a"], ["b"], ["c"]]); +}); + +test("preserves in-chunk concurrency — all items in a chunk start before any resolves", async () => { + const items = [1, 2, 3]; + let inFlight = 0; + let maxInFlight = 0; + + await syncInChunksWithSpacing( + items, + 3, + 0, + async (item) => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 20)); + inFlight--; + return item; + }, + () => {} + ); + + assert.equal(maxInFlight, 3, "all 3 items in the single chunk should overlap"); +}); + +test("delivers chunk + results to onChunkResults, including rejections", async () => { + const items = [1, 2, 3]; + const seen: Array<{ chunk: number[]; statuses: string[] }> = []; + + await syncInChunksWithSpacing( + items, + 2, + 0, + async (item) => { + if (item === 2) throw new Error("boom"); + return item * 10; + }, + (chunk, results) => { + seen.push({ chunk: [...chunk], statuses: results.map((r) => r.status) }); + } + ); + + assert.equal(seen.length, 2); + assert.deepEqual(seen[0].chunk, [1, 2]); + assert.deepEqual(seen[0].statuses, ["fulfilled", "rejected"]); + assert.deepEqual(seen[1].chunk, [3]); + assert.deepEqual(seen[1].statuses, ["fulfilled"]); +}); diff --git a/tests/unit/provider-limits-local-apikey-sync-spacing.test.ts b/tests/unit/provider-limits-local-apikey-sync-spacing.test.ts new file mode 100644 index 0000000000..ac0a285133 --- /dev/null +++ b/tests/unit/provider-limits-local-apikey-sync-spacing.test.ts @@ -0,0 +1,118 @@ +/** + * Local/API-key provider-limits sync must honor PROVIDER_LIMITS_SYNC_SPACING_MS + * too, not just the OAuth path (#6916). + * + * `syncAllProviderLimits` previously ran non-OAuth (local/API-key, e.g. Ollama) + * connections in `concurrency`-sized chunks with NO spacing at all between + * chunks, so setting `PROVIDER_LIMITS_SYNC_SPACING_MS` had no effect on that + * path. This is the direct regression guard: forces >1 chunk (concurrency=1) + * and asserts a measured gap >= spacingMs between chunk start times. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-apikey-spacing-sync-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = "test-apikey-spacing-sync-secret"; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const providerLimits = await import("../../src/lib/usage/providerLimits.ts"); + +const originalFetch = globalThis.fetch; + +test.beforeEach(() => { + globalThis.fetch = originalFetch; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + delete process.env.PROVIDER_LIMITS_SYNC_SPACING_MS; +}); + +test.after(() => { + globalThis.fetch = originalFetch; + delete process.env.PROVIDER_LIMITS_SYNC_SPACING_MS; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +async function createGlmApiKeyConnection(i: number) { + return providersDb.createProviderConnection({ + provider: "glm", + authType: "apikey", + name: `GLM Spacing ${i}`, + apiKey: `glm-spacing-key-${i}`, + }); +} + +function glmQuotaResponse() { + return new Response( + JSON.stringify({ + code: 200, + success: true, + data: { + planName: "max", + limits: [ + { + type: "TOKENS_LIMIT", + unit: 3, + number: 5, + percentage: 13, + nextResetTime: Math.floor(Date.now() / 1000) + 3 * 3600, + models: [], + }, + ], + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); +} + +test("syncAllProviderLimits spaces chunks for local/API-key connections when spacingMs is set", async () => { + process.env.PROVIDER_LIMITS_SYNC_SPACING_MS = "60"; + for (let i = 0; i < 3; i++) await createGlmApiKeyConnection(i); + + const chunkStarts: number[] = []; + const start = Date.now(); + + globalThis.fetch = (async () => { + chunkStarts.push(Date.now() - start); + return glmQuotaResponse(); + }) as typeof fetch; + + // concurrency: 1 forces 3 chunks of size 1 → 2 gaps must be >= spacingMs. + await providerLimits.syncAllProviderLimits({ source: "scheduled", concurrency: 1 }); + + assert.equal(chunkStarts.length, 3, "expected 3 fetches, one per connection"); + const gaps: number[] = []; + for (let i = 1; i < chunkStarts.length; i++) gaps.push(chunkStarts[i] - chunkStarts[i - 1]); + assert.ok( + gaps.every((g) => g >= 50), + `every chunk gap must be >= configured spacing (~60ms), gaps=${gaps.join(",")}` + ); +}); + +test("syncAllProviderLimits does not space local/API-key chunks when spacingMs=0 (opt-out)", async () => { + process.env.PROVIDER_LIMITS_SYNC_SPACING_MS = "0"; + for (let i = 0; i < 3; i++) await createGlmApiKeyConnection(i); + + const chunkStarts: number[] = []; + const start = Date.now(); + + globalThis.fetch = (async () => { + chunkStarts.push(Date.now() - start); + return glmQuotaResponse(); + }) as typeof fetch; + + await providerLimits.syncAllProviderLimits({ source: "scheduled", concurrency: 1 }); + + assert.equal(chunkStarts.length, 3); + const gaps: number[] = []; + for (let i = 1; i < chunkStarts.length; i++) gaps.push(chunkStarts[i] - chunkStarts[i - 1]); + assert.ok( + gaps.every((g) => g < 40), + `spacingMs=0 must not introduce a forced gap, gaps=${gaps.join(",")}` + ); +});