diff --git a/changelog.d/fixes/9763-ratelimit-mintime-floor.md b/changelog.d/fixes/9763-ratelimit-mintime-floor.md new file mode 100644 index 0000000000..2b145f1e1a --- /dev/null +++ b/changelog.d/fixes/9763-ratelimit-mintime-floor.md @@ -0,0 +1 @@ +- **fix(ratelimit):** respect operator `minTimeBetweenRequestsMs` floor when relaxing the limiter on headroom — the adaptive rate-limit learning no longer silently erases a configured minimum gap between requests when the upstream reports plenty of remaining capacity ([#9763](https://github.com/diegosouzapw/OmniRoute/issues/9763)). diff --git a/open-sse/services/rateLimitManager.ts b/open-sse/services/rateLimitManager.ts index a7bdeb2b09..18815b32a5 100644 --- a/open-sse/services/rateLimitManager.ts +++ b/open-sse/services/rateLimitManager.ts @@ -756,7 +756,7 @@ export function updateFromHeaders(provider, connectionId, headers, status, model ); } else if (remaining > limit * 0.5) { // Plenty of headroom — relax the limiter - updates.minTime = 0; + updates.minTime = resolveMinTime(currentRequestQueueSettings.minTimeBetweenRequestsMs); updates.reservoir = null; updates.reservoirRefreshAmount = null; updates.reservoirRefreshInterval = null; diff --git a/tests/unit/rateLimitManager-mintime-floor-9763.test.ts b/tests/unit/rateLimitManager-mintime-floor-9763.test.ts new file mode 100644 index 0000000000..2c4ee55755 --- /dev/null +++ b/tests/unit/rateLimitManager-mintime-floor-9763.test.ts @@ -0,0 +1,88 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const rlm = await import("../../open-sse/services/rateLimitManager.ts"); +const { + enableRateLimitProtection, + withRateLimit, + updateFromHeaders, + applyRequestQueueSettings, + __setLimiterFactoryForTests, + __resetRateLimitManagerForTests, +} = rlm; + +test.beforeEach(async () => { + await __resetRateLimitManagerForTests(); +}); + +test("headroom relaxation respects operator minTimeBetweenRequestsMs floor (#9763)", async () => { + // Apply an operator-configured minTime floor of 200ms + await applyRequestQueueSettings({ + minTimeBetweenRequestsMs: 200, + concurrentRequests: 0, + requestsPerMinute: 0, + maxWaitMs: 30000, + autoEnableApiKeyProvider: false, + }); + + let capturedMinTime: number | undefined; + + // Inject a fake limiter whose updateSettings captures the minTime. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const noop = (): any => undefined; + + __setLimiterFactoryForTests(() => { + const listeners: Record void>> = {}; + const fake = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + updateSettings(updates: Record) { + capturedMinTime = typeof updates.minTime === "number" ? updates.minTime : undefined; + return fake; + }, + on(event: string, fn: (...args: unknown[]) => void) { + (listeners[event] ??= []).push(fn); + return fake; + }, + schedule(arg0: unknown, arg1?: unknown) { + const fn = typeof arg1 === "function" ? arg1 : typeof arg0 === "function" ? arg0 : noop; + return fn(); + }, + disconnect() { + return Promise.resolve(); + }, + chain() { + return fake; + }, + counts() { + return { RECEIVED: 0, QUEUED: 0, RUNNING: 0, EXECUTING: 0 }; + }, + currentReservoir() { + return Promise.resolve(null); + }, + stop() { + return Promise.resolve(); + }, + }; + return fake; + }); + + enableRateLimitProtection("test-mintime-floor"); + + // Materialize the limiter with a dummy request + await withRateLimit("openai", "test-mintime-floor", "gpt-4", async () => "ok"); + + // Simulate a response with plenty of headroom: remaining=80 > limit*0.5=50 + const headers = new Headers({ + "x-ratelimit-limit-requests": "100", + "x-ratelimit-remaining-requests": "80", + }); + updateFromHeaders("openai", "test-mintime-floor", headers, 200, "gpt-4"); + + // The operator configured minTime=200, so headroom relaxation MUST NOT + // override it to 0. Before the fix, capturedMinTime === 0 (RED). + assert.strictEqual( + capturedMinTime, + 200, + `Expected minTime=200 (operator floor), got ${capturedMinTime}` + ); +});