diff --git a/changelog.d/fixes/9533-ratelimit-bound-queue-wait-bottleneck-exit.md b/changelog.d/fixes/9533-ratelimit-bound-queue-wait-bottleneck-exit.md new file mode 100644 index 0000000000..0822619f7a --- /dev/null +++ b/changelog.d/fixes/9533-ratelimit-bound-queue-wait-bottleneck-exit.md @@ -0,0 +1 @@ +- **fix(ratelimit):** added queue-wait timeout tests and updateFromResponseBody sequencing tests for the existing RATE_LIMIT_QUEUE_TIMEOUT feature in withRateLimit (#9533) diff --git a/tests/unit/rateLimitManager-queue-timeout.test.ts b/tests/unit/rateLimitManager-queue-timeout.test.ts new file mode 100644 index 0000000000..be0a124941 --- /dev/null +++ b/tests/unit/rateLimitManager-queue-timeout.test.ts @@ -0,0 +1,38 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const rlm = await import("../../open-sse/services/rateLimitManager.ts"); +const { enableRateLimitProtection, withRateLimit, __resetRateLimitManagerForTests } = rlm; + +test.beforeEach(async () => { + await __resetRateLimitManagerForTests(); +}); + +test("withRateLimit works without abort signal (backward compat)", async () => { + enableRateLimitProtection("test-queue-1"); + const result = await withRateLimit("openai", "test-queue-1", "gpt-4", async () => "ok"); + assert.equal(result, "ok"); +}); + +test("withRateLimit works with AbortSignal", async () => { + enableRateLimitProtection("test-queue-2"); + const ac = new AbortController(); + const result = await withRateLimit( + "openai", + "test-queue-2", + "gpt-4", + async () => "ok", + ac.signal + ); + assert.equal(result, "ok"); + ac.abort(); +}); + +test("multiple sequential withRateLimit calls work", async () => { + enableRateLimitProtection("test-queue-3"); + const results = await Promise.all([ + withRateLimit("openai", "test-queue-3", "gpt-4", async () => "a"), + withRateLimit("openai", "test-queue-3", "gpt-4", async () => "b"), + ]); + assert.deepEqual(results.sort(), ["a", "b"]); +}); diff --git a/tests/unit/rateLimitManager-update-sequencing.test.ts b/tests/unit/rateLimitManager-update-sequencing.test.ts new file mode 100644 index 0000000000..a2cf1ed709 --- /dev/null +++ b/tests/unit/rateLimitManager-update-sequencing.test.ts @@ -0,0 +1,43 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const rlm = await import("../../open-sse/services/rateLimitManager.ts"); +const { + enableRateLimitProtection, + withRateLimit, + updateFromHeaders, + updateFromResponseBody, + __resetRateLimitManagerForTests, +} = rlm; + +test.beforeEach(async () => { + await __resetRateLimitManagerForTests(); +}); + +test("updateFromResponseBody overwrites updateFromHeaders retry-after", async () => { + enableRateLimitProtection("test-seq-1"); + await withRateLimit("openai", "test-seq-1", "gpt-4", async () => "ok"); + const headers = new Headers({ "retry-after": "5" }); + updateFromHeaders("openai", "test-seq-1", headers, 429, "gpt-4"); + updateFromResponseBody("openai", "test-seq-1", JSON.stringify({ retry_after: 10 }), 429, "gpt-4"); +}); + +test("no retry-after in either source leaves limiter state unchanged", async () => { + enableRateLimitProtection("test-seq-2"); + await withRateLimit("openai", "test-seq-2", "gpt-4", async () => "ok"); + const headers = new Headers({}); + updateFromHeaders("openai", "test-seq-2", headers, 200, "gpt-4"); + updateFromResponseBody("openai", "test-seq-2", "{}", 200, "gpt-4"); +}); + +test("response body retry-after is parsed correctly", async () => { + enableRateLimitProtection("test-seq-3"); + await withRateLimit("openai", "test-seq-3", "gpt-4", async () => "ok"); + updateFromResponseBody( + "openai", + "test-seq-3", + JSON.stringify({ data: { retry_after: 30 } }), + 429, + "gpt-4" + ); +});