fix(ratelimit): add queue-wait timeout and update sequencing tests (#9533)

changelog.d/fixes/9533-ratelimit-bound-queue-wait-bottleneck-exit.md

@@ -0,0 +1 @@
+- **fix(ratelimit):** added queue-wait timeout tests and updateFromResponseBody sequencing tests for the existing RATE_LIMIT_QUEUE_TIMEOUT feature in withRateLimit (#9533)

tests/unit/rateLimitManager-queue-timeout.test.ts

@@ -0,0 +1,38 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const rlm = await import("../../open-sse/services/rateLimitManager.ts");
+const { enableRateLimitProtection, withRateLimit, __resetRateLimitManagerForTests } = rlm;
+
+test.beforeEach(async () => {
+  await __resetRateLimitManagerForTests();
+});
+
+test("withRateLimit works without abort signal (backward compat)", async () => {
+  enableRateLimitProtection("test-queue-1");
+  const result = await withRateLimit("openai", "test-queue-1", "gpt-4", async () => "ok");
+  assert.equal(result, "ok");
+});
+
+test("withRateLimit works with AbortSignal", async () => {
+  enableRateLimitProtection("test-queue-2");
+  const ac = new AbortController();
+  const result = await withRateLimit(
+    "openai",
+    "test-queue-2",
+    "gpt-4",
+    async () => "ok",
+    ac.signal
+  );
+  assert.equal(result, "ok");
+  ac.abort();
+});
+
+test("multiple sequential withRateLimit calls work", async () => {
+  enableRateLimitProtection("test-queue-3");
+  const results = await Promise.all([
+    withRateLimit("openai", "test-queue-3", "gpt-4", async () => "a"),
+    withRateLimit("openai", "test-queue-3", "gpt-4", async () => "b"),
+  ]);
+  assert.deepEqual(results.sort(), ["a", "b"]);
+});

tests/unit/rateLimitManager-update-sequencing.test.ts

@@ -0,0 +1,43 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const rlm = await import("../../open-sse/services/rateLimitManager.ts");
+const {
+  enableRateLimitProtection,
+  withRateLimit,
+  updateFromHeaders,
+  updateFromResponseBody,
+  __resetRateLimitManagerForTests,
+} = rlm;
+
+test.beforeEach(async () => {
+  await __resetRateLimitManagerForTests();
+});
+
+test("updateFromResponseBody overwrites updateFromHeaders retry-after", async () => {
+  enableRateLimitProtection("test-seq-1");
+  await withRateLimit("openai", "test-seq-1", "gpt-4", async () => "ok");
+  const headers = new Headers({ "retry-after": "5" });
+  updateFromHeaders("openai", "test-seq-1", headers, 429, "gpt-4");
+  updateFromResponseBody("openai", "test-seq-1", JSON.stringify({ retry_after: 10 }), 429, "gpt-4");
+});
+
+test("no retry-after in either source leaves limiter state unchanged", async () => {
+  enableRateLimitProtection("test-seq-2");
+  await withRateLimit("openai", "test-seq-2", "gpt-4", async () => "ok");
+  const headers = new Headers({});
+  updateFromHeaders("openai", "test-seq-2", headers, 200, "gpt-4");
+  updateFromResponseBody("openai", "test-seq-2", "{}", 200, "gpt-4");
+});
+
+test("response body retry-after is parsed correctly", async () => {
+  enableRateLimitProtection("test-seq-3");
+  await withRateLimit("openai", "test-seq-3", "gpt-4", async () => "ok");
+  updateFromResponseBody(
+    "openai",
+    "test-seq-3",
+    JSON.stringify({ data: { retry_after: 30 } }),
+    429,
+    "gpt-4"
+  );
+});