import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-rate-limit-manager-")); process.env.DATA_DIR = TEST_DATA_DIR; // Dynamic imports are required because DATA_DIR must be set before DB modules evaluate. const core = await import("../../src/lib/db/core.ts"); const providersDb = await import("../../src/lib/db/providers.ts"); const resilienceSettings = await import("../../src/lib/resilience/settings.ts"); const rateLimitManager = await import("../../open-sse/services/rateLimitManager.ts"); const rateLimitErrors = await import("../../open-sse/services/rateLimitManager/errors.ts"); const accountFallback = await import("../../open-sse/services/accountFallback.ts"); const Bottleneck = (await import("bottleneck")).default; // These integration-style tests exercise real Bottleneck timer/event behavior. function wait(ms) { const { promise, resolve } = Promise.withResolvers(); setTimeout(resolve, ms); return promise; } // A real deadline is intentional: these tests drive real Bottleneck queues, and // a broken cleanup path otherwise leaves Node's test process pending forever. async function settleWithin( promise: Promise, message: string, timeoutMs = 2_000 ): Promise { let timeout: NodeJS.Timeout; const deadline = new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error(message)), timeoutMs); }); try { return await Promise.race([promise, deadline]); } finally { clearTimeout(timeout); } } type TestBottleneck = InstanceType & { _drainAll: (...args: unknown[]) => Promise; }; /** * Fault injection for the observed Bottleneck failure mode: jobs enter the real * Bottleneck queue, but its internal drain loop stops making progress. Keep the * private mutation in this one helper so the tests otherwise exercise public * manager and Bottleneck behavior. */ function injectDrainWedge(limiter: InstanceType): TestBottleneck { const wedged = limiter as TestBottleneck; wedged._drainAll = () => Promise.resolve(null); return wedged; } async function waitForCondition( condition: () => boolean | Promise, message: string ): Promise { const deadline = Date.now() + 1_000; while (!(await condition())) { if (Date.now() >= deadline) throw new Error(message); await wait(5); } } async function expectWedgeError(promise: Promise): Promise { await assert.rejects( settleWithin(promise, "stranded limiter caller did not reject after wedge recovery"), (error: Error & { code?: string }) => { assert.equal(error.code, "RATE_LIMIT_QUEUE_WEDGED"); assert.deepEqual(rateLimitErrors.getTrustedLocalRateLimitError(error), { code: "RATE_LIMIT_QUEUE_WEDGED", status: 503, }); return true; } ); } async function flushBackgroundWork() { await wait(50); const { promise, resolve } = Promise.withResolvers(); setImmediate(resolve); await promise; } async function resetStorage() { core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); } test.beforeEach(async () => { await resetStorage(); }); test.afterEach(async () => { await rateLimitManager.__resetRateLimitManagerForTests(); await flushBackgroundWork(); }); test.after(async () => { await rateLimitManager.__resetRateLimitManagerForTests(); await flushBackgroundWork(); core.resetDbInstance(); fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); }); test("rate limit manager bypasses disabled connections and exposes inactive status", async () => { const result = await rateLimitManager.withRateLimit("openai", "disabled-conn", null, async () => { return "bypassed"; }); assert.equal(result, "bypassed"); assert.deepEqual(rateLimitManager.getRateLimitStatus("openai", "disabled-conn"), { enabled: false, active: false, queued: 0, running: 0, }); assert.deepEqual(rateLimitManager.getAllRateLimitStatus(), {}); }); test("idle-capacity watchdog honors grace, cleans up in order, and rejects the stranded caller", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 0, concurrentRequests: 1, minTimeBetweenRequestsMs: 0, maxQueueDepth: 0, }); const cleanupEvents: string[] = []; let limitersCreated = 0; rateLimitManager.__setLimiterFactoryForTests((options) => { const limiter = new Bottleneck(options); limitersCreated++; if (limitersCreated === 1) { injectDrainWedge(limiter); const originalStop = limiter.stop.bind(limiter); const originalDisconnect = limiter.disconnect.bind(limiter); limiter.stop = async (stopOptions) => { cleanupEvents.push("stop:start"); await originalStop(stopOptions); cleanupEvents.push("stop:done"); }; limiter.disconnect = async (flush) => { cleanupEvents.push("disconnect"); await originalDisconnect(flush); }; } return limiter; }); rateLimitManager.enableRateLimitProtection("idle-capacity-conn"); let executions = 0; const pending = rateLimitManager.withRateLimit( "openai", "idle-capacity-conn", "gpt-4o", async () => { executions++; return "must-not-run"; } ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", "idle-capacity-conn").queued === 1, "the injected drain failure never established a real queued job" ); const queuedObservedAt = Date.now(); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(queuedObservedAt + 9_000), "watchdog grace-period scan did not finish" ); assert.equal( rateLimitManager.getRateLimitStatus("openai", "idle-capacity-conn").queued, 1, "the queue must survive before the 10s stability grace" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(queuedObservedAt + 11_000), "watchdog wedge cleanup did not finish" ); await expectWedgeError(pending); assert.equal(executions, 0, "watchdog recovery must never replay application work"); assert.equal(limitersCreated, 1, "dropped callers must not create a replacement limiter"); assert.deepEqual(cleanupEvents, ["stop:start", "stop:done", "disconnect"]); }); test("wedge eviction rejects every queued caller and preserves learned state for future traffic", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 60, concurrentRequests: 6, minTimeBetweenRequestsMs: 0, maxQueueDepth: 0, }); const createdOptions: Bottleneck.ConstructorOptions[] = []; const createdLimiters: InstanceType[] = []; rateLimitManager.__setLimiterFactoryForTests((options) => { createdOptions.push({ ...options }); const limiter = new Bottleneck(options); createdLimiters.push(limiter); if (createdLimiters.length === 1) injectDrainWedge(limiter); return limiter; }); const connectionId = "learned-state-conn"; rateLimitManager.enableRateLimitProtection(connectionId); rateLimitManager.updateFromHeaders( "openai", connectionId, { "x-ratelimit-limit-requests": "100", "x-ratelimit-remaining-requests": "1", "x-ratelimit-reset-requests": "60s", }, 200, "gpt-4o" ); await waitForCondition( async () => (await rateLimitManager.__getLimiterStateForTests("openai", connectionId, "gpt-4o")) ?.reservoir === 1, "the learned reservoir was not applied" ); let executions = 0; const stranded = Array.from({ length: 3 }, () => rateLimitManager.withRateLimit("openai", connectionId, "gpt-4o", async () => { executions++; return "must-not-run"; }) ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 3, "all callers did not enter the wedged queue" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 11_000), "multi-caller wedge cleanup did not finish" ); const settled = await settleWithin( Promise.allSettled(stranded), "not every stranded limiter caller settled" ); assert.equal(executions, 0); assert.equal(createdLimiters.length, 1, "wedge recovery must not retry any dropped caller"); for (const result of settled) { assert.equal(result.status, "rejected"); assert.equal((result as PromiseRejectedResult).reason.code, "RATE_LIMIT_QUEUE_WEDGED"); } assert.equal( await rateLimitManager.withRateLimit("openai", connectionId, "gpt-4o", async () => "future"), "future" ); assert.equal(createdLimiters.length, 2, "future traffic should create one replacement limiter"); assert.equal(createdOptions[1].reservoir, 1, "replacement must retain the remaining reservoir"); assert.equal(createdOptions[1].minTime, 590, "replacement must retain learned request spacing"); const queuedAfterPreservedPermit = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "after-refill" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 1, "the preserved reservoir should allow only one request" ); await createdLimiters[1].incrementReservoir(1); assert.equal(await queuedAfterPreservedPermit, "after-refill"); }); test("global settings changed after eviction replace stale pending configuration", async () => { const connection = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "wedge-global-settings", apiKey: "sk-wedge-global-settings", isActive: true, rateLimitProtection: true, }); await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 60, concurrentRequests: 6, minTimeBetweenRequestsMs: 0, }); const createdOptions: Bottleneck.ConstructorOptions[] = []; rateLimitManager.__setLimiterFactoryForTests((options) => { createdOptions.push({ ...options }); const limiter = new Bottleneck(options); if (createdOptions.length === 1) injectDrainWedge(limiter); return limiter; }); const pending = rateLimitManager.withRateLimit( "openai", connection.id, "gpt-4o", async () => "must-not-run" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connection.id).queued === 1, "global-settings caller did not enter the wedged queue" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 11_000), "global-settings wedge cleanup did not finish" ); await expectWedgeError(pending); await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 8, concurrentRequests: 3, minTimeBetweenRequestsMs: 31, }); assert.equal( await rateLimitManager.withRateLimit( "openai", connection.id, "gpt-4o", async () => "new-policy" ), "new-policy" ); assert.equal(createdOptions[1].reservoir, 8); assert.equal(createdOptions[1].maxConcurrent, 3); assert.equal(createdOptions[1].minTime, 31); }); test("connection overrides changed after eviction replace stale pending configuration", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 60, concurrentRequests: 6, minTimeBetweenRequestsMs: 0, }); const createdOptions: Bottleneck.ConstructorOptions[] = []; rateLimitManager.__setLimiterFactoryForTests((options) => { createdOptions.push({ ...options }); const limiter = new Bottleneck(options); if (createdOptions.length === 1) injectDrainWedge(limiter); return limiter; }); const connectionId = "wedge-override-conn"; rateLimitManager.enableRateLimitProtection(connectionId); const pending = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "must-not-run" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 1, "override caller did not enter the wedged queue" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 11_000), "override wedge cleanup did not finish" ); await expectWedgeError(pending); rateLimitManager.refreshConnectionRateLimits(connectionId, { rpm: 7, maxConcurrent: 2, minTime: 25, }); assert.equal( await rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "new-override" ), "new-override" ); assert.equal(createdOptions[1].reservoir, 7); assert.equal(createdOptions[1].maxConcurrent, 2); assert.equal(createdOptions[1].minTime, 25); }); test("disable and re-enable discard learned state preserved by an earlier wedge", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 60, concurrentRequests: 6, minTimeBetweenRequestsMs: 0, }); const createdOptions: Bottleneck.ConstructorOptions[] = []; rateLimitManager.__setLimiterFactoryForTests((options) => { createdOptions.push({ ...options }); const limiter = new Bottleneck(options); if (createdOptions.length === 1) injectDrainWedge(limiter); return limiter; }); const connectionId = "wedge-reenabled-conn"; rateLimitManager.enableRateLimitProtection(connectionId); rateLimitManager.updateFromHeaders( "openai", connectionId, { "x-ratelimit-limit-requests": "100", "x-ratelimit-remaining-requests": "1", "x-ratelimit-reset-requests": "60s", }, 200, "gpt-4o" ); await waitForCondition( async () => (await rateLimitManager.__getLimiterStateForTests("openai", connectionId, "gpt-4o")) ?.reservoir === 1, "learned reservoir was not applied before disable/re-enable" ); const pending = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "must-not-run" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 1, "disable/re-enable caller did not enter the wedged queue" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 11_000), "disable/re-enable wedge cleanup did not finish" ); await expectWedgeError(pending); rateLimitManager.disableRateLimitProtection(connectionId); rateLimitManager.enableRateLimitProtection(connectionId); assert.equal( await rateLimitManager.withRateLimit("openai", connectionId, "gpt-4o", async () => "reenabled"), "reenabled" ); assert.equal(createdOptions[1].reservoir, 60); assert.equal(createdOptions[1].minTime, 0); }); test("idle-capacity watchdog preserves a legitimate exhausted-reservoir queue", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 1, concurrentRequests: 1, minTimeBetweenRequestsMs: 0, maxQueueDepth: 0, }); let limiter: TestBottleneck | null = null; rateLimitManager.__setLimiterFactoryForTests((options) => { limiter = new Bottleneck(options) as TestBottleneck; return limiter; }); rateLimitManager.enableRateLimitProtection("zero-reservoir-conn"); assert.equal( await rateLimitManager.withRateLimit( "openai", "zero-reservoir-conn", "gpt-4o", async () => "first" ), "first" ); const pending = rateLimitManager.withRateLimit( "openai", "zero-reservoir-conn", "gpt-4o", async () => "after-refresh" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", "zero-reservoir-conn").queued === 1, "the exhausted reservoir did not queue the follow-up" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 150_000), "zero-reservoir watchdog scan did not finish" ); assert.equal( rateLimitManager.getRateLimitStatus("openai", "zero-reservoir-conn").queued, 1, "a zero-reservoir wait must survive regardless of elapsed time" ); assert.ok(limiter); await limiter.incrementReservoir(1); assert.equal(await pending, "after-refresh"); }); test("idle-capacity watchdog preserves a real Bottleneck minTime wait", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 0, concurrentRequests: 1, minTimeBetweenRequestsMs: 100, maxQueueDepth: 0, }); rateLimitManager.enableRateLimitProtection("min-time-conn"); await rateLimitManager.withRateLimit("openai", "min-time-conn", "gpt-4o", async () => "first"); const pending = rateLimitManager.withRateLimit( "openai", "min-time-conn", "gpt-4o", async () => "after-min-time" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", "min-time-conn").running === 1, "Bottleneck did not place the minTime-delayed job in RUNNING" ); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 150_000), "minTime watchdog scan did not finish" ); assert.equal( rateLimitManager.getRateLimitStatus("openai", "min-time-conn").running, 1, "a legitimate RUNNING minTime delay must not be evicted" ); assert.equal(await pending, "after-min-time"); }); test("events from an evicted limiter cannot erase replacement queue progress", async () => { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, maxWaitMs: 240_000, requestsPerMinute: 0, concurrentRequests: 1, minTimeBetweenRequestsMs: 0, maxQueueDepth: 0, }); const limiters: InstanceType[] = []; rateLimitManager.__setLimiterFactoryForTests((options) => { const limiter = new Bottleneck(options); limiters.push(limiter); if (limiters.length === 2) injectDrainWedge(limiter); return limiter; }); const connectionId = "stale-listener-conn"; rateLimitManager.enableRateLimitProtection(connectionId); const { promise: oldGate, resolve: releaseOld } = Promise.withResolvers(); const oldExecuting = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => { await oldGate; return "old-first"; } ); await waitForCondition( () => limiters[0]?.counts().EXECUTING === 1, "the old limiter did not begin executing" ); const oldQueued = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "old-second" ); await waitForCondition( () => limiters[0]?.counts().QUEUED === 1, "the old limiter did not queue its second job" ); rateLimitManager.refreshConnectionRateLimits(connectionId, {}); const replacementPending = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "must-not-run" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 1, "the replacement limiter did not establish its queue" ); releaseOld(); assert.equal(await oldExecuting, "old-first"); assert.equal(await oldQueued, "old-second"); await settleWithin( rateLimitManager.__runLimiterWatchdogForTests(Date.now() + 11_000), "stale-listener watchdog cleanup did not finish" ); await expectWedgeError(replacementPending); }); test("watchdog ticks are serialized while an eligibility check is in flight", async () => { const { promise: checkGate, resolve: releaseCheck } = Promise.withResolvers(); let checks = 0; rateLimitManager.__setLimiterFactoryForTests((options) => { const limiter = injectDrainWedge(new Bottleneck(options)); const originalCheck = limiter.check.bind(limiter); limiter.check = async (weight) => { checks++; await checkGate; return originalCheck(weight); }; return limiter; }); const connectionId = "serialized-watchdog-conn"; rateLimitManager.enableRateLimitProtection(connectionId); const pending = rateLimitManager.withRateLimit( "openai", connectionId, "gpt-4o", async () => "must-not-run" ); await waitForCondition( () => rateLimitManager.getRateLimitStatus("openai", connectionId).queued === 1, "the serialized-watchdog fixture did not queue" ); const now = Date.now() + 11_000; const firstTick = rateLimitManager.__runLimiterWatchdogForTests(now); const secondTick = rateLimitManager.__runLimiterWatchdogForTests(now); await waitForCondition(() => checks === 1, "the first tick did not reach limiter.check()"); releaseCheck(); await settleWithin( Promise.all([firstTick, secondTick]), "serialized watchdog scans did not finish" ); await expectWedgeError(pending); assert.equal(checks, 1, "overlapping watchdog calls must share one scan"); }); test("application errors resembling Bottleneck failures remain untouched", async () => { rateLimitManager.enableRateLimitProtection("lookalike-error-conn"); for (const message of [ "This job timed out after 240000 ms.", "rate-limit-watchdog-wedge-reset", ]) { const applicationError = new Error(message); await assert.rejects( rateLimitManager.withRateLimit("openai", "lookalike-error-conn", "gpt-4o", async () => { throw applicationError; }), (error) => error === applicationError ); } }); test("withRateLimit forwards AbortController DOMException without mutating it", async () => { const connectionId = "conn-abort-domexception"; const controller = new AbortController(); rateLimitManager.enableRateLimitProtection(connectionId); const pending = rateLimitManager.withRateLimit( "github-models", connectionId, "microsoft/phi-4-reasoning", async () => { await wait(50); return "late"; }, controller.signal ); controller.abort(); await assert.rejects(pending, (error: unknown) => { assert.ok(error instanceof DOMException); assert.equal(error.name, "AbortError"); return true; }); }); test("rate limit manager handles soft over-limit warnings and normal header learning", async () => { rateLimitManager.enableRateLimitProtection("conn-over-limit"); rateLimitManager.updateFromHeaders( "openai", "conn-over-limit", { "x-ratelimit-over-limit": "yes" }, 200 ); const softStatus = rateLimitManager.getRateLimitStatus("openai", "conn-over-limit"); assert.equal(softStatus.enabled, true); assert.equal(softStatus.active, true); rateLimitManager.enableRateLimitProtection("conn-low-remaining"); rateLimitManager.updateFromHeaders( "openai", "conn-low-remaining", { "x-ratelimit-limit-requests": "100", "x-ratelimit-remaining-requests": "5", "x-ratelimit-reset-requests": "30s", }, 200 ); await rateLimitManager.__flushLearnedLimitsForTests(); const learnedLimits = rateLimitManager.getLearnedLimits(); const learnedEntry = learnedLimits["openai:conn-low-remaining"]; assert.equal(learnedEntry.provider, "openai"); assert.equal(learnedEntry.connectionId, "conn-low-remaining"); assert.equal(learnedEntry.limit, 100); assert.equal(learnedEntry.remaining, 5); assert.ok(learnedEntry.minTime > 0); rateLimitManager.enableRateLimitProtection("conn-high-remaining"); rateLimitManager.updateFromHeaders( "claude", "conn-high-remaining", { get(name) { const map = { "anthropic-ratelimit-requests-limit": "100", "anthropic-ratelimit-requests-remaining": "70", "anthropic-ratelimit-requests-reset": new Date(Date.now() + 30_000).toISOString(), }; return map[name] ?? null; }, }, 200 ); await rateLimitManager.__flushLearnedLimitsForTests(); const allStatuses = rateLimitManager.getAllRateLimitStatus(); assert.ok(allStatuses["openai:conn-over-limit"]); assert.ok(allStatuses["openai:conn-low-remaining"]); assert.ok(allStatuses["claude:conn-high-remaining"]); }); test("rate limit manager handles 429 limiter teardown and disable cleanup", async () => { rateLimitManager.enableRateLimitProtection("conn-429"); rateLimitManager.updateFromHeaders("openai", "conn-429", { "retry-after": "1s" }, 429, "gpt-4o"); await wait(25); assert.equal(rateLimitManager.getRateLimitStatus("openai", "conn-429").active, false); rateLimitManager.enableRateLimitProtection("conn-disable"); rateLimitManager.updateFromHeaders( "gemini", "conn-disable", { "x-ratelimit-limit-requests": "60", "x-ratelimit-remaining-requests": "4", "x-ratelimit-reset-requests": "10s", }, 200, "gemini-2.5-flash" ); await rateLimitManager.__flushLearnedLimitsForTests(); assert.ok(rateLimitManager.getAllRateLimitStatus()["gemini:conn-disable:gemini-2.5-flash"]); rateLimitManager.disableRateLimitProtection("conn-disable"); assert.equal(rateLimitManager.isRateLimitEnabled("conn-disable"), false); assert.equal(rateLimitManager.getRateLimitStatus("gemini", "conn-disable").active, false); }); test("rate limit manager uses model-scoped limiter keys for GitHub Copilot (#1624)", async () => { rateLimitManager.enableRateLimitProtection("conn-github"); rateLimitManager.updateFromHeaders( "github", "conn-github", { "x-ratelimit-limit-requests": "50", "x-ratelimit-remaining-requests": "3", "x-ratelimit-reset-requests": "15s", }, 200, "gpt-5.1-codex-max" ); await rateLimitManager.__flushLearnedLimitsForTests(); // GitHub should use model-scoped key: github:conn-github:gpt-5.1-codex-max const allStatuses = rateLimitManager.getAllRateLimitStatus(); assert.ok( allStatuses["github:conn-github:gpt-5.1-codex-max"], "GitHub limiter key should be model-scoped (github:conn:model)" ); // Verify the limiter state is model-scoped via test helper const limiterState = await rateLimitManager.__getLimiterStateForTests( "github", "conn-github", "gpt-5.1-codex-max" ); assert.equal(limiterState?.key, "github:conn-github:gpt-5.1-codex-max"); }); test("rate limit manager parses retry hints from response bodies and locks models", async () => { rateLimitManager.enableRateLimitProtection("conn-body"); rateLimitManager.updateFromResponseBody( "openai", "conn-body", { error: { details: [{ retryDelay: "2s" }], message: "Please retry later", }, }, 429, "gpt-4o" ); assert.equal(accountFallback.getModelLockoutInfo("openai", "conn-body", "gpt-4o"), null); const limiterState = await rateLimitManager.__getLimiterStateForTests( "openai", "conn-body", "gpt-4o" ); assert.equal(limiterState?.key, "openai:conn-body"); assert.equal(rateLimitManager.getRateLimitStatus("openai", "conn-body").active, true); rateLimitManager.updateFromResponseBody( "openai", "conn-body", JSON.stringify({ error: { type: "rate_limit_error" } }), 429, null ); assert.equal(rateLimitManager.getRateLimitStatus("openai", "conn-body").active, true); }); test("RATE_LIMIT_AUTO_ENABLE env var overrides dashboard auto-enable setting", async () => { const conn = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Env Override", apiKey: "sk-env", isActive: true, }); // Dashboard says auto-enable on, but env says off → off wins const original = process.env.RATE_LIMIT_AUTO_ENABLE; process.env.RATE_LIMIT_AUTO_ENABLE = "false"; try { await rateLimitManager.initializeRateLimits(); assert.equal(rateLimitManager.isRateLimitEnabled(conn.id), false); } finally { if (original === undefined) delete process.env.RATE_LIMIT_AUTO_ENABLE; else process.env.RATE_LIMIT_AUTO_ENABLE = original; } // Reset and verify the opposite: env=true forces on even when dashboard would be off await rateLimitManager.__resetRateLimitManagerForTests(); process.env.RATE_LIMIT_AUTO_ENABLE = "true"; try { await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, }); assert.equal(rateLimitManager.isRateLimitEnabled(conn.id), true); } finally { if (original === undefined) delete process.env.RATE_LIMIT_AUTO_ENABLE; else process.env.RATE_LIMIT_AUTO_ENABLE = original; } }); test("rate limit manager recomputes auto-enabled API key connections when queue settings change", async () => { const autoConnection = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Auto OpenAI", apiKey: "sk-auto", isActive: true, }); const explicitConnection = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "Explicit OpenAI", apiKey: "sk-explicit", isActive: true, rateLimitProtection: true, }); await rateLimitManager.initializeRateLimits(); assert.equal(rateLimitManager.isRateLimitEnabled(autoConnection.id), true); assert.equal(rateLimitManager.isRateLimitEnabled(explicitConnection.id), true); assert.ok(rateLimitManager.getAllRateLimitStatus()[`openai:${autoConnection.id}`]); await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: false, }); assert.equal(rateLimitManager.isRateLimitEnabled(autoConnection.id), false); assert.equal(rateLimitManager.isRateLimitEnabled(explicitConnection.id), true); assert.equal(rateLimitManager.getAllRateLimitStatus()[`openai:${autoConnection.id}`], undefined); await rateLimitManager.applyRequestQueueSettings({ ...resilienceSettings.DEFAULT_RESILIENCE_SETTINGS.requestQueue, autoEnableApiKeyProviders: true, }); assert.equal(rateLimitManager.isRateLimitEnabled(autoConnection.id), true); assert.equal(rateLimitManager.isRateLimitEnabled(explicitConnection.id), true); assert.ok(rateLimitManager.getAllRateLimitStatus()[`openai:${autoConnection.id}`]); }); test("withRateLimit rejects cleanly when the caller aborts with the default DOMException reason", async () => { // `AbortController.abort()` called with no argument (e.g. modelTestRunner's // timeout path) produces a native DOMException as `signal.reason`, whose // `name` is a read-only getter. withRateLimit's abort handling used to // mutate `reason.name = "AbortError"` in place, which throws // `TypeError: Cannot set property name of [object DOMException] which has // only a getter` instead of rejecting with a clean AbortError — surfacing // as an unhandled rejection rather than the intended timeout/slow result. const connection = await providersDb.createProviderConnection({ provider: "openai", authType: "apikey", name: "abort-reason-regression", apiKey: "sk-abort-reason-regression", isActive: true, }); rateLimitManager.enableRateLimitProtection(String(connection.id)); const controller = new AbortController(); const pending = rateLimitManager.withRateLimit( "openai", String(connection.id), "gpt-4o", () => { const { promise, reject } = Promise.withResolvers(); controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true, }); return promise; }, controller.signal ); controller.abort(); // no reason argument -> default DOMException await assert.rejects(pending, (err) => { assert.ok(err instanceof Error); assert.equal(err.name, "AbortError"); return true; }); });