From ca23eed77cd19476141b3a39c74abee403203a68 Mon Sep 17 00:00:00 2001 From: Ravi Tharuma <25951435+RaviTharuma@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:45:27 -0700 Subject: [PATCH] fix(models): memoize getModelsDevPricing (event loop / healthz) (#10055) * fix(models): memoize getModelsDevPricing for /v1/models catalog resolveCatalogPricing called getModelsDevPricing once per model while building GET /v1/models. Each call re-scanned models_dev_pricing and JSON.parsed every row (~10k SQL scans + multi-GB parse work), pegging the event loop so even /healthz timed out (#9685, #10052). Memoize the parsed map until saveModelsDevPricing / clearModelsDevPricing and add a unit test for invalidation. Signed-off-by: Ravi Tharuma * fix(db): invalidate modelsDevPricing cache on DB reset (#10055) Copilot review fixes: 1. Register invalidateModelsDevPricingCache() with DB state reset system so resetDbInstance() clears the process-local memo, preventing stale pricing data from surviving across DB reset/restore operations. 2. Add test assertion verifying DB reset bypasses the memo (Copilot #10055). The process-local memo at modelsDevSync.ts:204 caches getModelsDevPricing() results until saveModelsDevPricing()/clearModelsDevPricing() to avoid re-scanning all pricing rows on every /v1/models request. Without this hook, backup restore and test DB resets would serve stale cached data from the previous connection. Tests: npm run test:unit:serial -- tests/unit/modelsDevSync-extended.test.ts --------- Signed-off-by: Ravi Tharuma Co-authored-by: Ravi Tharuma Co-authored-by: Cursor Agent --- package-lock.json | 101 +++------------------- src/lib/modelsDevSync.ts | 26 ++++++ tests/unit/modelsDevSync-extended.test.ts | 39 +++++++++ 3 files changed, 76 insertions(+), 90 deletions(-) diff --git a/package-lock.json b/package-lock.json index c061366549..c7308935ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3692,9 +3692,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3711,9 +3708,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3730,9 +3724,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3749,9 +3740,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3768,9 +3756,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3787,9 +3772,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3806,9 +3788,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3825,9 +3804,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -3844,9 +3820,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3869,9 +3842,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3894,9 +3864,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3919,9 +3886,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3944,9 +3908,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3969,9 +3930,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3994,9 +3952,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -4019,9 +3974,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -5393,9 +5345,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -5412,9 +5361,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -5431,9 +5377,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -5450,9 +5393,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -10670,9 +10610,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -10690,9 +10627,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -10710,9 +10644,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -10730,9 +10661,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -12779,9 +12707,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "optional": true, "os": [ "linux" @@ -12795,9 +12720,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "optional": true, "os": [ "linux" @@ -12811,9 +12733,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "optional": true, "os": [ "linux" @@ -12827,9 +12746,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "optional": true, "os": [ "linux" @@ -12843,9 +12759,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "optional": true, "os": [ "linux" @@ -12859,9 +12772,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "optional": true, "os": [ "linux" @@ -24477,6 +24387,17 @@ "node": ">= 14" } }, + "node_modules/libxmljs2/node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/libxmljs2/node_modules/cacache": { "version": "19.0.1", "resolved": "https://registry.npmjs.org/cacache/-/cacache-19.0.1.tgz", diff --git a/src/lib/modelsDevSync.ts b/src/lib/modelsDevSync.ts index 8c34135feb..b433fa6674 100644 --- a/src/lib/modelsDevSync.ts +++ b/src/lib/modelsDevSync.ts @@ -20,6 +20,7 @@ import { getDbInstance } from "./db/core"; import { invalidateDbCache } from "./db/readCache"; import { backupDbFile } from "./db/backup"; +import { registerDbStateResetter } from "./db/stateReset"; import { transformModelsDevToPricing, @@ -193,10 +194,32 @@ function mapCapabilityRecord(record: Record): ModelCapabilityEn }; } +/** + * Process-local memo for models.dev pricing. + * + * `resolveCatalogPricing` used to call `getModelsDevPricing()` once per model + * while building `/v1/models` (~10k+ times). Each call re-ran the full SQL scan + * and `JSON.parse`d every pricing row, pegging the event loop for minutes + * (see #9685 / #10052). Memoize until the next save/clear write. + */ +let modelsDevPricingCache: PricingByProvider | null = null; + +function invalidateModelsDevPricingCache(): void { + modelsDevPricingCache = null; +} + +// Register cache invalidation with DB state reset system so resetDbInstance() clears the memo. +registerDbStateResetter(invalidateModelsDevPricingCache); + /** * Read synced pricing from `models_dev_pricing` namespace. + * Results are memoized until `saveModelsDevPricing` / `clearModelsDevPricing`. */ export function getModelsDevPricing(): PricingByProvider { + if (modelsDevPricingCache) { + return modelsDevPricingCache; + } + const db = getDbInstance(); const rows = db .prepare("SELECT key, value FROM key_value WHERE namespace = 'models_dev_pricing'") @@ -213,6 +236,7 @@ export function getModelsDevPricing(): PricingByProvider { console.warn(`[MODELS_DEV] Corrupted pricing data for provider "${key}", skipping`); } } + modelsDevPricingCache = synced; return synced; } @@ -233,6 +257,7 @@ export function saveModelsDevPricing(data: PricingByProvider): void { }); tx(); backupDbFile("pre-write"); + invalidateModelsDevPricingCache(); invalidateDbCache("pricing"); } @@ -243,6 +268,7 @@ export function clearModelsDevPricing(): void { const db = getDbInstance(); db.prepare("DELETE FROM key_value WHERE namespace = 'models_dev_pricing'").run(); backupDbFile("pre-write"); + invalidateModelsDevPricingCache(); invalidateDbCache("pricing"); } diff --git a/tests/unit/modelsDevSync-extended.test.ts b/tests/unit/modelsDevSync-extended.test.ts index 9b96a8440b..cd75919918 100644 --- a/tests/unit/modelsDevSync-extended.test.ts +++ b/tests/unit/modelsDevSync-extended.test.ts @@ -260,6 +260,45 @@ test("modelsDev pricing helpers persist records, skip corrupted rows, and clear assert.deepEqual(modelsDev.getModelsDevPricing(), {}); }); +test("getModelsDevPricing memoizes until save/clear (#9685)", async () => { + const modelsDev = await importFresh("pricing-memo"); + const pricing = modelsDev.transformModelsDevToPricing(MOCK_MODELS_DEV_DATA); + modelsDev.saveModelsDevPricing(pricing); + + const first = modelsDev.getModelsDevPricing(); + const second = modelsDev.getModelsDevPricing(); + assert.equal(first, second, "repeated reads must return the same memoized object"); + + // Mutating DB under the cache must not be visible until invalidation. + const db = core.getDbInstance(); + db.prepare("DELETE FROM key_value WHERE namespace = 'models_dev_pricing'").run(); + assert.equal( + modelsDev.getModelsDevPricing(), + first, + "raw SQL without save/clear must not bypass the memo" + ); + + modelsDev.clearModelsDevPricing(); + assert.deepEqual(modelsDev.getModelsDevPricing(), {}); + + modelsDev.saveModelsDevPricing(pricing); + const afterSave = modelsDev.getModelsDevPricing(); + assert.notEqual(afterSave, first, "save must invalidate the memo"); + assert.equal(afterSave.openai["gpt-4o"].input, 2.5); + + // Copilot review: DB reset must invalidate the memo so import/restore doesn't serve stale pricing. + const beforeReset = modelsDev.getModelsDevPricing(); + core.resetDbInstance(); + const afterReset = modelsDev.getModelsDevPricing(); + assert.notEqual( + afterReset, + beforeReset, + "resetDbInstance must invalidate the memo (Copilot #10055)" + ); + // Data is still on disk after resetDbInstance(), but the cache was cleared and re-read from fresh DB. + assert.equal(afterReset.openai["gpt-4o"].input, 2.5, "DB reset re-reads from fresh connection"); +}); + test("modelsDev capabilities helpers create the table, persist rows, filter by provider/model, and expose context limits", async () => { const modelsDev = await importFresh("capabilities-storage"); const capabilities = modelsDev.transformModelsDevToCapabilities(MOCK_MODELS_DEV_DATA);