pjandro/OmniRoute
1
0
Fork 0
mirror of https://github.com/diegosouzapw/OmniRoute.git synced 2026-08-13 10:43:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(models): memoize getModelsDevPricing (event loop / healthz) (#10055)

Browse Source 
* fix(models): memoize getModelsDevPricing for /v1/models catalog

resolveCatalogPricing called getModelsDevPricing once per model while
building GET /v1/models. Each call re-scanned models_dev_pricing and
JSON.parsed every row (~10k SQL scans + multi-GB parse work), pegging
the event loop so even /healthz timed out (#9685, #10052).

Memoize the parsed map until saveModelsDevPricing / clearModelsDevPricing
and add a unit test for invalidation.

Signed-off-by: Ravi Tharuma <RaviTharuma@users.noreply.github.com>

* fix(db): invalidate modelsDevPricing cache on DB reset (#10055)

Copilot review fixes:
1. Register invalidateModelsDevPricingCache() with DB state reset system
   so resetDbInstance() clears the process-local memo, preventing stale
   pricing data from surviving across DB reset/restore operations.
2. Add test assertion verifying DB reset bypasses the memo (Copilot #10055).

The process-local memo at modelsDevSync.ts:204 caches getModelsDevPricing()
results until saveModelsDevPricing()/clearModelsDevPricing() to avoid
re-scanning all pricing rows on every /v1/models request. Without this hook,
backup restore and test DB resets would serve stale cached data from the
previous connection.

Tests: npm run test:unit:serial -- tests/unit/modelsDevSync-extended.test.ts

---------

Signed-off-by: Ravi Tharuma <RaviTharuma@users.noreply.github.com>
Co-authored-by: Ravi Tharuma <RaviTharuma@users.noreply.github.com>
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Ravi Tharuma
2026-08-12 20:45:27 -07:00
committed by GitHub
parent 5f0a394091
commit ca23eed77c
3 changed files with 76 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
package-lock.json generated
View File

@@ -3692,9 +3692,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3711,9 +3708,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3730,9 +3724,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3749,9 +3740,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3768,9 +3756,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3787,9 +3772,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3806,9 +3788,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3825,9 +3804,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
@@ -3844,9 +3820,6 @@
"cpu": [
"arm"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3869,9 +3842,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3894,9 +3864,6 @@
"cpu": [
"ppc64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3919,9 +3886,6 @@
"cpu": [
"riscv64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3944,9 +3908,6 @@
"cpu": [
"s390x"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3969,9 +3930,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -3994,9 +3952,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -4019,9 +3974,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "Apache-2.0",
"optional": true,
"os": [
@@ -5393,9 +5345,6 @@
"cpu": [
"arm64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -5412,9 +5361,6 @@
"cpu": [
"arm64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -5431,9 +5377,6 @@
"cpu": [
"x64"
],
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -5450,9 +5393,6 @@
"cpu": [
"x64"
],
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -10670,9 +10610,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -10690,9 +10627,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -10710,9 +10644,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"license": "MIT",
"optional": true,
"os": [
@@ -10730,9 +10661,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"license": "MIT",
"optional": true,
"os": [
@@ -12779,9 +12707,6 @@
"arm"
],
"dev": true,
"libc": [
"glibc"
],
"optional": true,
"os": [
"linux"
@@ -12795,9 +12720,6 @@
"arm"
],
"dev": true,
"libc": [
"musl"
],
"optional": true,
"os": [
"linux"
@@ -12811,9 +12733,6 @@
"arm64"
],
"dev": true,
"libc": [
"glibc"
],
"optional": true,
"os": [
"linux"
@@ -12827,9 +12746,6 @@
"arm64"
],
"dev": true,
"libc": [
"musl"
],
"optional": true,
"os": [
"linux"
@@ -12843,9 +12759,6 @@
"x64"
],
"dev": true,
"libc": [
"glibc"
],
"optional": true,
"os": [
"linux"
@@ -12859,9 +12772,6 @@
"x64"
],
"dev": true,
"libc": [
"musl"
],
"optional": true,
"os": [
"linux"
@@ -24477,6 +24387,17 @@
"node": ">= 14"
}
},
"node_modules/libxmljs2/node_modules/brace-expansion": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
"integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
"dev": true,
"license": "MIT",
"optional": true,
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/libxmljs2/node_modules/cacache": {
"version": "19.0.1",
"resolved": "https://registry.npmjs.org/cacache/-/cacache-19.0.1.tgz",

26
src/lib/modelsDevSync.ts
View File

@@ -20,6 +20,7 @@
import { getDbInstance } from "./db/core";
import { invalidateDbCache } from "./db/readCache";
import { backupDbFile } from "./db/backup";
import { registerDbStateResetter } from "./db/stateReset";
import {
transformModelsDevToPricing,
@@ -193,10 +194,32 @@ function mapCapabilityRecord(record: Record<string, unknown>): ModelCapabilityEn
};
}
/**
* Process-local memo for models.dev pricing.
*
* `resolveCatalogPricing` used to call `getModelsDevPricing()` once per model
* while building `/v1/models` (~10k+ times). Each call re-ran the full SQL scan
* and `JSON.parse`d every pricing row, pegging the event loop for minutes
* (see #9685 / #10052). Memoize until the next save/clear write.
*/
let modelsDevPricingCache: PricingByProvider | null = null;
function invalidateModelsDevPricingCache(): void {
modelsDevPricingCache = null;
}
// Register cache invalidation with DB state reset system so resetDbInstance() clears the memo.
registerDbStateResetter(invalidateModelsDevPricingCache);
/**
* Read synced pricing from `models_dev_pricing` namespace.
* Results are memoized until `saveModelsDevPricing` / `clearModelsDevPricing`.
*/
export function getModelsDevPricing(): PricingByProvider {
if (modelsDevPricingCache) {
return modelsDevPricingCache;
}
const db = getDbInstance();
const rows = db
.prepare("SELECT key, value FROM key_value WHERE namespace = 'models_dev_pricing'")
@@ -213,6 +236,7 @@ export function getModelsDevPricing(): PricingByProvider {
console.warn(`[MODELS_DEV] Corrupted pricing data for provider "${key}", skipping`);
}
}
modelsDevPricingCache = synced;
return synced;
}
@@ -233,6 +257,7 @@ export function saveModelsDevPricing(data: PricingByProvider): void {
});
tx();
backupDbFile("pre-write");
invalidateModelsDevPricingCache();
invalidateDbCache("pricing");
}
@@ -243,6 +268,7 @@ export function clearModelsDevPricing(): void {
const db = getDbInstance();
db.prepare("DELETE FROM key_value WHERE namespace = 'models_dev_pricing'").run();
backupDbFile("pre-write");
invalidateModelsDevPricingCache();
invalidateDbCache("pricing");
}

39
tests/unit/modelsDevSync-extended.test.ts
View File

@@ -260,6 +260,45 @@ test("modelsDev pricing helpers persist records, skip corrupted rows, and clear
assert.deepEqual(modelsDev.getModelsDevPricing(), {});
});
test("getModelsDevPricing memoizes until save/clear (#9685)", async () => {
const modelsDev = await importFresh("pricing-memo");
const pricing = modelsDev.transformModelsDevToPricing(MOCK_MODELS_DEV_DATA);
modelsDev.saveModelsDevPricing(pricing);
const first = modelsDev.getModelsDevPricing();
const second = modelsDev.getModelsDevPricing();
assert.equal(first, second, "repeated reads must return the same memoized object");
// Mutating DB under the cache must not be visible until invalidation.
const db = core.getDbInstance();
db.prepare("DELETE FROM key_value WHERE namespace = 'models_dev_pricing'").run();
assert.equal(
modelsDev.getModelsDevPricing(),
first,
"raw SQL without save/clear must not bypass the memo"
);
modelsDev.clearModelsDevPricing();
assert.deepEqual(modelsDev.getModelsDevPricing(), {});
modelsDev.saveModelsDevPricing(pricing);
const afterSave = modelsDev.getModelsDevPricing();
assert.notEqual(afterSave, first, "save must invalidate the memo");
assert.equal(afterSave.openai["gpt-4o"].input, 2.5);
// Copilot review: DB reset must invalidate the memo so import/restore doesn't serve stale pricing.
const beforeReset = modelsDev.getModelsDevPricing();
core.resetDbInstance();
const afterReset = modelsDev.getModelsDevPricing();
assert.notEqual(
afterReset,
beforeReset,
"resetDbInstance must invalidate the memo (Copilot #10055)"
);
// Data is still on disk after resetDbInstance(), but the cache was cleared and re-read from fresh DB.
assert.equal(afterReset.openai["gpt-4o"].input, 2.5, "DB reset re-reads from fresh connection");
});
test("modelsDev capabilities helpers create the table, persist rows, filter by provider/model, and expose context limits", async () => {
const modelsDev = await importFresh("capabilities-storage");
const capabilities = modelsDev.transformModelsDevToCapabilities(MOCK_MODELS_DEV_DATA);