perf(startup): warm model catalog cache at module init (#6920)

* perf(startup): warm model catalog cache at module init Fire-and-forget call to getUnifiedModelsResponse after DB ready so first GET /v1/models request doesn't pay cold-build cost (15-30s). Non-fatal — if warmup fails the next request builds fresh. * test: add warm catalog cache source-pattern test Verifies registerNodejs() includes the model catalog warmup import and call to getUnifiedModelsResponse. * fix(perf): warm the durable OpenRouter catalog cache, not just the 1.5s TTL Response cache The warmup called getUnifiedModelsResponse() with no Authorization header, so it only ever populated the top-level per-key Response cache (catalogCache in catalog.ts) at key "|0|" — a real client sending an apiKey gets a different key and misses that cache entry. But that cache also has only a 1.5s TTL (CATALOG_CACHE_TTL_MS, a #6408 burst-dedup window for concurrent requests, not a startup-warm cache), so even a perfectly key-matched entry would almost always have expired before real traffic arrives regardless. The one genuinely durable, apiKey-independent cost in the catalog build is getOpenRouterCatalog()'s 24h disk-cached network fetch (src/lib/catalog/openrouterCatalog.ts) — buildUnifiedModelsResponseCore() calls it unconditionally whenever an OpenRouter connection is configured, fully decoupled from the per-key Response cache. Extract the warmup into an exported warmModelCatalogCache() (testable in isolation, without exercising all of registerNodejs()) that explicitly warms this cache too, guarded on an OpenRouter connection actually existing so deployments that never use OpenRouter don't pay an unconditional third-party network call at every boot. Replace the source-text-grep test with a behavioral one: warm once with a mocked fetch, confirm exactly one network call, then confirm a real request using a DIFFERENT apiKey than the warmup reuses the cache instead of re-fetching — the actual, durable, apiKey-independent benefit. Also covers the no-connection-configured and fetch-failure-is-non-fatal cases. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: oyi77 <oyi77@users.noreply.github.com> Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
2026-07-26 09:52:11 +03:00 · 2026-07-18 21:33:44 +07:00
parent 6e71553797
commit e79de5c294
2 changed files with 220 additions and 0 deletions
--- a/src/instrumentation-node.ts
+++ b/src/instrumentation-node.ts
@@ -144,6 +144,59 @@ async function ensureSecrets(): Promise<void> {
  }
 }

+/**
+ * Warm the model catalog's durable, apiKey-independent sub-caches at startup
+ * so real /v1/models traffic (any client, any API key) avoids paying their
+ * cold-build cost on first use. Fire-and-forget from the caller, non-fatal.
+ *
+ * getUnifiedModelsResponse()'s own top-level Response cache (`catalogCache`
+ * in catalog.ts) is keyed by prefix/isCodex/apiKey AND has only a 1.5s TTL
+ * (CATALOG_CACHE_TTL_MS — a burst-dedup window added for #6408 to coalesce
+ * concurrent SDK/dashboard requests, not a startup-warm cache). Warming that
+ * cache with an unauthenticated dummy request has essentially no lasting
+ * effect: real traffic almost never arrives within 1.5s of this warmup
+ * completing, regardless of whether its cache key happens to match a real
+ * client's apiKey. The one genuinely durable, apiKey-independent cost in the
+ * catalog build is getOpenRouterCatalog()'s 24h disk-cached network fetch
+ * (src/lib/catalog/openrouterCatalog.ts) — buildUnifiedModelsResponseCore()
+ * calls it unconditionally whenever an OpenRouter connection is configured,
+ * decoupled from the per-key Response cache entirely, so warming it directly
+ * here benefits every subsequent /v1/models request regardless of that
+ * request's own apiKey. Only fetched when an OpenRouter connection actually
+ * exists, so deployments that never use OpenRouter don't pay an unconditional
+ * third-party network call at every boot.
+ *
+ * Exported (rather than left inline in registerNodejs()) so it can be unit
+ * tested directly without exercising the rest of the startup sequence.
+ */
+export async function warmModelCatalogCache(): Promise<void> {
+  try {
+    const { getUnifiedModelsResponse } = await import("@/app/api/v1/models/catalog");
+    await getUnifiedModelsResponse(new Request("http://127.0.0.1/v1/models"));
+    console.log("[STARTUP] Model catalog cache warmed");
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn("[STARTUP] Model catalog warmup failed (non-fatal):", msg);
+  }
+  try {
+    const [{ getProviderConnections }, { getOpenRouterCatalog }] = await Promise.all([
+      import("@/lib/db/providers"),
+      import("@/lib/catalog/openrouterCatalog"),
+    ]);
+    const openrouterConnections = await getProviderConnections({
+      provider: "openrouter",
+      isActive: true,
+    });
+    if (openrouterConnections.length > 0) {
+      await getOpenRouterCatalog();
+      console.log("[STARTUP] OpenRouter model catalog cache warmed");
+    }
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn("[STARTUP] OpenRouter catalog warmup failed (non-fatal):", msg);
+  }
+}
+
 export async function registerNodejs(): Promise<void> {
  // Rename the process title so OmniRoute is identifiable in ps/htop instead
  // of the generic "next-server" standalone server name.
@@ -358,6 +411,11 @@ export async function registerNodejs(): Promise<void> {
    console.warn("[STARTUP] Could not initialize vacuum scheduler (non-fatal):", msg);
  }

+  // Warm the model catalog's durable, apiKey-independent sub-caches at
+  // startup — see warmModelCatalogCache() for why the top-level Response
+  // cache alone doesn't deliver this. Fire-and-forget, non-fatal.
+  void warmModelCatalogCache();
+
  if (!isBackgroundServicesDisabled()) {
    try {
      const { bootstrapEmbeddedServices } = await import("@/lib/services/bootstrap");
--- a/tests/unit/instrumentation-warm-catalog-cache.test.ts
+++ b/tests/unit/instrumentation-warm-catalog-cache.test.ts
@@ -0,0 +1,162 @@
+/**
+ * Behavioral regression test for warmModelCatalogCache() (src/instrumentation-node.ts).
+ *
+ * The original PR warmed getUnifiedModelsResponse()'s top-level Response
+ * cache with an unauthenticated dummy request. That cache (`catalogCache` in
+ * catalog.ts) is keyed by prefix/isCodex/apiKey AND has only a 1.5s TTL
+ * (CATALOG_CACHE_TTL_MS — a #6408 burst-dedup window for concurrent SDK/
+ * dashboard requests, not a startup-warm cache), so warming it has no lasting
+ * effect once real traffic (which almost never arrives within 1.5s of boot,
+ * and which typically DOES send an Authorization header — a different cache
+ * key) starts flowing.
+ *
+ * The one genuinely durable, apiKey-independent cost in the catalog build is
+ * getOpenRouterCatalog()'s 24h disk-cached network fetch — buildUnifiedModelsResponseCore()
+ * calls it unconditionally whenever an OpenRouter connection is configured,
+ * decoupled entirely from the per-key Response cache. warmModelCatalogCache()
+ * now warms that explicitly. These tests prove:
+ *   1. warmup fetches the OpenRouter catalog exactly once when a connection exists,
+ *   2. a REAL request using a DIFFERENT apiKey than the warmup afterwards reuses
+ *      that warmed cache instead of re-fetching (the actual claimed benefit),
+ *   3. warmup makes no OpenRouter network call at all when no connection is configured,
+ *   4. warmup never rejects even when the OpenRouter fetch fails.
+ *
+ * mock.module() is unavailable in this tsx/ESM + Node native test-runner setup
+ * (see tests/unit/proxyfetch-undici-retry.test.ts), so globalThis.fetch — a
+ * plain mutable global, not a frozen ESM binding — is monkey-patched directly
+ * (same pattern as Math.random in tests/unit/apikey-connection-health-check.test.ts).
+ */
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+process.env.NODE_ENV = "test";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-warm-catalog-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+
+const core = await import("../../src/lib/db/core.ts");
+const providersDb = await import("../../src/lib/db/providers.ts");
+const { warmModelCatalogCache } = await import("../../src/instrumentation-node.ts");
+const { getUnifiedModelsResponse } = await import("../../src/app/api/v1/models/catalog.ts");
+
+async function resetStorage() {
+  core.resetDbInstance();
+  for (let attempt = 0; attempt < 10; attempt++) {
+    try {
+      if (fs.existsSync(TEST_DATA_DIR)) {
+        fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+      }
+      break;
+    } catch (error) {
+      const code = (error as { code?: string } | undefined)?.code;
+      if ((code === "EBUSY" || code === "EPERM") && attempt < 9) {
+        await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1)));
+      } else {
+        throw error;
+      }
+    }
+  }
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+test.after(async () => {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+});
+
+const REAL_FETCH = globalThis.fetch;
+let fetchCallCount = 0;
+
+function installFakeOpenRouterFetch(): void {
+  fetchCallCount = 0;
+  globalThis.fetch = (async () => {
+    fetchCallCount++;
+    return new Response(JSON.stringify({ data: [{ id: "test/fake-model", architecture: {} }] }), {
+      status: 200,
+      headers: { "content-type": "application/json" },
+    });
+  }) as typeof fetch;
+}
+
+function installFailingOpenRouterFetch(): void {
+  fetchCallCount = 0;
+  globalThis.fetch = (async () => {
+    fetchCallCount++;
+    throw new Error("simulated OpenRouter network failure");
+  }) as typeof fetch;
+}
+
+function restoreRealFetch(): void {
+  globalThis.fetch = REAL_FETCH;
+}
+
+test("warmModelCatalogCache warms the OpenRouter catalog once, and a real request with a different apiKey reuses it", async () => {
+  await resetStorage();
+  installFakeOpenRouterFetch();
+  try {
+    await providersDb.createProviderConnection({
+      provider: "openrouter",
+      authType: "apikey",
+      name: "test-openrouter",
+      apiKey: "sk-or-test",
+      isActive: true,
+    });
+
+    await warmModelCatalogCache();
+    assert.equal(fetchCallCount, 1, "warmup should fetch the OpenRouter catalog exactly once");
+
+    // A real client authenticating with a DIFFERENT apiKey than the warmup's
+    // anonymous request must NOT re-trigger the network fetch — this is the
+    // actual, durable, apiKey-independent benefit warmModelCatalogCache()
+    // delivers, unlike the top-level per-key Response cache (1.5s TTL).
+    const realReq = new Request("http://127.0.0.1/v1/models", {
+      headers: { Authorization: "Bearer sk-a-totally-different-real-client-key" },
+    });
+    await (await getUnifiedModelsResponse(realReq)).text();
+    assert.equal(
+      fetchCallCount,
+      1,
+      "a real request with a different apiKey should reuse the warmed OpenRouter cache, not re-fetch"
+    );
+  } finally {
+    restoreRealFetch();
+  }
+});
+
+test("warmModelCatalogCache makes no OpenRouter network call when no OpenRouter connection is configured", async () => {
+  await resetStorage();
+  installFakeOpenRouterFetch();
+  try {
+    // No openrouter connection created — warmup must not make an
+    // unconditional third-party network call for deployments that never use it.
+    await warmModelCatalogCache();
+    assert.equal(
+      fetchCallCount,
+      0,
+      "warmup should not call the OpenRouter API without a configured connection"
+    );
+  } finally {
+    restoreRealFetch();
+  }
+});
+
+test("warmModelCatalogCache never rejects, even when the OpenRouter fetch fails", async () => {
+  await resetStorage();
+  await providersDb.createProviderConnection({
+    provider: "openrouter",
+    authType: "apikey",
+    name: "test-openrouter-failing",
+    apiKey: "sk-or-test-fail",
+    isActive: true,
+  });
+  installFailingOpenRouterFetch();
+  try {
+    await assert.doesNotReject(() => warmModelCatalogCache());
+    assert.ok(fetchCallCount > 0, "precondition: the fetch was actually attempted and failed");
+  } finally {
+    restoreRealFetch();
+  }
+});