From 6647654bcad2d57b96b850dab9bbc6751a9dcda9 Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Mon, 14 Sep 2026 19:05:59 -0300 Subject: [PATCH] fix(models): resolve auto-combo target metadata once per catalog build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #12046 derives vision/modalities for the built-in auto/* combos by resolving catalog metadata for every target of every combo. The ~40 auto combos share one candidate pool and the loop neither memoized nor yielded, so the #9147 fixture (60 connections, 720 synced models) went from a ~4s build with a 167ms longest event-loop gap to ~11s and a 860-1070ms gap on an idle box — past the 800ms contract and past #12628's 8s cold-build bound, which is why the test came back 500 catalog_build_timeout on every release-green run. Metadata depends only on the target's provider/model/connection scope within a build, so memoize it per build and yield between misses. Same fixture: 2.3-3.1s build, 56-72ms longest gap. The 9147 test is unchanged. Refs #12732 --- ...12732-catalog-auto-target-metadata-memo.md | 1 + src/app/api/v1/models/catalog.ts | 4 +- src/app/api/v1/models/catalogHelpers.ts | 33 ++++++++++ .../catalog-target-metadata-memo-9147.test.ts | 60 +++++++++++++++++++ 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md create mode 100644 tests/unit/catalog-target-metadata-memo-9147.test.ts diff --git a/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md b/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md new file mode 100644 index 0000000000..60a0da256d --- /dev/null +++ b/changelog.d/fixes/12732-catalog-auto-target-metadata-memo.md @@ -0,0 +1 @@ +- **fix(models):** a cold `GET /v1/models` on a large deployment no longer blocks the event loop for about a second at a time or overruns the 8s cold-build bound: since #12046 the built-in `auto/*` combos resolved catalog metadata for every target of every combo without memoizing or yielding, and they all draw on the same candidate pool, so 720 synced models took the build from ~4s to ~18s. Each distinct target is now resolved once per build, with a yield between misses ([#12732](https://github.com/diegosouzapw/OmniRoute/issues/12732)) diff --git a/src/app/api/v1/models/catalog.ts b/src/app/api/v1/models/catalog.ts index 87991da4d3..0098980f46 100644 --- a/src/app/api/v1/models/catalog.ts +++ b/src/app/api/v1/models/catalog.ts @@ -104,6 +104,7 @@ import { mergeComboCapabilities, getConnectionScopedEffortTiers, type ConnectionScopedReasoningCatalog, + memoizeTargetMetadata, } from "./catalogHelpers"; import { qualifyOpenRouterModelId, @@ -848,6 +849,7 @@ async function buildUnifiedModelsResponseCore( // catalog build. Runtime auto routing still prepares fresh request-scoped inputs. let preparedAutoInputs: Awaited> | undefined; let materializedAutoCount = 0; + const autoMeta = memoizeTargetMetadata(getComboTargetCatalogMetadata, maybeYieldCatalogBuild); for (const autoId of [ ...Object.keys(AUTO_TEMPLATE_VARIANTS), ...AUTO_SUFFIX_VARIANTS, @@ -890,7 +892,7 @@ async function buildUnifiedModelsResponseCore( connectionId: m.connectionId, ...(m.allowedConnectionIds ? { allowedConnectionIds: m.allowedConnectionIds } : {}), })); - const autoTargetMetadata = autoTargets.map((t) => getComboTargetCatalogMetadata(t)); + const autoTargetMetadata = await autoMeta(autoTargets); // #9147: once per build const knownAutoMeta = autoTargetMetadata.filter( (m): m is ComboTargetCatalogMetadata => m !== null ); diff --git a/src/app/api/v1/models/catalogHelpers.ts b/src/app/api/v1/models/catalogHelpers.ts index 4973a37e89..12b9815b8a 100644 --- a/src/app/api/v1/models/catalogHelpers.ts +++ b/src/app/api/v1/models/catalogHelpers.ts @@ -222,3 +222,36 @@ export function mergeComboCapabilities( } return capabilities; } + +/** + * Memoize per-target catalog metadata for one catalog build, yielding between misses. + * #12046 resolves metadata for every target of every built-in `auto/*` combo, and those + * ~40 combos draw on the same candidate pool: unmemoized, the build repeated the same + * lookups tens of thousands of times without yielding (#9147 — 720 synced models took the + * cold build from ~4s to ~18s, past the 8s cold-build bound). Metadata depends only on + * the target fields in the key, so each distinct target is resolved once per build. + */ +export function memoizeTargetMetadata( + resolve: (target: ComboCatalogTarget) => T | null, + afterMiss: () => Promise +): (targets: ComboCatalogTarget[]) => Promise> { + const byKey = new Map(); + return async (targets) => { + const resolved: Array = []; + for (const target of targets) { + const key = JSON.stringify([ + target.providerId ?? null, + target.provider ?? null, + target.modelStr ?? null, + target.connectionId ?? null, + target.allowedConnectionIds ?? null, + ]); + if (!byKey.has(key)) { + byKey.set(key, resolve(target)); + await afterMiss(); + } + resolved.push(byKey.get(key) ?? null); + } + return resolved; + }; +} diff --git a/tests/unit/catalog-target-metadata-memo-9147.test.ts b/tests/unit/catalog-target-metadata-memo-9147.test.ts new file mode 100644 index 0000000000..3567d4f478 --- /dev/null +++ b/tests/unit/catalog-target-metadata-memo-9147.test.ts @@ -0,0 +1,60 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { memoizeTargetMetadata } from "../../src/app/api/v1/models/catalogHelpers.ts"; + +// #9147 / #12046: the built-in auto/* combos all resolve metadata for the same candidate +// pool, so the catalog build must resolve each distinct target once and yield between misses. + +test("memoizeTargetMetadata resolves each distinct target once across calls", async () => { + const resolved: string[] = []; + let yields = 0; + const resolveTargets = memoizeTargetMetadata( + (target) => { + resolved.push(`${target.providerId}/${target.modelStr}`); + return target.modelStr === "missing" ? null : { id: target.modelStr }; + }, + async () => { + yields++; + } + ); + const pool = [ + { providerId: "openai", modelStr: "gpt-a" }, + { providerId: "openai", modelStr: "missing" }, + { providerId: "claude", modelStr: "gpt-a" }, + ]; + + const first = await resolveTargets(pool); + const second = await resolveTargets([...pool].reverse()); + + assert.deepEqual(first, [{ id: "gpt-a" }, null, { id: "gpt-a" }]); + assert.deepEqual(second, [{ id: "gpt-a" }, null, { id: "gpt-a" }]); + assert.deepEqual(resolved, ["openai/gpt-a", "openai/missing", "claude/gpt-a"]); + assert.equal(yields, 3, "one yield per cache miss, none on hits"); +}); + +test("memoizeTargetMetadata keys on connection scope, not just provider/model", async () => { + let calls = 0; + const resolveTargets = memoizeTargetMetadata( + (target) => { + calls++; + return { scope: target.connectionId ?? target.allowedConnectionIds?.join(",") ?? "any" }; + }, + async () => {} + ); + + const result = await resolveTargets([ + { providerId: "openai", modelStr: "gpt-a" }, + { providerId: "openai", modelStr: "gpt-a", connectionId: "conn-1" }, + { providerId: "openai", modelStr: "gpt-a", allowedConnectionIds: ["conn-1", "conn-2"] }, + { providerId: "openai", modelStr: "gpt-a", connectionId: "conn-1" }, + ]); + + assert.deepEqual(result, [ + { scope: "any" }, + { scope: "conn-1" }, + { scope: "conn-1,conn-2" }, + { scope: "conn-1" }, + ]); + assert.equal(calls, 3); +});