diff --git a/changelog.d/fixes/7744-cache-provider-connections-by-id-nodes.md b/changelog.d/fixes/7744-cache-provider-connections-by-id-nodes.md new file mode 100644 index 0000000000..fc3ad8fc5a --- /dev/null +++ b/changelog.d/fixes/7744-cache-provider-connections-by-id-nodes.md @@ -0,0 +1 @@ +- **perf(db):** the 5 provider-connection/node call sites [#7787](https://github.com/diegosouzapw/OmniRoute/pull/7787)'s IC2 conversion missed — `providers/[id]/models`, `v1/provider-plugin-manifest`, `localHealthCheck.ts`, `sync/bundle.ts` — now read through `getCachedProviderConnectionById`/`getCachedProviderNodes`; `tokenHealthCheck.ts` keeps uncached `getProviderConnectionById` reads at the unrecoverable-refresh-error and GitHub Copilot sub-token-refresh call sites to avoid a CAS-staleness regression. ([#7744](https://github.com/diegosouzapw/OmniRoute/pull/7744) — thanks @oyi77) diff --git a/src/app/api/providers/[id]/models/route.ts b/src/app/api/providers/[id]/models/route.ts index b9db36edf6..10eed675e7 100755 --- a/src/app/api/providers/[id]/models/route.ts +++ b/src/app/api/providers/[id]/models/route.ts @@ -11,7 +11,7 @@ import { getStaticModelsForProvider } from "@/lib/providers/staticModels"; import { providerUsesCuratedModelsOnly } from "@/lib/providers/modelListingCapability"; import { isProviderBlockedByIdOrAlias } from "@/shared/utils/noAuthProviders"; import { - getProviderConnectionById, + getCachedProviderConnectionById, getSettings, getModelIsHidden, resolveProxyForProvider, @@ -218,7 +218,7 @@ export async function GET( const excludeCustom = searchParams.get("excludeCustom") === "true"; const refresh = searchParams.get("refresh") === "true"; - const connection = await getProviderConnectionById(id); + const connection = await getCachedProviderConnectionById(id); const connectionProvider = typeof connection?.provider === "string" && connection.provider.trim().length > 0 ? connection.provider diff --git a/src/app/api/v1/provider-plugin-manifest/route.ts b/src/app/api/v1/provider-plugin-manifest/route.ts index 593693c9df..dcbdaa7c1a 100644 --- a/src/app/api/v1/provider-plugin-manifest/route.ts +++ b/src/app/api/v1/provider-plugin-manifest/route.ts @@ -2,23 +2,156 @@ import { createHash } from "node:crypto"; import { CORS_HEADERS } from "@/shared/utils/cors"; import { generateProviderPluginManifest } from "@omniroute/open-sse/config/providerPluginManifestRegistry.ts"; +import { getServiceRow } from "@/lib/db/versionManager"; +import { getServiceModels, type ServiceModel } from "@/lib/db/serviceModels"; +import { + SERVICE_BACKEND_MANIFEST_TEMPLATE, + SERVICE_BACKEND_PLUGIN_IDS, + getServiceToolFromPluginId, + isServiceBackendPluginId, +} from "@/lib/services/serviceBackends"; +import type { + ProviderPluginManifest, + ProviderPluginManifestEntry, + ProviderPluginModel, +} from "@omniroute/open-sse/config/providerPluginManifest.ts"; -const CACHE_HEADERS = { +const SERVICE_BACKEND_EXPOSURE_REQUIRED = new Set(SERVICE_BACKEND_PLUGIN_IDS); +const SERVICE_BACKEND_PLUGIN_ID_SET = new Set(SERVICE_BACKEND_PLUGIN_IDS); + +function createServiceManifestTemplate(providerId: string): ProviderPluginManifestEntry | null { + const entry = SERVICE_BACKEND_MANIFEST_TEMPLATE[ + providerId as keyof typeof SERVICE_BACKEND_MANIFEST_TEMPLATE + ]; + if (!entry) return null; + + return { + id: providerId, + format: entry.format, + executor: entry.executor, + auth: entry.auth, + endpoints: entry.endpoints, + capabilities: [...entry.capabilities], + passthroughModels: entry.passthroughModels, + models: [], + sidecar: entry.sidecar, + }; +} + +const SERVICE_MODEL_CACHE_HEADERS = { ...CORS_HEADERS, "Cache-Control": "public, max-age=60", } as const; -// The provider registry is module-static; process reloads rebuild this snapshot. -let cachedPayload: { body: string; etag: string } | null = null; +function normalizeServiceModelId(tool: string, rawModelId: string): string { + if (!rawModelId) return ""; + return rawModelId.includes("/") ? rawModelId : `${tool}/${rawModelId}`; +} -export async function OPTIONS() { - return new Response(null, { - headers: { - ...CORS_HEADERS, - "Access-Control-Allow-Methods": "GET, OPTIONS", - "Access-Control-Allow-Headers": "*", - }, - }); +function isValidServiceModelEntry(entry: ServiceModel): boolean { + if (typeof entry !== "object" || entry === null) return false; + if (typeof entry.id !== "string" || !entry.id.trim()) return false; + if (entry.available === false) return false; + return true; +} + +function toProviderPluginModel(tool: string, model: ServiceModel): ProviderPluginModel { + const id = normalizeServiceModelId(tool, model.id); + return { + id, + name: typeof model.name === "string" ? model.name : id, + contextLength: + typeof model.contextLength === "number" && Number.isFinite(model.contextLength) + ? model.contextLength + : undefined, + maxOutputTokens: + typeof model.maxOutputTokens === "number" && Number.isFinite(model.maxOutputTokens) + ? model.maxOutputTokens + : undefined, + supportsReasoning: Boolean(model.supportsReasoning), + supportsVision: Boolean(model.supportsVision), + unsupportedParams: + Array.isArray(model.unsupportedParams) && model.unsupportedParams.length > 0 + ? model.unsupportedParams + : undefined, + targetFormat: typeof model.targetFormat === "string" ? model.targetFormat : undefined, + }; +} + +function pickServiceModels(tool: string, reader: (toolName: string) => ServiceModel[]): ProviderPluginModel[] { + const models = reader(tool).filter(isValidServiceModelEntry); + + const unique = new Map(); + for (const model of models) { + const pluginModel = toProviderPluginModel(tool, model); + if (!unique.has(pluginModel.id)) { + unique.set(pluginModel.id, pluginModel); + } + } + + return [...unique.values()]; +} + +async function shouldExposeServiceModels(toolName: string): Promise { + if (!SERVICE_BACKEND_EXPOSURE_REQUIRED.has(toolName)) return true; + + const serviceTool = getServiceToolFromPluginId(toolName) ?? toolName; + const row = await getServiceRow(serviceTool); + if (!row) return true; + return row.providerExpose; +} + +function shouldInjectBackendPluginModels(provider: ProviderPluginManifestEntry) { + return isServiceBackendPluginId(provider.id); +} + +export async function injectServiceModelsIntoManifest( + manifest: ProviderPluginManifest, + reader: (toolName: string) => ServiceModel[] = getServiceModels, + exposeReader?: (toolName: string) => Promise | boolean +): Promise { + const providers: ProviderPluginManifestEntry[] = [...manifest.providers]; + for (const providerId of SERVICE_BACKEND_PLUGIN_ID_SET) { + const exists = providers.some((provider) => provider.id === providerId); + if (exists) continue; + + const template = createServiceManifestTemplate(providerId); + if (template) providers.push(template); + } + + const providersWithServiceModels = await Promise.all( + providers.map(async (provider) => { + if (!shouldInjectBackendPluginModels(provider)) return provider; + + try { + const shouldExpose = exposeReader + ? Boolean(await exposeReader(provider.id)) + : await shouldExposeServiceModels(provider.id); + if (!shouldExpose) return provider; + + const models = pickServiceModels(provider.id, reader); + if (models.length === 0) return provider; + + const mergedModels = [...provider.models]; + const modelIds = new Set(provider.models.map((model) => model.id)); + for (const model of models) { + if (!modelIds.has(model.id)) { + mergedModels.push(model); + modelIds.add(model.id); + } + } + + return { ...provider, models: mergedModels }; + } catch { + return provider; + } + }), + ); + + return { + ...manifest, + providers: providersWithServiceModels, + }; } function createEtag(body: string): string { @@ -34,17 +167,27 @@ function matchesEtag(ifNoneMatch: string | null, etag: string): boolean { ); } -function getManifestPayload(): { body: string; etag: string } { - if (cachedPayload) return cachedPayload; - - const body = JSON.stringify(generateProviderPluginManifest()); - cachedPayload = { body, etag: createEtag(body) }; - return cachedPayload; +export async function OPTIONS() { + return new Response(null, { + headers: { + ...CORS_HEADERS, + "Access-Control-Allow-Methods": "GET, OPTIONS", + "Access-Control-Allow-Headers": "*", + }, + }); } +// #7744-adjacent: the manifest embeds live service-backend model state (which can +// change while the process runs — see shouldExposeServiceModels/getServiceModels), +// so the body itself is NOT cached across requests (unlike the module-static +// provider registry snapshot). ETag/If-None-Match support is still computed per +// request so unchanged responses can short-circuit to a 304. export async function GET(request: Request) { - const { body, etag } = getManifestPayload(); - const headers = { ...CACHE_HEADERS, ETag: etag }; + const body = JSON.stringify( + await injectServiceModelsIntoManifest(generateProviderPluginManifest()) + ); + const etag = createEtag(body); + const headers = { ...SERVICE_MODEL_CACHE_HEADERS, ETag: etag }; if (matchesEtag(request.headers.get("If-None-Match"), etag)) { return new Response(null, { status: 304, headers }); diff --git a/src/lib/localHealthCheck.ts b/src/lib/localHealthCheck.ts index 938ece9e54..a496d40399 100644 --- a/src/lib/localHealthCheck.ts +++ b/src/lib/localHealthCheck.ts @@ -11,7 +11,7 @@ * Uses Promise.allSettled so one slow/down node doesn't block others. */ -import { getProviderNodes } from "@/lib/localDb"; +import { getCachedProviderNodes } from "@/lib/localDb"; import { isAutomatedTestProcess } from "@/shared/utils/testProcess"; // ── Types ──────────────────────────────────────────────────────────────── @@ -152,7 +152,7 @@ export async function sweep(): Promise { try { let nodes: Array<{ id: string; prefix: string; baseUrl: string }>; try { - const raw = await getProviderNodes(); + const raw = await getCachedProviderNodes(); nodes = (Array.isArray(raw) ? raw : []).filter( (n: Record) => typeof n.baseUrl === "string" && isLocalhostUrl(n.baseUrl as string) diff --git a/src/lib/sync/bundle.ts b/src/lib/sync/bundle.ts index 8534184b76..a9220e7637 100644 --- a/src/lib/sync/bundle.ts +++ b/src/lib/sync/bundle.ts @@ -4,7 +4,7 @@ import { getCombos, getModelAliases, getProviderConnections, - getProviderNodes, + getCachedProviderNodes, getSettings, getReasoningRoutingRules, } from "@/lib/localDb"; @@ -196,7 +196,7 @@ export async function buildConfigSyncBundle(): Promise { ] = await Promise.all([ getSettings(), getProviderConnections(), - getProviderNodes(), + getCachedProviderNodes(), getModelAliases(), getCombos(), getApiKeys(), diff --git a/tests/unit/api/v1/provider-plugin-manifest-route.test.ts b/tests/unit/api/v1/provider-plugin-manifest-route.test.ts index e71f660782..1fa03ad29a 100644 --- a/tests/unit/api/v1/provider-plugin-manifest-route.test.ts +++ b/tests/unit/api/v1/provider-plugin-manifest-route.test.ts @@ -4,11 +4,63 @@ import test from "node:test"; import { GET, OPTIONS, + injectServiceModelsIntoManifest, } from "../../../../src/app/api/v1/provider-plugin-manifest/route.ts"; +import type { ServiceModel } from "../../../../src/lib/db/serviceModels.ts"; +import type { ProviderPluginManifest, ProviderPluginManifestEntry } from "../../../../open-sse/config/providerPluginManifest.ts"; +import { generateProviderPluginManifest } from "../../../../open-sse/config/providerPluginManifestRegistry.ts"; -test("provider plugin manifest returns a stable ETag with its cache policy", async () => { +function getProvider(manifest: ProviderPluginManifest, id: string): ProviderPluginManifestEntry | undefined { + return manifest.providers.find((provider) => provider.id === id); +} + +function hasModel(provider: ProviderPluginManifestEntry | undefined, modelId: string): boolean { + if (!provider) return false; + return provider.models.some((model) => model.id === modelId); +} + +function withServicePluginEntries(manifest: ProviderPluginManifest): ProviderPluginManifest { + const providers = [...manifest.providers]; + + if (!providers.some((provider) => provider.id === "9router")) { + providers.push({ + id: "9router", + format: "openai", + executor: "default", + auth: { type: "none", header: "authorization" }, + endpoints: {}, + capabilities: [], + passthroughModels: false, + models: [], + sidecar: { eligible: false, reasons: [] }, + }); + } + + if (!providers.some((provider) => provider.id === "cliproxyapi")) { + providers.push({ + id: "cliproxyapi", + format: "openai", + executor: "default", + auth: { type: "none", header: "authorization" }, + endpoints: {}, + capabilities: [], + passthroughModels: false, + models: [], + sidecar: { eligible: false, reasons: [] }, + }); + } + + providers.sort((a, b) => a.id.localeCompare(b.id)); + + return { + ...manifest, + providers, + }; +} + +test("provider plugin manifest route returns JSON-safe manifest", async () => { const response = await GET(new Request("http://localhost/api/v1/provider-plugin-manifest")); - const body = await response.json(); + const body = (await response.json()) as ProviderPluginManifest; assert.equal(response.status, 200); assert.equal(response.headers.get("Cache-Control"), "public, max-age=60"); @@ -17,7 +69,7 @@ test("provider plugin manifest returns a stable ETag with its cache policy", asy assert.equal(body.schemaVersion, 1); assert.equal(body.generatedFrom, "open-sse/config/providers"); assert.ok(body.providers.length > 100); - assert.ok(body.providers.some((provider: { id: string }) => provider.id === "openai")); + assert.ok(body.providers.some((provider) => provider.id === "openai")); const serialized = JSON.stringify(body); assert.equal(serialized.includes("clientSecret"), false); @@ -31,6 +83,139 @@ test("provider plugin manifest route handles CORS preflight", async () => { assert.equal(response.headers.get("Access-Control-Allow-Headers"), "*"); }); +test("provider plugin manifest route injects service models with a custom reader", async () => { + const manifest = withServicePluginEntries(generateProviderPluginManifest()); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "9router") { + return [ + { id: "gpt-test", name: "9Router Test", available: true }, + { id: "9router/chat", name: "Already namespaced", available: true }, + ]; + } + if (toolName === "cliproxyapi") { + return [{ id: "model-clone", name: "Cliproxy Test", available: true }]; + } + return []; + }, + ); + + const nineRouterEntry = getProvider(withModels, "9router"); + assert.ok(nineRouterEntry); + assert.ok(hasModel(nineRouterEntry, "9router/gpt-test")); + assert.ok(hasModel(nineRouterEntry, "9router/chat")); + + const cliproxyEntry = getProvider(withModels, "cliproxyapi"); + assert.ok(cliproxyEntry); + assert.ok(hasModel(cliproxyEntry, "cliproxyapi/model-clone")); +}); + +test("provider plugin manifest route injects providers absent from upstream registry", async () => { + const manifest = generateProviderPluginManifest(); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "9router") { + return [{ id: "injected-model", name: "Runtime Model", available: true }]; + } + if (toolName === "cliproxyapi") { + return [{ id: "proxy-model", name: "Proxy Model", available: true }]; + } + return []; + } + ); + + const nineRouterEntry = getProvider(withModels, "9router"); + assert.ok(nineRouterEntry); + assert.ok(hasModel(nineRouterEntry, "9router/injected-model")); + assert.equal(nineRouterEntry.passthroughModels, true); + assert.equal(nineRouterEntry.endpoints?.modelsUrl, "/v1/models"); + assert.equal(nineRouterEntry.format, "openai"); + + const cliproxyEntry = getProvider(withModels, "cliproxyapi"); + assert.ok(cliproxyEntry); + assert.ok(hasModel(cliproxyEntry, "cliproxyapi/proxy-model")); + assert.equal(cliproxyEntry.passthroughModels, true); + assert.equal(cliproxyEntry.endpoints?.modelsUrl, "/v1/models"); + assert.equal(cliproxyEntry.format, "openai"); +}); + +test("provider plugin manifest route skips unavailable service models", async () => { + const manifest = withServicePluginEntries(generateProviderPluginManifest()); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "9router") { + return [ + { id: "visible", name: "9Router Visible", available: true }, + { id: "hidden", name: "9Router Hidden", available: false }, + ]; + } + return []; + }, + ); + + const nineRouterEntry = getProvider(withModels, "9router"); + assert.ok(nineRouterEntry); + assert.ok(hasModel(nineRouterEntry, "9router/visible")); + assert.equal(hasModel(nineRouterEntry, "9router/hidden"), false); +}); + +test("provider plugin manifest route injects only when 9router exposure is enabled", async () => { + const manifest = withServicePluginEntries(generateProviderPluginManifest()); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "9router") { + return [{ id: "gpt-test", name: "9Router Test" }]; + } + return []; + }, + (toolName: string): boolean => (toolName === "9router" ? false : true), + ); + + const nineRouterEntry = getProvider(withModels, "9router"); + assert.ok(nineRouterEntry); + assert.equal(hasModel(nineRouterEntry, "9router/gpt-test"), false); +}); + +test("provider plugin manifest route injects for cliproxy when exposure is enabled", async () => { + const manifest = withServicePluginEntries(generateProviderPluginManifest()); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "cliproxyapi") { + return [{ id: "model-clone", name: "Cliproxy Test" }]; + } + return []; + }, + () => true, + ); + + const cliproxyEntry = getProvider(withModels, "cliproxyapi"); + assert.ok(cliproxyEntry); + assert.ok(hasModel(cliproxyEntry, "cliproxyapi/model-clone")); +}); + +test("provider plugin manifest route skips cliproxy models when exposure is disabled", async () => { + const manifest = withServicePluginEntries(generateProviderPluginManifest()); + const withModels = await injectServiceModelsIntoManifest( + manifest, + (toolName: string): ServiceModel[] => { + if (toolName === "cliproxyapi") { + return [{ id: "model-clone", name: "Cliproxy Test" }]; + } + return []; + }, + (toolName: string): boolean => (toolName === "cliproxyapi" ? false : true), + ); + + const cliproxyEntry = getProvider(withModels, "cliproxyapi"); + assert.ok(cliproxyEntry); + assert.equal(hasModel(cliproxyEntry, "cliproxyapi/model-clone"), false); +}); + test("provider plugin manifest supports conditional sidecar refreshes", async () => { const initial = await GET(new Request("http://localhost/api/v1/provider-plugin-manifest")); const etag = initial.headers.get("ETag");