diff --git a/open-sse/services/autoCombo/virtualFactory.ts b/open-sse/services/autoCombo/virtualFactory.ts index 87e1006a8e..45b0397208 100644 --- a/open-sse/services/autoCombo/virtualFactory.ts +++ b/open-sse/services/autoCombo/virtualFactory.ts @@ -23,6 +23,7 @@ import { import type { AutoVariant } from "./autoPrefix"; import { buildFamilyCandidateFilter, type ModelFamily } from "./modelFamily"; import { getHiddenModelsByProvider } from "@/models"; +import { getSyncedAvailableModelsByConnection, getCustomModels } from "@/lib/db/models"; import { filterPaidOnlyCandidates } from "./paidModelFilter"; import { isModelExcludedByConnection } from "@/domain/connectionModelRules"; import { filterExcludedCandidates } from "./candidateOverrides"; @@ -481,15 +482,41 @@ export async function prepareVirtualAutoComboInputs( const defaultModelIds = providerConnections .map((conn) => (typeof conn.defaultModel === "string" ? conn.defaultModel.trim() : "")) .filter(Boolean); - const modelIds = Array.from(new Set([...registryModelIds, ...defaultModelIds])); const hiddenModels = hiddenModelsMap.get(providerId); + // #auto-pool-visible-only: build the credentialed pool from the models the user + // actually has available (synced + custom non-hidden) when any exist, falling + // back to the static catalog only when the user has none. This keeps catalog-only + // models (e.g. openrouter/auto) out of every auto/* pool when the operator only + // synced a subset (e.g. OpenRouter with importFreeModelsOnly). + const [syncedByConnection, customModels] = await Promise.all([ + getSyncedAvailableModelsByConnection(providerId), + getCustomModels(providerId), + ]); + const userVisibleIds = new Set(); + for (const models of Object.values(syncedByConnection)) { + for (const m of models) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + } + for (const m of customModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + const hasUserModels = userVisibleIds.size > 0; + const modelIds = hasUserModels + ? Array.from(userVisibleIds) + : Array.from(new Set([...registryModelIds, ...defaultModelIds])); + for (const modelId of modelIds) { if (hiddenModels?.has(modelId)) continue; const allowedConnectionIds = providerConnections .filter((conn) => { if (isModelExcludedByConnection(modelId, conn.providerSpecificData)) return false; + if (hasUserModels) { + // User-synced models are scoped to the connections that carry them; + // custom models are provider-wide like registry models. + const connSynced = syncedByConnection[conn.id] ?? []; + const isSyncedForConn = connSynced.some((m) => m.id === modelId); + const isCustomForProvider = customModels.some((m) => m.id === modelId); + return isSyncedForConn || isCustomForProvider || conn.defaultModel?.trim() === modelId; + } // Registry models are provider-wide. A non-registry default (for a custom // or passthrough model) is scoped only to connections that selected it. return registryModelIdSet.has(modelId) || conn.defaultModel?.trim() === modelId; diff --git a/open-sse/services/combo/autoStrategy.ts b/open-sse/services/combo/autoStrategy.ts index cdb0124de2..a4a0c0dba2 100644 --- a/open-sse/services/combo/autoStrategy.ts +++ b/open-sse/services/combo/autoStrategy.ts @@ -39,6 +39,11 @@ import { } from "../autoCombo/scoring.ts"; import type { RoutingHint } from "../manifestAdapter"; import { getCachedProviderConnections } from "../../../src/lib/db/readCache"; +import { + getSyncedAvailableModels, + getCustomModels, + getHiddenModelsByProvider, +} from "../../../src/lib/db/models"; import { getProviderModels } from "../../config/providerModels.ts"; import { getConnectionRoutingTags, @@ -458,10 +463,27 @@ export async function expandAutoComboCandidatePool( // expansion doesn't turn into O(n^2) per provider. See #OOM incident // (zero-config auto combo expanding to 1000s of provider/model targets). const seenModelStrs = new Set(eligibleTargets.map((t) => t.modelStr)); + const hiddenModelsMap = getHiddenModelsByProvider(); for (const providerId of providerIds) { - const providerModels = getProviderModels(providerId); - for (const model of providerModels) { - const modelStr = `${providerId}/${model.id}`; + // #auto-pool-visible-only: when the operator has synced/custom models for + // this provider, expand ONLY those (minus hidden); fall back to the static + // catalog only when the user has none. This keeps catalog-only models + // (e.g. openrouter/auto) out of pure-auto pools when the operator only + // synced a subset (e.g. OpenRouter with importFreeModelsOnly). + const [syncedModels, customModels] = await Promise.all([ + getSyncedAvailableModels(providerId), + getCustomModels(providerId), + ]); + const hiddenModels = hiddenModelsMap.get(providerId); + const userVisibleIds = new Set(); + for (const m of syncedModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + for (const m of customModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id); + const hasUserModels = userVisibleIds.size > 0; + const expandIds = hasUserModels + ? Array.from(userVisibleIds) + : getProviderModels(providerId).map((m) => m.id); + for (const modelId of expandIds) { + const modelStr = `${providerId}/${modelId}`; if (!seenModelStrs.has(modelStr)) { seenModelStrs.add(modelStr); eligibleTargets.push({ diff --git a/tests/unit/combo-auto-pool-visible-only.test.ts b/tests/unit/combo-auto-pool-visible-only.test.ts new file mode 100644 index 0000000000..e0748ea77b --- /dev/null +++ b/tests/unit/combo-auto-pool-visible-only.test.ts @@ -0,0 +1,173 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// Regression coverage for the "auto combos must only pick user-visible models" +// fix (2026-08-15): a provider whose connection only has synced/free models +// (e.g. OpenRouter with importFreeModelsOnly) must NOT surface catalog-only +// models like `openrouter/auto` in any auto candidate pool. The pool must be +// built from what the user actually has visible (synced + custom non-hidden), +// falling back to the static catalog only when the user has no synced/custom +// models for that provider at all. + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auto-visible-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const virtualFactory = await import("../../open-sse/services/autoCombo/virtualFactory.ts"); +const combo = await import("../../open-sse/services/combo.ts"); + +function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.beforeEach(() => resetStorage()); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; +}); + +async function createOpenRouterConnectionWithFreeSync() { + const conn = await providersDb.createProviderConnection({ + provider: "openrouter", + authType: "apikey", + name: "OpenRouter", + apiKey: "sk-test-openrouter", + providerSpecificData: { importFreeModelsOnly: true }, + }); + const connectionId = (conn as { id?: string }).id; + assert.ok(connectionId, "created openrouter connection must expose an id"); + await modelsDb.replaceSyncedAvailableModelsForConnection("openrouter", connectionId, [ + { + id: "liquid/lfm-2.5-2.6b:free", + name: "LiquidAI: LFM2.5-2.6B (free)", + source: "imported" as const, + }, + { + id: "nvidia/nemotron-3.5-lightning:free", + name: "NVIDIA: Nemotron 3.5 Lightning (free)", + source: "imported" as const, + }, + ]); + return connectionId; +} + +test("virtual auto-combo pool excludes catalog-only models (openrouter/auto) when only free models are synced", async () => { + await createOpenRouterConnectionWithFreeSync(); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + const pool = prepared.regularCandidates; + assert.ok(pool.length > 0, "expected a non-empty pool for the active openrouter connection"); + + assert.ok( + !pool.some((c) => c.provider === "openrouter" && c.model === "auto"), + "openrouter/auto must NOT be a candidate: the user never synced it (catalog-only model)" + ); + + assert.ok( + pool.some((c) => c.provider === "openrouter" && c.model === "liquid/lfm-2.5-2.6b:free"), + "a synced free model must remain a candidate" + ); + assert.ok( + pool.some( + (c) => c.provider === "openrouter" && c.model === "nvidia/nemotron-3.5-lightning:free" + ), + "the second synced free model must remain a candidate" + ); +}); + +test("expandAutoComboCandidatePool excludes catalog-only models (openrouter/auto) when only free models are synced", async () => { + await createOpenRouterConnectionWithFreeSync(); + + const expanded = await combo.expandAutoComboCandidatePool([], { config: {} }); + assert.ok(expanded.length > 0, "expected expansion from the active openrouter connection"); + + assert.ok( + !expanded.some((t) => t.provider === "openrouter" && t.modelStr === "openrouter/auto"), + "expanded pool must NOT include openrouter/auto: the user never synced it" + ); + + assert.ok( + expanded.some((t) => t.provider === "openrouter" && t.modelStr === "openrouter/liquid/lfm-2.5-2.6b:free"), + "a synced free model must be expanded into the pool" + ); +}); + +test("virtual auto-combo pool falls back to the static catalog when the provider has no synced/custom models", async () => { + await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "OpenAI", + apiKey: "sk-test-openai", + defaultModel: "gpt-4o-mini", + }); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + const pool = prepared.regularCandidates; + const openaiCandidates = pool.filter((c) => c.provider === "openai"); + assert.ok( + openaiCandidates.length > 0, + "openai with no synced models must still get catalog candidates (fallback)" + ); + assert.ok( + openaiCandidates.some((c) => c.model === "gpt-4o-mini"), + "the configured default must remain among catalog-fallback candidates" + ); +}); +test("virtual auto-combo pool filters EVERY provider with partial sync, not just openrouter", async () => { + // openai: sync only gpt-4o-mini (gpt-4o and gpt-4o-turbo exist in the static + // catalog but are NOT synced → must be absent). kilocode: 359 synced models, + // all with the kilocode provider prefix in the static registry → must be the + // only kilocode candidates. + const openaiConn = await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + name: "OpenAI", + apiKey: "sk-test-openai", + }); + const kilocodeConn = await providersDb.createProviderConnection({ + provider: "kilocode", + authType: "apikey", + name: "KiloCode", + apiKey: "sk-test-kilocode", + }); + const openaiId = (openaiConn as { id?: string }).id; + const kilocodeId = (kilocodeConn as { id?: string }).id; + await modelsDb.replaceSyncedAvailableModelsForConnection("openai", openaiId, [ + { id: "gpt-4o-mini", name: "GPT-4o mini", source: "imported" as const }, + ]); + await modelsDb.replaceSyncedAvailableModelsForConnection("kilocode", kilocodeId, [ + { id: "kilocode/gpt-oss-120b", name: "GPT-OSS 120B", source: "imported" as const }, + { id: "kilocode/qwen3-coder", name: "Qwen3 Coder", source: "imported" as const }, + ]); + + const prepared = await virtualFactory.prepareVirtualAutoComboInputs(); + const pool = prepared.regularCandidates; + + const openaiCandidates = pool.filter((c) => c.provider === "openai"); + assert.ok( + openaiCandidates.some((c) => c.model === "gpt-4o-mini"), + "synced openai model must be a candidate" + ); + assert.ok( + !openaiCandidates.some((c) => c.model !== "gpt-4o-mini"), + `only the synced openai model may be a candidate, got: ${openaiCandidates.map((c) => c.model).join(", ")}` + ); + + const kilocodeCandidates = pool.filter((c) => c.provider === "kilocode"); + assert.deepEqual( + kilocodeCandidates.map((c) => c.model).sort(), + ["kilocode/gpt-oss-120b", "kilocode/qwen3-coder"], + "kilocode pool must contain exactly the two synced models" + ); +});