diff --git a/changelog.d/fixes/11089-chat-routing-synced-inventory.md b/changelog.d/fixes/11089-chat-routing-synced-inventory.md new file mode 100644 index 0000000000..922b96a659 --- /dev/null +++ b/changelog.d/fixes/11089-chat-routing-synced-inventory.md @@ -0,0 +1 @@ +- **fix(resilience):** filter chat connection selection by each connection's *synced* model inventory on multi-host self-hosted providers (`ollama-local`, `lm-studio`, `vllm`, …), so a request for a model only one host advertises is pinned to that host instead of failing over onto a host that never had it ([#11089](https://github.com/diegosouzapw/OmniRoute/issues/11089)) diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 28accfeae5..0a3239ea7f 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -432,7 +432,8 @@ "src/shared/components/analytics/charts.tsx": 1346, "src/shared/services/cliRuntime.ts": 1459, "src/sse/handlers/chat.ts": 2493, - "src/sse/services/auth.ts": 3260, + "src/sse/services/auth.ts": 3337, + "_rebaseline_2026_08_23_11186_synced_inventory_routing": "PR #11186 (pacocartones) own growth: src/sse/services/auth.ts 3260->3337 (+77, loadAdvertisedModelsForSelfHostedConnections + the modelNotAdvertised candidate-filter predicate — pins chat routing to the connection whose synced inventory actually advertises the model, fixing spurious model-not-found on multi-host self-hosted setups; at the existing credential-selection chokepoint, not extractable without splitting the selection flow). Covered by tests/unit/chat-routing-synced-inventory-11089.test.ts. Owner pre-authorized baseline bumps 2026-08-22.", "tests/unit/account-fallback-service.test.ts": 2044, "tests/unit/provider-validation-specialty.test.ts": 3880, "open-sse/executors/hyperagent.ts": 1334, diff --git a/src/domain/connectionModelRules.ts b/src/domain/connectionModelRules.ts index 316ade72d8..7831bbc84b 100644 --- a/src/domain/connectionModelRules.ts +++ b/src/domain/connectionModelRules.ts @@ -80,3 +80,26 @@ export function hasEligibleConnectionForModel( (connection) => !isModelExcludedByConnection(modelId, connection?.providerSpecificData) ); } + +/** + * #11089: does this connection's *synced* inventory advertise the model? + * + * Unlike `excludedModels` (a manually maintained denylist) this reads the + * per-connection catalog written by model discovery, so a multi-host local + * provider never routes a model to a host that never had it. Ids are matched + * with the same candidate semantics as the denylist (provider prefix and the + * `[1m]` extended-context suffix are tolerated), but never as wildcard + * patterns — a synced id is a literal. + * + * Fails OPEN on an empty inventory: a host that has not been synced yet is + * "unknown", not "does not have it". + */ +export function isModelAdvertisedByConnection( + modelId: unknown, + advertisedModelIds: ReadonlySet | null | undefined +): boolean { + if (!advertisedModelIds || advertisedModelIds.size === 0) return true; + if (typeof modelId !== "string" || modelId.trim().length === 0) return true; + + return getModelMatchCandidates(modelId).some((candidate) => advertisedModelIds.has(candidate)); +} diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index 35304a05e6..5d3d885178 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -106,8 +106,17 @@ import { resolveProviderId, NOAUTH_PROVIDERS, WEB_COOKIE_PROVIDERS, + isSelfHostedChatProvider, } from "@/shared/constants/providers"; -import { isModelExcludedByConnection } from "@/domain/connectionModelRules"; +import { + isModelExcludedByConnection, + isModelAdvertisedByConnection, +} from "@/domain/connectionModelRules"; +import { + getSyncedAvailableModelsByConnection, + SYNCED_AVAILABLE_MODELS_MALFORMED, + type SyncedAvailableModelsByConnection, +} from "@/lib/db/models"; import { isFreeModel } from "@/shared/utils/freeModels"; import { applySessionAffinityPin, @@ -1160,6 +1169,54 @@ function materializeConnection( }; } +/** + * #11089: load the per-connection synced model inventory for self-hosted chat + * providers so connection selection can drop hosts that never advertised the + * requested model. + * + * Scoped to SELF_HOSTED_CHAT_PROVIDER_IDS: those are the providers where one + * provider id fans out to several independent hosts with genuinely different + * inventories. Hosted providers share one catalog per provider, so filtering + * there would only add a DB read. + * + * Returns an empty map (= no filtering) when there is no model to match, when + * no candidate is self-hosted, or when the persisted rows are malformed — a + * partial read must never silently shrink the pool. + */ +async function loadAdvertisedModelsForSelfHostedConnections( + connections: ProviderConnectionView[], + requestedModel: string | null +): Promise>> { + const advertised = new Map>(); + if (!requestedModel) return advertised; + + const selfHostedProviders = new Set( + connections + .map((c) => c.provider) + .filter((p): p is string => typeof p === "string" && isSelfHostedChatProvider(p)) + ); + if (selfHostedProviders.size === 0) return advertised; + + await Promise.all( + [...selfHostedProviders].map(async (providerId) => { + let byConnection: SyncedAvailableModelsByConnection; + try { + byConnection = await getSyncedAvailableModelsByConnection(providerId); + } catch { + return; + } + // Malformed persisted rows: fail open for the whole provider. + if (byConnection[SYNCED_AVAILABLE_MODELS_MALFORMED]) return; + for (const [connectionId, models] of Object.entries(byConnection)) { + if (!Array.isArray(models) || models.length === 0) continue; + advertised.set(connectionId, new Set(models.map((m) => m.id))); + } + }) + ); + + return advertised; +} + /** * Get provider credentials from localDb * Filters out unavailable accounts and returns the selected account based on strategy @@ -1435,6 +1492,14 @@ export async function getProviderCredentials( let modelLockedCount = 0; let familyLockedCount = 0; const connectionFilterStatus = new Map(); + // #11089: multi-host self-hosted providers keep a per-connection synced + // inventory. Without it, a request can be routed to a host that never had + // the model, producing a spurious model-not-found instead of pinning to + // the host that does. Empty map = no inventory known = no filtering. + const advertisedModelsByConnection = await loadAdvertisedModelsForSelfHostedConnections( + connections, + requestedModel + ); // Filter out unavailable accounts and excluded connection let availableConnections = connections.filter((c) => { if (excludedConnectionIds.has(c.id)) { @@ -1445,6 +1510,13 @@ export async function getProviderCredentials( connectionFilterStatus.set(c.id, "modelExcluded"); return false; } + if ( + requestedModel && + !isModelAdvertisedByConnection(requestedModel, advertisedModelsByConnection.get(c.id)) + ) { + connectionFilterStatus.set(c.id, "modelNotAdvertised"); + return false; + } if (!allowSuppressedConnections) { if (!allowRateLimitedConnections && isAccountUnavailable(c.rateLimitedUntil)) { connectionFilterStatus.set(c.id, "rateLimited"); @@ -1510,6 +1582,7 @@ export async function getProviderCredentials( const codexScopeLimited = status === "codexScopeLimited"; const modelLocked = status === "modelLocked"; const modelExcluded = status === "modelExcluded"; + const modelNotAdvertised = status === "modelNotAdvertised"; if (excluded || rateLimited) { log.debug( "AUTH", @@ -1520,6 +1593,11 @@ export async function getProviderCredentials( "AUTH", ` → ${c.id?.slice(0, 8)} | excluded by per-account model rule for ${requestedModel}` ); + } else if (modelNotAdvertised) { + log.debug( + "AUTH", + ` → ${c.id?.slice(0, 8)} | synced inventory does not advertise ${requestedModel}` + ); } else if (terminalStatus) { log.debug( "AUTH", diff --git a/tests/unit/chat-routing-synced-inventory-11089.test.ts b/tests/unit/chat-routing-synced-inventory-11089.test.ts new file mode 100644 index 0000000000..441b14893b --- /dev/null +++ b/tests/unit/chat-routing-synced-inventory-11089.test.ts @@ -0,0 +1,184 @@ +/** + * tests/unit/chat-routing-synced-inventory-11089.test.ts + * + * #11089 — Chat routing ignores per-connection model inventory on multi-host + * local providers. + * + * One self-hosted provider (`ollama-local`) with TWO connections pointing at + * different hosts and DISJOINT synced inventories: + * + * studio (priority 1) → gemma3:4b, flux2-klein:9b + * jetson (priority 2) → gemma3:4b + * + * `getProviderCredentials` only ever consulted the manual `excludedModels` + * denylist, never the synced inventory persisted per connection, so a request + * for `flux2-klein:9b` could land on jetson — a host that never had the model. + * + * Cases: + * 1. Model advertised by only one connection → the other is never selected. + * 2. Higher-priority host cooling → must NOT preemptively fall to a host that + * lacks the model. + * 3. The advertising connection stays selectable. + * 4. Model advertised by both → both remain eligible (no over-filtering). + * 5. Provider with NO synced inventory at all → fail open, selection unchanged. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-chat-synced-11089-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const modelsDb = await import("../../src/lib/db/models.ts"); +const auth = await import("../../src/sse/services/auth.ts"); + +const PROVIDER = "ollama-local"; + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +async function createConnection(data: Record): Promise { + const created = (await providersDb.createProviderConnection(data)) as { id: string }; + return created.id; +} + +/** The connection id the selector handed back, or null if it returned no account. */ +function selectedConnectionId(selected: unknown): string | null { + if (!selected || typeof selected !== "object") return null; + const id = (selected as { connectionId?: unknown }).connectionId; + return typeof id === "string" ? id : null; +} + +/** Create the two-host ollama-local topology from the issue report. */ +async function seedTwoHosts(options: { studioRateLimitedUntil?: string } = {}) { + const studioId = await createConnection({ + provider: PROVIDER, + authType: "none", + name: "Mac Studio", + baseUrl: "http://studio.lan:11434/v1", + priority: 1, + isActive: true, + }); + const jetsonId = await createConnection({ + provider: PROVIDER, + authType: "none", + name: "Jetson", + baseUrl: "http://jetson.lan:11434/v1", + priority: 2, + isActive: true, + }); + + await modelsDb.replaceSyncedAvailableModelsForConnection(PROVIDER, studioId, [ + { id: "gemma3:4b", name: "gemma3:4b" }, + { id: "flux2-klein:9b", name: "flux2-klein:9b" }, + ]); + await modelsDb.replaceSyncedAvailableModelsForConnection(PROVIDER, jetsonId, [ + { id: "gemma3:4b", name: "gemma3:4b" }, + ]); + + if (options.studioRateLimitedUntil) { + await providersDb.updateProviderConnection(studioId, { + rateLimitedUntil: options.studioRateLimitedUntil, + }); + } + + return { studioId, jetsonId }; +} + +test("#11089 selects only the host whose synced inventory advertises the model", async () => { + await resetStorage(); + const { studioId, jetsonId } = await seedTwoHosts(); + + // Exclude studio to force the selector to look elsewhere. Jetson does not + // advertise flux2-klein:9b, so it must NOT be handed back. + const selected = await auth.getProviderCredentials(PROVIDER, studioId, null, "flux2-klein:9b"); + + assert.notEqual( + selectedConnectionId(selected), + jetsonId, + "jetson never synced flux2-klein:9b and must not be selected for it" + ); +}); + +test("#11089 does not preemptively fail over to a host lacking the model when the owner is cooling", async () => { + await resetStorage(); + const coolingUntil = new Date(Date.now() + 10 * 60 * 1000).toISOString(); + const { jetsonId } = await seedTwoHosts({ studioRateLimitedUntil: coolingUntil }); + + const selected = await auth.getProviderCredentials(PROVIDER, null, null, "flux2-klein:9b"); + + assert.notEqual( + selectedConnectionId(selected), + jetsonId, + "a cooling studio must surface a cooldown, not silently route to a host without the model" + ); +}); + +test("#11089 keeps the connection that does advertise the model selectable", async () => { + await resetStorage(); + const { studioId } = await seedTwoHosts(); + + const selected = await auth.getProviderCredentials(PROVIDER, null, null, "flux2-klein:9b"); + + assert.equal( + selectedConnectionId(selected), + studioId, + "studio advertises flux2-klein:9b and must be selected" + ); +}); + +test("#11089 a model advertised by every host leaves both connections eligible", async () => { + await resetStorage(); + const { studioId, jetsonId } = await seedTwoHosts(); + + const first = await auth.getProviderCredentials(PROVIDER, null, null, "gemma3:4b"); + assert.equal( + selectedConnectionId(first), + studioId, + "fill-first prefers priority 1 for a shared model" + ); + + // Excluding studio (the normal account-fallback path) must still reach jetson, + // because jetson genuinely advertises gemma3:4b. + const second = await auth.getProviderCredentials(PROVIDER, studioId, null, "gemma3:4b"); + assert.equal( + selectedConnectionId(second), + jetsonId, + "jetson advertises gemma3:4b and must remain a valid failover" + ); +}); + +test("#11089 fails open when the provider has no synced inventory at all", async () => { + await resetStorage(); + + const connectionId = await createConnection({ + provider: PROVIDER, + authType: "none", + baseUrl: "http://127.0.0.1:11434/v1", + priority: 1, + isActive: true, + }); + + // No replaceSyncedAvailableModelsForConnection call: discovery never ran. + // Routing must behave exactly as before rather than filtering everything out. + const selected = await auth.getProviderCredentials(PROVIDER, null, null, "never-synced-model"); + + assert.equal( + selectedConnectionId(selected), + connectionId, + "an unsynced provider must not be filtered to zero candidates" + ); +});