mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-25 08:32:11 +03:00
fix(ollama): route models by advertised capability (#11087) — port of #11088 to the release line (#11271)
Validated on the combined 8-PR board: ollama-local-capabilities-routing 3/3, managed-model-import 9/9 (including the integration with the carried Gemini-3.5-Flash cleanup from #11259), 88/88 across the board's focused suites, typecheck:core + dashboard-typecheck clean, gates within baseline. This brings #11088 to the release line — it had squash-merged to main by base error (mine) — AND fixes the two defects the port caught: the global filter drop that leaked image/video models into OpenAI chat selections (now scoped to self-hosted providers) and the unregistered hard-lease credential site. Exemplary port discipline: byte-identical carry + the corrections in a separate reviewable commit + the superpowers docs deliberately left out. main still needs the same two-line fix. Thank you @yourspraveen!
This commit is contained in:
committed by
GitHub
parent
12986c44c9
commit
6d4c4843e9
1
changelog.d/fixes/11271-ollama-capability-routing.md
Normal file
1
changelog.d/fixes/11271-ollama-capability-routing.md
Normal file
@@ -0,0 +1 @@
|
||||
- **fix(ollama):** Ollama Local models are no longer flattened to `chat` at sync time — the synced store persists every advertised capability and chat filtering moves to read time, so `/v1/embeddings` and `/v1/images/generations` stop rejecting models the daemon reports as capable ([#11271](https://github.com/diegosouzapw/OmniRoute/pull/11271)) — thanks @yourspraveen
|
||||
@@ -28,6 +28,7 @@ import type {
|
||||
ResolvedComboTarget,
|
||||
} from "./types.ts";
|
||||
import { extractSessionAffinityKey } from "@/sse/services/auth";
|
||||
import { filterChatSelectableModels } from "../modelEndpointPolicy.ts";
|
||||
import { DEFAULT_INTENT_CONFIG, type IntentClassifierConfig } from "../intentClassifier.ts";
|
||||
import { getTaskFitness } from "../autoCombo/taskFitness.ts";
|
||||
import {
|
||||
@@ -470,10 +471,13 @@ export async function expandAutoComboCandidatePool(
|
||||
// catalog only when the user has none. This keeps catalog-only models
|
||||
// (e.g. openrouter/auto) out of pure-auto pools when the operator only
|
||||
// synced a subset (e.g. OpenRouter with importFreeModelsOnly).
|
||||
const [syncedModels, customModels] = await Promise.all([
|
||||
// #11088 (option 1): the synced store now persists non-chat models too —
|
||||
// chat combo pools must keep filtering them out at read time.
|
||||
const [syncedModelsRaw, customModels] = await Promise.all([
|
||||
getSyncedAvailableModels(providerId),
|
||||
getCustomModels(providerId),
|
||||
]);
|
||||
const syncedModels = filterChatSelectableModels(providerId, syncedModelsRaw);
|
||||
const hiddenModels = hiddenModelsMap.get(providerId);
|
||||
const userVisibleIds = new Set<string>();
|
||||
for (const m of syncedModels) if (m.id && !hiddenModels?.has(m.id)) userVisibleIds.add(m.id);
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
import { isSelfHostedChatProvider } from "@/shared/constants/providers";
|
||||
import { getStaticModelsForProvider, type LocalCatalogModel } from "@/lib/providers/staticModels";
|
||||
import { SAFE_OUTBOUND_FETCH_PRESETS, safeOutboundFetch } from "@/shared/network/safeOutboundFetch";
|
||||
import { getProviderValidationGuard } from "@/shared/network/outboundUrlGuardPolicy";
|
||||
import {
|
||||
buildOllamaShowUrl,
|
||||
enrichOllamaModelsWithCapabilities,
|
||||
} from "@/lib/providerModels/ollamaCapabilities";
|
||||
|
||||
export type JsonRecord = Record<string, unknown>;
|
||||
|
||||
@@ -102,3 +108,35 @@ export function buildNamedOpenAiStyleHeaders(
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
// #11087 — Ollama's OpenAI-compatible /v1/models response carries no capability
|
||||
// data, so every local model looked like a chat model and image/embedding
|
||||
// requests were routed to text-only models. Probe /api/show per model (bounded
|
||||
// concurrency, failures degrade to the unenriched entry) to recover the
|
||||
// advertised capabilities. Lives here rather than inline in route.ts to keep the
|
||||
// route file under its frozen file-size cap.
|
||||
export async function enrichOllamaLocalModels(
|
||||
models: unknown[],
|
||||
baseUrl: string,
|
||||
proxy: unknown,
|
||||
token: string | null | undefined
|
||||
): Promise<JsonRecord[]> {
|
||||
const showUrl = buildOllamaShowUrl(baseUrl);
|
||||
return enrichOllamaModelsWithCapabilities(models, async (modelId) => {
|
||||
try {
|
||||
const showResponse = await safeOutboundFetch(showUrl, {
|
||||
...SAFE_OUTBOUND_FETCH_PRESETS.modelsProbe,
|
||||
// Same guard tier as the discovery probe above: local-first, so LAN
|
||||
// Ollama hosts are reachable while the outbound guard stays enforced.
|
||||
guard: getProviderValidationGuard(),
|
||||
proxyConfig: proxy,
|
||||
method: "POST",
|
||||
headers: buildOptionalBearerHeaders(token),
|
||||
body: JSON.stringify({ model: modelId, verbose: false }),
|
||||
});
|
||||
return showResponse.ok ? await showResponse.json() : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -108,6 +108,7 @@ import {
|
||||
mergeSpecialtyCatalogIntoLiveModels,
|
||||
buildOptionalBearerHeaders,
|
||||
buildNamedOpenAiStyleHeaders,
|
||||
enrichOllamaLocalModels,
|
||||
} from "./discovery/helpers";
|
||||
import {
|
||||
fetchAntigravityDiscoveryModelsCached,
|
||||
@@ -794,6 +795,8 @@ export async function GET(
|
||||
models = isNamedOpenAIStyleProvider(provider)
|
||||
? normalizeOpenAiLikeModelsResponse(data, provider)
|
||||
: data.data || data.models || [];
|
||||
if (provider === "ollama-local")
|
||||
models = await enrichOllamaLocalModels(models, baseUrl, proxy, token);
|
||||
break; // Success!
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,10 @@ import { getComboByName } from "@/lib/db/combos";
|
||||
import { getAllCustomModels } from "@/lib/db/models";
|
||||
import { resolveProxyForConnection } from "@/lib/db/settings";
|
||||
import { resolveImageRouteModel } from "@/lib/images/imageRouteModel";
|
||||
import {
|
||||
resolveLocalSyncedEndpointRoute,
|
||||
type LocalSyncedEndpointRoute,
|
||||
} from "@/lib/providerModels/syncedEndpointRouting";
|
||||
import { runWithProxyContext } from "@omniroute/open-sse/utils/proxyFetch.ts";
|
||||
import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta";
|
||||
import { calculateModalCost } from "@/lib/usage/costCalculator";
|
||||
@@ -145,6 +149,16 @@ async function postHandler(request, context) {
|
||||
// Parse model to get provider
|
||||
let { provider, model: requestedModel } = parseImageModel(body.model);
|
||||
let isCustomModel = false;
|
||||
let syncedEndpointRoute: LocalSyncedEndpointRoute | null = null;
|
||||
|
||||
if (!provider) {
|
||||
syncedEndpointRoute = await resolveLocalSyncedEndpointRoute(body.model, "images");
|
||||
if (syncedEndpointRoute) {
|
||||
provider = syncedEndpointRoute.provider;
|
||||
body.model = `${syncedEndpointRoute.provider}/${syncedEndpointRoute.model}`;
|
||||
isCustomModel = true;
|
||||
}
|
||||
}
|
||||
|
||||
// If not in built-in registry, check custom models tagged for images
|
||||
if (!provider) {
|
||||
@@ -231,9 +245,8 @@ async function postHandler(request, context) {
|
||||
credentials = await getProviderCredentialsWithQuotaPreflight(
|
||||
provider,
|
||||
null,
|
||||
null,
|
||||
requestedModel
|
||||
);
|
||||
syncedEndpointRoute?.connectionIds ?? null,
|
||||
requestedModel );
|
||||
if (!credentials) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
|
||||
@@ -31,6 +31,7 @@ import { isPrivateHost, isCloudMetadataHost } from "@/shared/network/outboundUrl
|
||||
import { calculateCost } from "@/lib/usage/costCalculator";
|
||||
import { attachOmniRouteMetaHeaders } from "@/domain/omnirouteResponseMeta";
|
||||
import { generateRequestId } from "@/shared/utils/requestId";
|
||||
import { resolveLocalSyncedEndpointRoute } from "@/lib/providerModels/syncedEndpointRouting";
|
||||
|
||||
type ValidatedEmbeddingBody = Record<string, unknown> & { model: string };
|
||||
type ProviderCredentialsResult = Awaited<ReturnType<typeof getProviderCredentials>>;
|
||||
@@ -164,7 +165,17 @@ export async function createEmbeddingResponse(
|
||||
model: options.resolvedModel ?? body.model,
|
||||
}
|
||||
: parseEmbeddingModel(body.model, dynamicProviders);
|
||||
const { provider, model: resolvedModel } = parsedModel;
|
||||
let { provider, model: resolvedModel } = parsedModel;
|
||||
// #11088: a bare local-model request routes through the connection that
|
||||
// advertises the requested endpoint — only when no explicit resolvedProvider
|
||||
// already won above (explicit resolution takes precedence).
|
||||
const syncedEndpointRoute = options.resolvedProvider
|
||||
? null
|
||||
: await resolveLocalSyncedEndpointRoute(body.model, "embeddings");
|
||||
if (syncedEndpointRoute) {
|
||||
provider = syncedEndpointRoute.provider;
|
||||
resolvedModel = syncedEndpointRoute.model;
|
||||
}
|
||||
if (!provider) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
@@ -172,6 +183,7 @@ export async function createEmbeddingResponse(
|
||||
);
|
||||
}
|
||||
|
||||
let credentials: ProviderCredentialsResult | null = null;
|
||||
let providerConfig: EmbeddingProvider | null =
|
||||
options.resolvedProvider ||
|
||||
dynamicProviders.find((dp) => dp.id === provider) ||
|
||||
@@ -179,6 +191,48 @@ export async function createEmbeddingResponse(
|
||||
null;
|
||||
let credentialsProviderId = provider;
|
||||
|
||||
if (syncedEndpointRoute) {
|
||||
credentials = await getProviderCredentials(
|
||||
provider,
|
||||
null,
|
||||
syncedEndpointRoute.connectionIds,
|
||||
syncedEndpointRoute.model
|
||||
);
|
||||
if (!credentials) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
`No credentials for embedding provider: ${provider}`
|
||||
);
|
||||
}
|
||||
if ("allRateLimited" in credentials && credentials.allRateLimited) {
|
||||
return unavailableResponse(
|
||||
HTTP_STATUS.RATE_LIMITED,
|
||||
`[${provider}] All accounts rate limited`,
|
||||
credentials.retryAfter,
|
||||
credentials.retryAfterHuman
|
||||
);
|
||||
}
|
||||
|
||||
const providerSpecificData = (credentials as { providerSpecificData?: Record<string, unknown> })
|
||||
.providerSpecificData;
|
||||
const configuredBaseUrl = providerSpecificData?.baseUrl;
|
||||
if (typeof configuredBaseUrl !== "string" || configuredBaseUrl.trim().length === 0) {
|
||||
return errorResponse(
|
||||
HTTP_STATUS.BAD_REQUEST,
|
||||
`No base URL configured for embedding provider: ${provider}`
|
||||
);
|
||||
}
|
||||
let baseUrl = configuredBaseUrl.trim();
|
||||
while (baseUrl.endsWith("/")) baseUrl = baseUrl.slice(0, -1);
|
||||
providerConfig = {
|
||||
id: provider,
|
||||
baseUrl: baseUrl.endsWith("/embeddings") ? baseUrl : `${baseUrl}/embeddings`,
|
||||
authType: "apikey",
|
||||
authHeader: "bearer",
|
||||
models: [],
|
||||
};
|
||||
}
|
||||
|
||||
if (!providerConfig) {
|
||||
try {
|
||||
const allNodes = (await getCachedProviderNodes()) as unknown as EmbeddingProviderNodeRow[];
|
||||
@@ -226,8 +280,7 @@ export async function createEmbeddingResponse(
|
||||
);
|
||||
}
|
||||
|
||||
let credentials: ProviderCredentialsResult | null = null;
|
||||
if (providerConfig.authType !== "none") {
|
||||
if (!credentials && providerConfig.authType !== "none") {
|
||||
credentials = await getProviderCredentials(credentialsProviderId);
|
||||
if (!credentials) {
|
||||
return errorResponse(
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
} from "@omniroute/open-sse/config/antigravityModelAliases.ts";
|
||||
import { filterChatSelectableModels } from "@omniroute/open-sse/services/modelEndpointPolicy.ts";
|
||||
import { filterSelectableModels } from "@omniroute/open-sse/services/modelLifecycle.ts";
|
||||
import { isSelfHostedChatProvider } from "@/shared/constants/providers";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
@@ -253,10 +254,18 @@ export async function importManagedModels({
|
||||
const previousSyncedAvailableModels =
|
||||
previousSyncedAvailableModelsInput ??
|
||||
(await getSyncedAvailableModelsForConnection(providerId, connectionId));
|
||||
const discoveredModels = filterChatSelectableModels(
|
||||
// #11088 (option 1): self-hosted providers keep their non-chat models — chat
|
||||
// filtering happens at read time (resolveLocalSyncedEndpointRoute). Every other
|
||||
// provider keeps the import-time chat filter: the read-time path is gated on
|
||||
// isSelfHostedChatProvider, so dropping it globally leaked image/video models
|
||||
// into OpenAI chat selections (#11271).
|
||||
const selectableModels = filterSelectableModels(
|
||||
providerId,
|
||||
filterSelectableModels(providerId, normalizeDiscoveredModels(fetchedModels, providerId))
|
||||
normalizeDiscoveredModels(fetchedModels, providerId)
|
||||
);
|
||||
const discoveredModels = isSelfHostedChatProvider(providerId)
|
||||
? selectableModels
|
||||
: filterChatSelectableModels(providerId, selectableModels);
|
||||
const candidateImportedModels = normalizeImportedModels(discoveredModels);
|
||||
const importedIds = new Set(candidateImportedModels.map((model) => model.id));
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
} from "@/lib/db/models";
|
||||
import { CANONICAL_EFFORT_VALUES } from "@/shared/reasoning/effortStandardization";
|
||||
import { isObsoleteKiroModelAlias } from "@omniroute/open-sse/services/kiroModels.ts";
|
||||
import { filterChatSelectableModels } from "@omniroute/open-sse/services/modelEndpointPolicy.ts";
|
||||
import { filterSelectableModels } from "@omniroute/open-sse/services/modelLifecycle.ts";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
@@ -379,9 +378,13 @@ export async function persistDiscoveredModels(
|
||||
connectionId: string,
|
||||
models: unknown
|
||||
): Promise<SyncedAvailableModel[]> {
|
||||
const normalized = filterChatSelectableModels(
|
||||
// #11088 (option 1): the synced store is endpoint-agnostic — images/embeddings
|
||||
// models must persist so per-connection endpoint routing (#11088) and the
|
||||
// /v1/models catalog can see them. Chat selectability is applied at read time
|
||||
// (auto-pool expansion, chat projections), not at write time.
|
||||
const normalized = filterSelectableModels(
|
||||
providerId,
|
||||
filterSelectableModels(providerId, normalizeDiscoveredModels(models, providerId))
|
||||
normalizeDiscoveredModels(models, providerId)
|
||||
);
|
||||
await replaceSyncedAvailableModelsForConnection(providerId, connectionId, normalized);
|
||||
return normalized;
|
||||
|
||||
98
src/lib/providerModels/ollamaCapabilities.ts
Normal file
98
src/lib/providerModels/ollamaCapabilities.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import { z } from "zod";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
const ollamaShowResponseSchema = z
|
||||
.object({
|
||||
capabilities: z.array(z.string().max(64)).max(32).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
const OLLAMA_CAPABILITY_TO_ENDPOINT: Readonly<Record<string, string>> = {
|
||||
completion: "chat",
|
||||
embedding: "embeddings",
|
||||
image: "images",
|
||||
};
|
||||
|
||||
const MAX_CONCURRENT_SHOW_REQUESTS = 4;
|
||||
|
||||
function asRecord(value: unknown): JsonRecord {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
||||
}
|
||||
|
||||
export function buildOllamaShowUrl(openAiBaseUrl: string): string {
|
||||
let base = openAiBaseUrl.trim();
|
||||
while (base.endsWith("/")) base = base.slice(0, -1);
|
||||
base = base.replace(/\/(?:chat\/completions|completions|embeddings|images\/generations)$/i, "");
|
||||
if (base.endsWith("/v1")) base = base.slice(0, -3);
|
||||
return `${base}/api/show`;
|
||||
}
|
||||
|
||||
export function applyOllamaShowCapabilities(model: unknown, showResponse: unknown): JsonRecord {
|
||||
const record = asRecord(model);
|
||||
const parsed = ollamaShowResponseSchema.safeParse(showResponse);
|
||||
if (!parsed.success || !parsed.data.capabilities) return record;
|
||||
|
||||
const capabilities = Array.from(
|
||||
new Set(parsed.data.capabilities.map((value) => value.trim().toLowerCase()).filter(Boolean))
|
||||
);
|
||||
const supportedEndpoints = Array.from(
|
||||
new Set(
|
||||
capabilities
|
||||
.map((capability) => OLLAMA_CAPABILITY_TO_ENDPOINT[capability])
|
||||
.filter((endpoint): endpoint is string => Boolean(endpoint))
|
||||
)
|
||||
);
|
||||
if (supportedEndpoints.length === 0) return record;
|
||||
|
||||
const apiFormat = supportedEndpoints.includes("chat")
|
||||
? "chat-completions"
|
||||
: supportedEndpoints.includes("embeddings")
|
||||
? "embeddings"
|
||||
: "images-generations";
|
||||
|
||||
return {
|
||||
...record,
|
||||
apiFormat,
|
||||
supportedEndpoints,
|
||||
...(capabilities.includes("vision") ? { supportsVision: true } : {}),
|
||||
...(capabilities.includes("tools") ? { supportsTools: true } : {}),
|
||||
...(capabilities.includes("thinking") ? { supportsThinking: true } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function enrichOllamaModelsWithCapabilities(
|
||||
models: unknown[],
|
||||
fetchShow: (modelId: string) => Promise<unknown | null>
|
||||
): Promise<JsonRecord[]> {
|
||||
const output: JsonRecord[] = new Array(models.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
const worker = async () => {
|
||||
while (nextIndex < models.length) {
|
||||
const index = nextIndex++;
|
||||
const model = asRecord(models[index]);
|
||||
const modelId =
|
||||
typeof model.id === "string"
|
||||
? model.id
|
||||
: typeof model.name === "string"
|
||||
? model.name
|
||||
: typeof model.model === "string"
|
||||
? model.model
|
||||
: null;
|
||||
if (!modelId) {
|
||||
output[index] = model;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
output[index] = applyOllamaShowCapabilities(model, await fetchShow(modelId));
|
||||
} catch {
|
||||
output[index] = model;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const workerCount = Math.min(MAX_CONCURRENT_SHOW_REQUESTS, Math.max(1, models.length));
|
||||
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
||||
return output;
|
||||
}
|
||||
31
src/lib/providerModels/syncedEndpointRouting.ts
Normal file
31
src/lib/providerModels/syncedEndpointRouting.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
import { getSyncedAvailableModelsByConnection } from "@/lib/db/models";
|
||||
import { isSelfHostedChatProvider, resolveProviderId } from "@/shared/constants/providers";
|
||||
|
||||
export type LocalSyncedEndpointRoute = {
|
||||
provider: string;
|
||||
model: string;
|
||||
connectionIds: string[];
|
||||
};
|
||||
|
||||
export async function resolveLocalSyncedEndpointRoute(
|
||||
modelStr: string,
|
||||
endpoint: "embeddings" | "images"
|
||||
): Promise<LocalSyncedEndpointRoute | null> {
|
||||
const slashIndex = modelStr.indexOf("/");
|
||||
if (slashIndex <= 0 || slashIndex === modelStr.length - 1) return null;
|
||||
|
||||
const provider = resolveProviderId(modelStr.slice(0, slashIndex));
|
||||
const model = modelStr.slice(slashIndex + 1);
|
||||
if (!isSelfHostedChatProvider(provider)) return null;
|
||||
|
||||
const byConnection = await getSyncedAvailableModelsByConnection(provider);
|
||||
const connectionIds = Object.entries(byConnection)
|
||||
.filter(([, models]) =>
|
||||
models.some(
|
||||
(candidate) => candidate.id === model && candidate.supportedEndpoints?.includes(endpoint)
|
||||
)
|
||||
)
|
||||
.map(([connectionId]) => connectionId);
|
||||
|
||||
return connectionIds.length > 0 ? { provider, model, connectionIds } : null;
|
||||
}
|
||||
@@ -40,7 +40,11 @@ const EXPECTED: Record<InventoryKind, Record<string, number>> = {
|
||||
"src/app/api/v1/session-leases/route.ts": 1,
|
||||
"src/app/api/v1/videos/generations/route.ts": 2,
|
||||
"src/app/api/v1/web/fetch/route.ts": 1,
|
||||
"src/lib/embeddings/service.ts": 2,
|
||||
// #11088/#11271: third site is the synced local-endpoint route — it resolves
|
||||
// credentials through getProviderCredentials with the connection allowlist
|
||||
// from resolveLocalSyncedEndpointRoute, and handles allRateLimited, so it is
|
||||
// fenced the same way as the two pre-existing sites.
|
||||
"src/lib/embeddings/service.ts": 3,
|
||||
"src/lib/memory/embedding/index.ts": 1,
|
||||
"src/lib/search/executeWebSearch.ts": 2,
|
||||
"src/lib/skills/webFetchExecution.ts": 1,
|
||||
|
||||
205
tests/unit/ollama-local-capabilities-routing.test.ts
Normal file
205
tests/unit/ollama-local-capabilities-routing.test.ts
Normal file
@@ -0,0 +1,205 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-ollama-capabilities-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
process.env.APP_LOG_TO_FILE = "false";
|
||||
process.env.API_KEY_SECRET = "ollama-capabilities-test-secret";
|
||||
process.env.REQUIRE_API_KEY = "false";
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const modelsDb = await import("../../src/lib/db/models.ts");
|
||||
const providerModelsRoute = await import("../../src/app/api/providers/[id]/models/route.ts");
|
||||
const v1ModelsCatalog = await import("../../src/app/api/v1/models/catalog.ts");
|
||||
const imageRoute = await import("../../src/app/api/v1/images/generations/route.ts");
|
||||
const { createEmbeddingResponse } = await import("../../src/lib/embeddings/service.ts");
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
function resetStorage() {
|
||||
globalThis.fetch = originalFetch;
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
async function seedOllamaConnection(baseUrl = "http://127.0.0.1:11434/v1", priority = 1) {
|
||||
return providersDb.createProviderConnection({
|
||||
provider: "ollama-local",
|
||||
authType: "apikey",
|
||||
name: "Ollama test host",
|
||||
apiKey: "test-key",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority,
|
||||
providerSpecificData: { baseUrl },
|
||||
});
|
||||
}
|
||||
|
||||
test.beforeEach(resetStorage);
|
||||
|
||||
test.after(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("Ollama discovery maps /api/show capabilities into connection-scoped model metadata", async () => {
|
||||
const connection = await seedOllamaConnection();
|
||||
const showCapabilities: Record<string, string[]> = {
|
||||
"image-model": ["image"],
|
||||
"embedding-model": ["embedding"],
|
||||
"chat-model": ["completion", "vision", "tools", "thinking"],
|
||||
};
|
||||
const calledUrls: string[] = [];
|
||||
|
||||
globalThis.fetch = async (input, init = {}) => {
|
||||
const url = String(input);
|
||||
calledUrls.push(url);
|
||||
if (url.endsWith("/v1/models")) {
|
||||
return Response.json({
|
||||
data: Object.keys(showCapabilities).map((id) => ({ id, object: "model" })),
|
||||
});
|
||||
}
|
||||
if (url.endsWith("/api/show")) {
|
||||
const body = JSON.parse(String(init.body || "{}")) as { model?: string };
|
||||
return Response.json({ capabilities: showCapabilities[body.model || ""] || [] });
|
||||
}
|
||||
return new Response("not found", { status: 404 });
|
||||
};
|
||||
|
||||
const response = await providerModelsRoute.GET(
|
||||
new Request(`http://localhost/api/providers/${connection.id}/models?refresh=true`),
|
||||
{ params: { id: connection.id } }
|
||||
);
|
||||
const body = (await response.json()) as {
|
||||
models: Array<{
|
||||
id: string;
|
||||
apiFormat?: string;
|
||||
supportedEndpoints?: string[];
|
||||
supportsVision?: boolean;
|
||||
supportsTools?: boolean;
|
||||
supportsThinking?: boolean;
|
||||
}>;
|
||||
};
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.ok(calledUrls.some((url) => url.endsWith("/api/show")));
|
||||
assert.deepEqual(body.models.find((model) => model.id === "image-model")?.supportedEndpoints, [
|
||||
"images",
|
||||
]);
|
||||
assert.equal(
|
||||
body.models.find((model) => model.id === "image-model")?.apiFormat,
|
||||
"images-generations"
|
||||
);
|
||||
assert.deepEqual(
|
||||
body.models.find((model) => model.id === "embedding-model")?.supportedEndpoints,
|
||||
["embeddings"]
|
||||
);
|
||||
assert.equal(
|
||||
body.models.find((model) => model.id === "embedding-model")?.apiFormat,
|
||||
"embeddings"
|
||||
);
|
||||
const chatModel = body.models.find((model) => model.id === "chat-model");
|
||||
assert.deepEqual(chatModel?.supportedEndpoints, ["chat"]);
|
||||
assert.equal(chatModel?.supportsVision, true);
|
||||
assert.equal(chatModel?.supportsTools, true);
|
||||
assert.equal(chatModel?.supportsThinking, true);
|
||||
|
||||
const persisted = await modelsDb.getSyncedAvailableModelsForConnection(
|
||||
"ollama-local",
|
||||
connection.id
|
||||
);
|
||||
assert.deepEqual(persisted.find((model) => model.id === "image-model")?.supportedEndpoints, [
|
||||
"images",
|
||||
]);
|
||||
assert.deepEqual(persisted.find((model) => model.id === "embedding-model")?.supportedEndpoints, [
|
||||
"embeddings",
|
||||
]);
|
||||
|
||||
const catalogResponse = await v1ModelsCatalog.getUnifiedModelsResponse(
|
||||
new Request("http://localhost/v1/models")
|
||||
);
|
||||
const catalog = (await catalogResponse.json()) as {
|
||||
data: Array<{
|
||||
id: string;
|
||||
type?: string;
|
||||
supported_endpoints?: string[];
|
||||
capabilities?: Record<string, boolean>;
|
||||
}>;
|
||||
};
|
||||
const imageCatalogModel = catalog.data.find((model) => model.id.endsWith("/image-model"));
|
||||
assert.equal(imageCatalogModel?.type, "image");
|
||||
assert.deepEqual(imageCatalogModel?.supported_endpoints, ["images"]);
|
||||
const embeddingCatalogModel = catalog.data.find((model) => model.id.endsWith("/embedding-model"));
|
||||
assert.equal(embeddingCatalogModel?.type, "embedding");
|
||||
assert.deepEqual(embeddingCatalogModel?.supported_endpoints, ["embeddings"]);
|
||||
const chatCatalogModel = catalog.data.find((model) => model.id.endsWith("/chat-model"));
|
||||
assert.equal(chatCatalogModel?.capabilities?.vision, true);
|
||||
assert.equal(chatCatalogModel?.capabilities?.tool_calling, true);
|
||||
assert.equal(chatCatalogModel?.capabilities?.reasoning, true);
|
||||
});
|
||||
|
||||
test("Ollama image model routes through its advertising connection", async () => {
|
||||
await seedOllamaConnection("http://127.0.0.1:11434/v1", 1);
|
||||
const connection = await seedOllamaConnection("http://127.0.0.1:11435/v1", 2);
|
||||
await modelsDb.replaceSyncedAvailableModelsForConnection("ollama-local", connection.id, [
|
||||
{
|
||||
id: "image-model",
|
||||
name: "Image Model",
|
||||
apiFormat: "images-generations",
|
||||
supportedEndpoints: ["images"],
|
||||
},
|
||||
]);
|
||||
|
||||
let capturedUrl = "";
|
||||
globalThis.fetch = async (input) => {
|
||||
capturedUrl = String(input);
|
||||
return Response.json({ data: [{ b64_json: "aW1hZ2U=" }] });
|
||||
};
|
||||
|
||||
const response = await imageRoute.POST(
|
||||
new Request("http://localhost/v1/images/generations", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ model: "ollama-local/image-model", prompt: "test image" }),
|
||||
})
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200, await response.text());
|
||||
assert.equal(capturedUrl, "http://127.0.0.1:11435/v1/images/generations");
|
||||
});
|
||||
|
||||
test("Ollama embedding model routes through its advertising connection", async () => {
|
||||
await seedOllamaConnection("http://127.0.0.1:11434/v1", 1);
|
||||
const connection = await seedOllamaConnection("http://127.0.0.1:11436/v1", 2);
|
||||
await modelsDb.replaceSyncedAvailableModelsForConnection("ollama-local", connection.id, [
|
||||
{
|
||||
id: "embedding-model",
|
||||
name: "Embedding Model",
|
||||
apiFormat: "embeddings",
|
||||
supportedEndpoints: ["embeddings"],
|
||||
},
|
||||
]);
|
||||
|
||||
let capturedUrl = "";
|
||||
globalThis.fetch = async (input) => {
|
||||
capturedUrl = String(input);
|
||||
return Response.json({
|
||||
data: [{ object: "embedding", embedding: [0.1, 0.2], index: 0 }],
|
||||
usage: { prompt_tokens: 2, total_tokens: 2 },
|
||||
});
|
||||
};
|
||||
|
||||
const response = await createEmbeddingResponse({
|
||||
model: "ollama-local/embedding-model",
|
||||
input: "hello",
|
||||
});
|
||||
|
||||
assert.equal(response.status, 200, await response.text());
|
||||
assert.equal(capturedUrl, "http://127.0.0.1:11436/v1/embeddings");
|
||||
});
|
||||
Reference in New Issue
Block a user