From e43f1a5ff143c4420c01258c8decfd4c13c0310a Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 3 Sep 2026 13:55:13 -0400 Subject: [PATCH] feat(cache): enhance semantic cache UI, auto-detect embedding models, and sync Redis hit counters --- open-sse/config/semanticCacheConfig.ts | 86 +- open-sse/handlers/chatCore/semanticCache.ts | 23 +- .../handlers/chatCore/semanticCacheStore.ts | 1 + .../chatCore/streamingSemanticCacheStore.ts | 1 + open-sse/services/cache/embeddingClient.ts | 10 + open-sse/services/cache/redisVectorStore.ts | 10 +- .../services/cache/semanticCacheManager.ts | 2 + open-sse/services/cache/vectorStore.ts | 1 + open-sse/services/modelEndpointPolicy.ts | 14 +- src/app/(dashboard)/dashboard/combos/page.tsx | 3 +- .../[id]/hooks/useModelImportHandlers.ts | 12 +- .../settings/components/CacheSettingsTab.tsx | 738 ++++++++++++++++-- src/app/api/provider-models/route.ts | 10 +- .../cache-config/embedding-options/route.ts | 18 + .../settings/cache-config/embeddingOptions.ts | 159 ++++ src/app/api/settings/cache-config/route.ts | 74 +- .../cache-config/test-embedding/route.ts | 76 ++ src/app/api/v1/chat/completions/route.ts | 2 + src/lib/api/modelTestRunner.ts | 5 +- src/lib/cache/semanticCacheDbBridge.ts | 79 ++ src/lib/db/databaseSettings.ts | 12 +- src/lib/db/models.ts | 14 +- src/lib/db/models/synced.ts | 16 + src/lib/providerModels/modelDiscovery.ts | 160 +++- src/lib/semanticCache.ts | 21 + src/shared/validation/schemas/provider.ts | 7 +- src/types/databaseSettings.ts | 22 +- tests/unit/cache-config-route-8219.test.ts | 49 ++ ...odel-embedding-discovery-and-cache.test.ts | 187 +++++ 29 files changed, 1686 insertions(+), 126 deletions(-) create mode 100644 src/app/api/settings/cache-config/embedding-options/route.ts create mode 100644 src/app/api/settings/cache-config/embeddingOptions.ts create mode 100644 src/app/api/settings/cache-config/test-embedding/route.ts create mode 100644 src/lib/cache/semanticCacheDbBridge.ts create mode 100644 tests/unit/model-embedding-discovery-and-cache.test.ts diff --git a/open-sse/config/semanticCacheConfig.ts b/open-sse/config/semanticCacheConfig.ts index c9318196c9..03526f0403 100644 --- a/open-sse/config/semanticCacheConfig.ts +++ b/open-sse/config/semanticCacheConfig.ts @@ -85,82 +85,114 @@ function parseNumber(val: string | undefined, fallback: number): number { return Number.isFinite(parsed) ? parsed : fallback; } +type DynamicConfigResolver = () => Partial | null | undefined; +let dynamicResolver: DynamicConfigResolver | null = null; + +export function registerSemanticCacheConfigResolver(resolver: DynamicConfigResolver): void { + dynamicResolver = resolver; +} + /** * Resolves semantic cache configuration from environment variables, merged - * with optional explicit overrides. + * with optional dynamic database settings and explicit overrides. */ export function resolveSemanticCacheConfig( overrides?: Partial ): SemanticCacheConfig { + const dynamic = dynamicResolver ? dynamicResolver() : null; const env = process.env; const backendEnv = (env.OMNIROUTE_SEMANTIC_CACHE_BACKEND || "").toLowerCase().trim(); - const backend: SemanticCacheBackend = backendEnv === "redis" ? "redis" : "memory"; + const backend: SemanticCacheBackend = + backendEnv === "redis" + ? "redis" + : backendEnv === "memory" + ? "memory" + : (dynamic?.backend ?? DEFAULT_SEMANTIC_CACHE_CONFIG.backend); const resolved: SemanticCacheConfig = { - enabled: parseBoolean( - env.OMNIROUTE_SEMANTIC_CACHE_ENABLED, - DEFAULT_SEMANTIC_CACHE_CONFIG.enabled - ), + enabled: + env.OMNIROUTE_SEMANTIC_CACHE_ENABLED !== undefined + ? parseBoolean(env.OMNIROUTE_SEMANTIC_CACHE_ENABLED, DEFAULT_SEMANTIC_CACHE_CONFIG.enabled) + : (dynamic?.enabled ?? DEFAULT_SEMANTIC_CACHE_CONFIG.enabled), backend, - similarityThreshold: parseNumber( - env.OMNIROUTE_SEMANTIC_CACHE_THRESHOLD, - DEFAULT_SEMANTIC_CACHE_CONFIG.similarityThreshold - ), - ttlMs: parseNumber(env.OMNIROUTE_SEMANTIC_CACHE_TTL_MS, DEFAULT_SEMANTIC_CACHE_CONFIG.ttlMs), - maxEntries: parseNumber( - env.OMNIROUTE_SEMANTIC_CACHE_MAX_ENTRIES, - DEFAULT_SEMANTIC_CACHE_CONFIG.maxEntries - ), + similarityThreshold: + env.OMNIROUTE_SEMANTIC_CACHE_THRESHOLD !== undefined + ? parseNumber( + env.OMNIROUTE_SEMANTIC_CACHE_THRESHOLD, + DEFAULT_SEMANTIC_CACHE_CONFIG.similarityThreshold + ) + : (dynamic?.similarityThreshold ?? DEFAULT_SEMANTIC_CACHE_CONFIG.similarityThreshold), + ttlMs: + env.OMNIROUTE_SEMANTIC_CACHE_TTL_MS !== undefined + ? parseNumber(env.OMNIROUTE_SEMANTIC_CACHE_TTL_MS, DEFAULT_SEMANTIC_CACHE_CONFIG.ttlMs) + : (dynamic?.ttlMs ?? DEFAULT_SEMANTIC_CACHE_CONFIG.ttlMs), + maxEntries: + env.OMNIROUTE_SEMANTIC_CACHE_MAX_ENTRIES !== undefined + ? parseNumber( + env.OMNIROUTE_SEMANTIC_CACHE_MAX_ENTRIES, + DEFAULT_SEMANTIC_CACHE_CONFIG.maxEntries + ) + : (dynamic?.maxEntries ?? DEFAULT_SEMANTIC_CACHE_CONFIG.maxEntries), embeddingProvider: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_PROVIDER?.trim() || + dynamic?.embeddingProvider || DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingProvider, embeddingModel: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_MODEL?.trim() || + dynamic?.embeddingModel || DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingModel, embeddingDimension: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_DIMENSION ? parseNumber(env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_DIMENSION, 1536) - : DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingDimension, + : (dynamic?.embeddingDimension ?? DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingDimension), embeddingTimeoutMs: parseNumber( env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_TIMEOUT_MS, - DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingTimeoutMs + dynamic?.embeddingTimeoutMs ?? DEFAULT_SEMANTIC_CACHE_CONFIG.embeddingTimeoutMs ), cacheByModel: parseBoolean( env.OMNIROUTE_SEMANTIC_CACHE_BY_MODEL, - DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByModel + dynamic?.cacheByModel ?? DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByModel ), cacheByProvider: parseBoolean( env.OMNIROUTE_SEMANTIC_CACHE_BY_PROVIDER, - DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByProvider + dynamic?.cacheByProvider ?? DEFAULT_SEMANTIC_CACHE_CONFIG.cacheByProvider ), conversationHistoryDepth: parseNumber( env.OMNIROUTE_SEMANTIC_CACHE_HISTORY_DEPTH, - DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryDepth + dynamic?.conversationHistoryDepth ?? DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryDepth ), conversationHistoryThreshold: parseNumber( env.OMNIROUTE_SEMANTIC_CACHE_HISTORY_THRESHOLD, - DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryThreshold + dynamic?.conversationHistoryThreshold ?? + DEFAULT_SEMANTIC_CACHE_CONFIG.conversationHistoryThreshold ), excludeSystemPrompt: parseBoolean( env.OMNIROUTE_SEMANTIC_CACHE_EXCLUDE_SYSTEM, - DEFAULT_SEMANTIC_CACHE_CONFIG.excludeSystemPrompt + dynamic?.excludeSystemPrompt ?? DEFAULT_SEMANTIC_CACHE_CONFIG.excludeSystemPrompt ), embeddingBaseUrl: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_BASE_URL?.trim() || + dynamic?.embeddingBaseUrl || overrides?.embeddingBaseUrl || undefined, embeddingApiKey: env.OMNIROUTE_SEMANTIC_CACHE_EMBEDDING_API_KEY?.trim() || + dynamic?.embeddingApiKey || overrides?.embeddingApiKey || undefined, - redisUrl: env.OMNIROUTE_SEMANTIC_CACHE_REDIS_URL || env.REDIS_URL || undefined, + redisUrl: + env.OMNIROUTE_SEMANTIC_CACHE_REDIS_URL || env.REDIS_URL || dynamic?.redisUrl || undefined, redisPrefix: env.OMNIROUTE_SEMANTIC_CACHE_REDIS_PREFIX?.trim() || + dynamic?.redisPrefix || DEFAULT_SEMANTIC_CACHE_CONFIG.redisPrefix, - requireZeroTemperature: parseBoolean( - env.OMNIROUTE_SEMANTIC_CACHE_REQUIRE_ZERO_TEMP, - DEFAULT_SEMANTIC_CACHE_CONFIG.requireZeroTemperature - ), + requireZeroTemperature: + env.OMNIROUTE_SEMANTIC_CACHE_REQUIRE_ZERO_TEMP !== undefined + ? parseBoolean( + env.OMNIROUTE_SEMANTIC_CACHE_REQUIRE_ZERO_TEMP, + DEFAULT_SEMANTIC_CACHE_CONFIG.requireZeroTemperature + ) + : (dynamic?.requireZeroTemperature ?? DEFAULT_SEMANTIC_CACHE_CONFIG.requireZeroTemperature), ...overrides, }; diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts index d561b5a33e..d9b700af84 100644 --- a/open-sse/handlers/chatCore/semanticCache.ts +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -1,4 +1,9 @@ -import { generateSignature, getCachedResponse, isCacheableForRead } from "@/lib/semanticCache"; +import { + generateSignature, + getCachedResponse, + isCacheableForRead, + recordSemanticCacheHit, +} from "@/lib/semanticCache"; import { calculateCost } from "@/lib/usage/costCalculator"; import { trackPendingRequest } from "@/lib/usageDb"; import { synthesizeOpenAiSseFromJson } from "../../utils/jsonToSse.ts"; @@ -112,6 +117,22 @@ export async function checkSemanticCache({ ? (Number(cachedUsage.prompt_tokens) || 0) + (Number(cachedUsage.completion_tokens) || 0) : 0; + const requestSignature = generateSignature( + model, + body.messages ?? body.input, + body.temperature, + body.top_p, + apiKeyId ?? undefined + ); + + const targetSignature = + managerResult.entry?.signature || + (hitType === "exact" ? requestSignature : managerResult.entry?.hash); + + if (targetSignature) { + recordSemanticCacheHit(targetSignature, tokensSaved); + } + const headers: Record = { "Content-Type": cachedSse ? "text/event-stream" : "application/json", [OMNIROUTE_RESPONSE_HEADERS.cache]: diff --git a/open-sse/handlers/chatCore/semanticCacheStore.ts b/open-sse/handlers/chatCore/semanticCacheStore.ts index 4c26026cab..15d6aeec1a 100644 --- a/open-sse/handlers/chatCore/semanticCacheStore.ts +++ b/open-sse/handlers/chatCore/semanticCacheStore.ts @@ -85,6 +85,7 @@ export function storeSemanticCacheResponse( ((args.translatedResponse as Record).provider as string) || "", apiKeyId: args.apiKeyId, + signature, tokensSaved, }) .catch(() => {}); diff --git a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts index 2aa8e1e830..72ce25823c 100644 --- a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts +++ b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts @@ -88,6 +88,7 @@ function writeStreamingCacheEntry( model: args.model, provider: args.provider || (cleanBody.provider as string) || "", apiKeyId: args.apiKeyId, + signature: sig, tokensSaved, }) .catch(() => {}); diff --git a/open-sse/services/cache/embeddingClient.ts b/open-sse/services/cache/embeddingClient.ts index 75f66bf074..47a0c117f7 100644 --- a/open-sse/services/cache/embeddingClient.ts +++ b/open-sse/services/cache/embeddingClient.ts @@ -160,6 +160,16 @@ export function createDefaultEmbeddingGenerator(config: { const provider = options?.provider || config.embeddingProvider || "openai"; let targetUrl = config.embeddingBaseUrl; + if (targetUrl) { + targetUrl = targetUrl.trim(); + if (!targetUrl.endsWith("/embeddings")) { + if (!targetUrl.endsWith("/v1")) { + targetUrl = `${targetUrl.replace(/\/+$/, "")}/v1/embeddings`; + } else { + targetUrl = `${targetUrl.replace(/\/+$/, "")}/embeddings`; + } + } + } const apiKey = config.embeddingApiKey; if (!targetUrl) { diff --git a/open-sse/services/cache/redisVectorStore.ts b/open-sse/services/cache/redisVectorStore.ts index a92d76df07..7841655ad4 100644 --- a/open-sse/services/cache/redisVectorStore.ts +++ b/open-sse/services/cache/redisVectorStore.ts @@ -117,12 +117,18 @@ export class RedisVectorStore implements IVectorStore { } } - public async set(entry: CacheEntry, ttlMs: number): Promise { + public async set(entry: CacheEntry, ttlMs?: number): Promise { try { const client = await this.getClient(); if (!client) return; - const ttlSeconds = Math.max(1, Math.ceil(ttlMs / 1000)); + const effectiveTtlMs = + typeof ttlMs === "number" && Number.isFinite(ttlMs) && ttlMs > 0 + ? ttlMs + : entry.expiresAt > 0 + ? Math.max(1000, entry.expiresAt - Date.now()) + : 1800000; + const ttlSeconds = Math.max(1, Math.ceil(effectiveTtlMs / 1000)); const serialized = JSON.stringify(entry); // Store entry and exact hash mapping with TTL diff --git a/open-sse/services/cache/semanticCacheManager.ts b/open-sse/services/cache/semanticCacheManager.ts index 80488e0b00..e20fa2c213 100644 --- a/open-sse/services/cache/semanticCacheManager.ts +++ b/open-sse/services/cache/semanticCacheManager.ts @@ -65,6 +65,7 @@ export interface CacheStoreParams { model: string; provider: string; apiKeyId?: string | null; + signature?: string; tokensSaved?: number; ttlMs?: number; } @@ -386,6 +387,7 @@ export class SemanticCacheManager { const entry: CacheEntry = { id: crypto.randomUUID(), hash: directHash, + signature: params.signature || undefined, embedding, promptText, model: params.model, diff --git a/open-sse/services/cache/vectorStore.ts b/open-sse/services/cache/vectorStore.ts index 37849b924c..8e3174841a 100644 --- a/open-sse/services/cache/vectorStore.ts +++ b/open-sse/services/cache/vectorStore.ts @@ -9,6 +9,7 @@ export interface CacheEntry { id: string; hash: string; + signature?: string; embedding?: number[]; promptText: string; model: string; diff --git a/open-sse/services/modelEndpointPolicy.ts b/open-sse/services/modelEndpointPolicy.ts index 665149f124..4ab6fb3176 100644 --- a/open-sse/services/modelEndpointPolicy.ts +++ b/open-sse/services/modelEndpointPolicy.ts @@ -7,7 +7,8 @@ * provider knowledge here so discovery, import, and catalog projection agree. */ -export type ModelEndpointKind = "chat" | "image" | "video" | "non-chat" | "unknown"; +export type ModelEndpointKind = + "chat" | "image" | "video" | "embedding" | "rerank" | "non-chat" | "unknown"; export type ModelEndpointDecision = { kind: ModelEndpointKind; @@ -27,6 +28,8 @@ const CHAT_ENDPOINTS = new Set([ "messages", "responses", ]); +const EMBEDDING_ENDPOINTS = new Set(["embeddings", "embedding"]); +const RERANK_ENDPOINTS = new Set(["rerank", "reranking"]); const IMAGE_ENDPOINTS = new Set(["image", "images", "images/generations"]); const VIDEO_ENDPOINTS = new Set(["video", "videos", "videos/generations"]); @@ -43,6 +46,12 @@ function classifyExplicitEndpoints( if (endpoints.some((endpoint) => CHAT_ENDPOINTS.has(endpoint))) { return { kind: "chat", chatSelectable: true, reason: "explicit-endpoints" }; } + if (endpoints.some((endpoint) => EMBEDDING_ENDPOINTS.has(endpoint))) { + return { kind: "embedding", chatSelectable: false, reason: "explicit-endpoints" }; + } + if (endpoints.some((endpoint) => RERANK_ENDPOINTS.has(endpoint))) { + return { kind: "rerank", chatSelectable: false, reason: "explicit-endpoints" }; + } if (endpoints.some((endpoint) => IMAGE_ENDPOINTS.has(endpoint))) { return { kind: "image", chatSelectable: false, reason: "explicit-endpoints" }; } @@ -58,6 +67,9 @@ function normalizeOpenAiModelId(modelId: string): string { function classifyOpenAiModel(modelId: string): ModelEndpointDecision | null { const normalized = normalizeOpenAiModelId(modelId).toLowerCase(); + if (normalized.startsWith("text-embedding-")) { + return { kind: "embedding", chatSelectable: false, reason: "provider-policy" }; + } if ( normalized.startsWith("gpt-image-") || normalized.startsWith("dall-e-") || diff --git a/src/app/(dashboard)/dashboard/combos/page.tsx b/src/app/(dashboard)/dashboard/combos/page.tsx index 0105dd8723..ca7432c903 100644 --- a/src/app/(dashboard)/dashboard/combos/page.tsx +++ b/src/app/(dashboard)/dashboard/combos/page.tsx @@ -771,7 +771,8 @@ function CombosPageContent() { const [showUsageGuide, setShowUsageGuide] = useState(true); useEffect(() => { try { - setShowUsageGuide(globalThis.localStorage?.getItem(COMBO_USAGE_GUIDE_STORAGE_KEY) !== "1"); + const isVisible = globalThis.localStorage?.getItem(COMBO_USAGE_GUIDE_STORAGE_KEY) !== "1"; + queueMicrotask(() => setShowUsageGuide(isVisible)); } catch { // Ignore storage access errors (privacy mode / restricted environments) } diff --git a/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts b/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts index 99370e606e..d3d6edd711 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts +++ b/src/app/(dashboard)/dashboard/providers/[id]/hooks/useModelImportHandlers.ts @@ -138,7 +138,7 @@ export function useModelImportHandlers({ }); try { - const res = await fetch(`/api/providers/${importTargetId}/models?refresh=true&chatOnly=true`); + const res = await fetch(`/api/providers/${importTargetId}/models?refresh=true`); const data = await res.json(); if (!res.ok) { setImportProgress((prev) => ({ @@ -230,6 +230,16 @@ export function useModelImportHandlers({ ...(Array.isArray(model.supportedEndpoints) ? { supportedEndpoints: model.supportedEndpoints } : {}), + ...(typeof model.dimensions === "number" && model.dimensions > 0 + ? { dimensions: model.dimensions } + : {}), + ...(Array.isArray(model.supportedInputTypes) + ? { supportedInputTypes: model.supportedInputTypes } + : {}), + ...(typeof model.modelType === "string" ? { modelType: model.modelType } : {}), + ...(typeof model.inputTokenLimit === "number" && model.inputTokenLimit > 0 + ? { max_input_tokens: model.inputTokenLimit } + : {}), }), }); if (!modelAliases[baseAlias]) { diff --git a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx index 990e3324b8..0bbb40298a 100644 --- a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx @@ -1,13 +1,44 @@ "use client"; import { useCallback, useEffect, useState } from "react"; -import { Button, Card } from "@/shared/components"; +import { Button, Card, Badge, Toggle, Select, SegmentedControl } from "@/shared/components"; import { useTranslations } from "next-intl"; type Message = { type: "success" | "error"; text: string }; +interface AvailableEmbeddingModelOption { + id: string; + rawId: string; + name: string; + dimensions?: number; + maxTokens?: number; + supportedInputTypes: string[]; +} + +interface EmbeddingProviderOption { + id: string; + name: string; + hasConnection: boolean; + baseUrl?: string; + models: AvailableEmbeddingModelOption[]; +} + interface CacheConfigResponse { modelCatalogCacheTtlMs: number; + semanticCacheEnabled?: boolean; + semanticCacheMaxSize?: number; + semanticCacheTTL?: number; + semanticCacheBackend?: "memory" | "redis"; + semanticCacheThreshold?: number; + semanticCacheEmbeddingProvider?: string; + semanticCacheEmbeddingModel?: string; + semanticCacheEmbeddingDimension?: number; + semanticCacheEmbeddingBaseUrl?: string; + semanticCacheEmbeddingApiKey?: string; + semanticCacheRedisUrl?: string; + semanticCacheRedisPrefix?: string; + semanticCacheRequireZeroTemp?: boolean; + embeddingOptions?: EmbeddingProviderOption[]; [key: string]: unknown; } @@ -17,12 +48,52 @@ const MAX_TTL_MS = 60000; export default function CacheSettingsTab() { const t = useTranslations("settings"); - const [value, setValue] = useState(String(DEFAULT_TTL_MS)); - const [savedValue, setSavedValue] = useState(String(DEFAULT_TTL_MS)); - const [loading, setLoading] = useState(true); - const [saving, setSaving] = useState(false); - const [message, setMessage] = useState(null); + // Model Catalog Cache State + const [catalogTtl, setCatalogTtl] = useState(String(DEFAULT_TTL_MS)); + const [savedCatalogTtl, setSavedCatalogTtl] = useState(String(DEFAULT_TTL_MS)); + const [catalogLoading, setCatalogLoading] = useState(true); + const [catalogSaving, setCatalogSaving] = useState(false); + const [catalogMessage, setCatalogMessage] = useState(null); + + // Semantic Cache State + const [semEnabled, setSemEnabled] = useState(true); + const [semBackend, setSemBackend] = useState<"memory" | "redis">("memory"); + const [semThreshold, setSemThreshold] = useState(0.8); + const [semTtlMinutes, setSemTtlMinutes] = useState(30); + const [semMaxSize, setSemMaxSize] = useState(1000); + const [semProvider, setSemProvider] = useState("lemonade"); + const [semModel, setSemModel] = useState("harrier-oss-v1-0.6b"); + const [semDimension, setSemDimension] = useState(1024); + const [semBaseUrl, setSemBaseUrl] = useState(""); + const [semApiKey, setSemApiKey] = useState(""); + const [semRedisUrl, setSemRedisUrl] = useState(""); + const [semRedisPrefix, setSemRedisPrefix] = useState("omniroute:semcache:"); + const [semRequireZeroTemp, setSemRequireZeroTemp] = useState(true); + + // Saved Semantic Cache State + const [semSaving, setSemSaving] = useState(false); + const [semMessage, setSemMessage] = useState(null); + const [showAdvanced, setShowAdvanced] = useState(false); + + // Dynamic Options + const [embeddingOptions, setEmbeddingOptions] = useState([]); + + // Test Connection State + const [testingConnection, setTestingConnection] = useState(false); + const [testResult, setTestResult] = useState<{ + ok: boolean; + latencyMs?: number; + dimensions?: number; + resolvedBaseUrl?: string; + error?: string; + } | null>(null); + + // Clear Cache State + const [clearingCache, setClearingCache] = useState(false); + const [clearMessage, setClearMessage] = useState(null); + + // Load Cache Config and Dynamic Options in a single request useEffect(() => { let active = true; @@ -34,16 +105,60 @@ export default function CacheSettingsTab() { .then((config) => { if (!active) return; const ms = config.modelCatalogCacheTtlMs ?? DEFAULT_TTL_MS; - const str = typeof ms === "number" && Number.isFinite(ms) ? String(ms) : String(DEFAULT_TTL_MS); - setValue(str); - setSavedValue(str); + const str = + typeof ms === "number" && Number.isFinite(ms) ? String(ms) : String(DEFAULT_TTL_MS); + setCatalogTtl(str); + setSavedCatalogTtl(str); + + if (config.semanticCacheEnabled !== undefined) { + setSemEnabled(config.semanticCacheEnabled); + } + if (config.semanticCacheBackend === "redis" || config.semanticCacheBackend === "memory") { + setSemBackend(config.semanticCacheBackend); + } + if (typeof config.semanticCacheThreshold === "number") { + setSemThreshold(config.semanticCacheThreshold); + } + if (typeof config.semanticCacheTTL === "number") { + setSemTtlMinutes(Math.round(config.semanticCacheTTL / 60000)); + } + if (typeof config.semanticCacheMaxSize === "number") { + setSemMaxSize(config.semanticCacheMaxSize); + } + if (config.semanticCacheEmbeddingProvider) { + setSemProvider(config.semanticCacheEmbeddingProvider); + } + if (config.semanticCacheEmbeddingModel) { + setSemModel(config.semanticCacheEmbeddingModel); + } + if (typeof config.semanticCacheEmbeddingDimension === "number") { + setSemDimension(config.semanticCacheEmbeddingDimension); + } + if (typeof config.semanticCacheEmbeddingBaseUrl === "string") { + setSemBaseUrl(config.semanticCacheEmbeddingBaseUrl); + } + if (typeof config.semanticCacheEmbeddingApiKey === "string") { + setSemApiKey(config.semanticCacheEmbeddingApiKey); + } + if (typeof config.semanticCacheRedisUrl === "string") { + setSemRedisUrl(config.semanticCacheRedisUrl); + } + if (typeof config.semanticCacheRedisPrefix === "string") { + setSemRedisPrefix(config.semanticCacheRedisPrefix); + } + if (config.semanticCacheRequireZeroTemp !== undefined) { + setSemRequireZeroTemp(config.semanticCacheRequireZeroTemp); + } + if (Array.isArray(config.embeddingOptions)) { + setEmbeddingOptions(config.embeddingOptions); + } }) .catch((error) => { console.error("Failed to load cache config:", error); - if (active) setMessage({ type: "error", text: t("cacheConfigLoadFailed") }); + if (active) setCatalogMessage({ type: "error", text: t("cacheConfigLoadFailed") }); }) .finally(() => { - if (active) setLoading(false); + if (active) setCatalogLoading(false); }); return () => { @@ -51,17 +166,18 @@ export default function CacheSettingsTab() { }; }, [t]); - const dirty = value.trim() !== savedValue; + // Catalog TTL validation and save + const catalogDirty = catalogTtl.trim() !== savedCatalogTtl; - const saveTtl = useCallback(async () => { - if (!dirty) return; + const saveCatalogTtl = useCallback(async () => { + if (!catalogDirty) return; - const parsed = Number(value.trim()); + const parsed = Number(catalogTtl.trim()); if (!Number.isInteger(parsed)) return; if (parsed < MIN_TTL_MS || parsed > MAX_TTL_MS) return; - setSaving(true); - setMessage(null); + setCatalogSaving(true); + setCatalogMessage(null); try { const response = await fetch("/api/settings/cache-config", { @@ -74,19 +190,19 @@ export default function CacheSettingsTab() { const config = (await response.json()) as CacheConfigResponse; const saved = String(config.modelCatalogCacheTtlMs ?? parsed); - setValue(saved); - setSavedValue(saved); - setMessage({ type: "success", text: t("cacheConfigSaveSuccess") }); + setCatalogTtl(saved); + setSavedCatalogTtl(saved); + setCatalogMessage({ type: "success", text: t("cacheConfigSaveSuccess") }); } catch (error) { console.error("Failed to save cache config:", error); - setMessage({ type: "error", text: t("cacheConfigSaveFailed") }); + setCatalogMessage({ type: "error", text: t("cacheConfigSaveFailed") }); } finally { - setSaving(false); + setCatalogSaving(false); } - }, [dirty, t, value]); + }, [catalogDirty, t, catalogTtl]); - const validationError = (() => { - const trimmed = value.trim(); + const catalogValidationError = (() => { + const trimmed = catalogTtl.trim(); if (!trimmed) return "Required"; const parsed = Number(trimmed); if (!Number.isInteger(parsed)) return t("modelCatalogTtlWholeNumberError"); @@ -95,62 +211,526 @@ export default function CacheSettingsTab() { return null; })(); + // Current selected provider and model details + const selectedProviderOption = embeddingOptions.find((p) => p.id === semProvider); + const availableModelsForProvider = selectedProviderOption?.models || []; + const selectedModelOption = availableModelsForProvider.find( + (m) => m.rawId === semModel || m.id === semModel + ); + + // Sync dimensions when model selection changes + const handleModelChange = (modelIdOrRaw: string) => { + setSemModel(modelIdOrRaw); + const m = availableModelsForProvider.find( + (item) => item.rawId === modelIdOrRaw || item.id === modelIdOrRaw + ); + if (m?.dimensions) { + setSemDimension(m.dimensions); + } + setTestResult(null); + }; + + const handleProviderChange = (newProvider: string) => { + setSemProvider(newProvider); + const provider = embeddingOptions.find((p) => p.id === newProvider); + if (provider && provider.models.length > 0) { + const firstModel = provider.models[0]; + setSemModel(firstModel.rawId || firstModel.id); + if (firstModel.dimensions) { + setSemDimension(firstModel.dimensions); + } + } + setTestResult(null); + }; + + // Save Semantic Cache Config + const saveSemanticCache = async () => { + setSemSaving(true); + setSemMessage(null); + + const payload = { + semanticCacheEnabled: semEnabled, + semanticCacheBackend: semBackend, + semanticCacheThreshold: Number(semThreshold), + semanticCacheTTL: semTtlMinutes * 60000, + semanticCacheMaxSize: Number(semMaxSize), + semanticCacheEmbeddingProvider: semProvider, + semanticCacheEmbeddingModel: semModel, + semanticCacheEmbeddingDimension: semDimension ? Number(semDimension) : null, + semanticCacheEmbeddingBaseUrl: semBaseUrl.trim() || selectedProviderOption?.baseUrl || null, + semanticCacheEmbeddingApiKey: semApiKey.trim() || null, + semanticCacheRedisUrl: semRedisUrl.trim() || null, + semanticCacheRedisPrefix: semRedisPrefix.trim() || "omniroute:semcache:", + semanticCacheRequireZeroTemp: semRequireZeroTemp, + }; + + try { + const res = await fetch("/api/settings/cache-config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + + if (!res.ok) throw new Error(`Save failed with status ${res.status}`); + + setSemMessage({ type: "success", text: "Semantic cache settings saved successfully." }); + } catch (err) { + console.error("Failed to save semantic cache settings:", err); + setSemMessage({ type: "error", text: "Failed to save semantic cache settings." }); + } finally { + setSemSaving(false); + } + }; + + // Test embedding connection + const handleTestConnection = async () => { + setTestingConnection(true); + setTestResult(null); + + try { + const res = await fetch("/api/settings/cache-config/test-embedding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: semProvider, + model: semModel, + baseUrl: semBaseUrl.trim() || selectedProviderOption?.baseUrl || undefined, + apiKey: semApiKey.trim() || undefined, + dimensions: semDimension, + }), + }); + + const data = await res.json(); + setTestResult(data); + } catch (err: unknown) { + setTestResult({ ok: false, error: String(err) }); + } finally { + setTestingConnection(false); + } + }; + + // Clear cache + const handleClearCache = async () => { + setClearingCache(true); + setClearMessage(null); + + try { + const res = await fetch("/api/cache", { method: "DELETE" }); + if (!res.ok) throw new Error("Failed to clear cache"); + setClearMessage("Semantic cache purged successfully."); + } catch (err: unknown) { + setClearMessage(`Failed to purge cache: ${String(err)}`); + } finally { + setClearingCache(false); + } + }; + return ( - -
-
-

{t("modelCatalogCacheTtl")}

-

{t("modelCatalogCacheTtlDescription")}

-
-
- - { - setValue(event.target.value); - setMessage(null); - }} - onKeyDown={(event) => { - if (event.key === "Enter" && dirty) void saveTtl(); - }} - className="w-32 px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" - disabled={loading || saving} - /> - ms - - {dirty && ( - - {t("modelCatalogCacheTtlCurrent", { value: savedValue })} - +
+ {/* ── 1. Semantic Caching Card ── */} + +
+ {/* Card Header & Master Toggle */} +
+
+
+

Semantic Caching

+ + {semEnabled ? "Active" : "Disabled"} + +
+

+ Local vector-similarity cache. Reuses high-confidence matching responses to cut + latency and upstream token costs. +

+
+ +
+ + {semEnabled && ( +
+ {/* Provider & Model Selection Row */} +
+
+ + handleModelChange(e.target.value)} + disabled={ + catalogLoading || semSaving || availableModelsForProvider.length === 0 + } + options={ + availableModelsForProvider.length > 0 + ? availableModelsForProvider.map((m) => ({ + value: m.rawId || m.id, + label: m.dimensions + ? `${m.name || m.rawId} (${m.dimensions} dims)` + : m.name || m.rawId, + })) + : [{ value: semModel, label: semModel }] + } + /> + + {/* Model Metadata Badges */} +
+ {semDimension ? ( + + {semDimension} Dimensions + + ) : null} + {selectedModelOption?.maxTokens ? ( + + {selectedModelOption.maxTokens.toLocaleString()} Max Tokens + + ) : null} + {selectedModelOption?.supportedInputTypes ? ( + + Input: {selectedModelOption.supportedInputTypes.join(", ")} + + ) : null} +
+
+
+ + {/* Threshold Slider & TTL */} +
+
+
+ + + {semThreshold.toFixed(2)} + +
+ setSemThreshold(parseFloat(e.target.value))} + className="w-full h-2 bg-surface-2 rounded-lg appearance-none cursor-pointer accent-primary" + disabled={semSaving} + /> +

+ 0.80 recommended. Lower values match more loosely; 1.00 is exact match only. +

+
+ +
+ +
+ setSemTtlMinutes(Math.max(1, parseInt(e.target.value) || 1))} + className="w-28 px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" + disabled={semSaving} + /> + minutes +
+

+ Default 30 minutes. Entries expire after this duration. +

+
+
+ + {/* Storage Backend Selection */} +
+ + setSemBackend(val as "memory" | "redis")} + options={[ + { value: "memory", label: "In-Memory Vector (LRU)" }, + { value: "redis", label: "Redis Vector Store" }, + ]} + /> + + {semBackend === "memory" ? ( +
+ + setSemMaxSize(parseInt(e.target.value) || 100)} + className="w-32 px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" + disabled={semSaving} + /> +
+ ) : ( +
+
+ + setSemRedisUrl(e.target.value)} + className="w-full px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" + disabled={semSaving} + /> +
+
+ + setSemRedisPrefix(e.target.value)} + className="w-full px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" + disabled={semSaving} + /> +
+
+ )} +
+ + {/* Determinism Toggle */} +
+
+

+ Require Strict Determinism (temperature = 0) +

+

+ Only cache and serve responses when temperature is 0, avoiding stochastic + variance. +

+
+ +
+ + {/* Advanced Overrides Accordion */} +
+ + + {showAdvanced && ( +
+
+ + setSemBaseUrl(e.target.value)} + className="w-full px-3 py-1.5 rounded bg-surface-2 border border-border text-xs text-text-primary" + /> +
+
+ + setSemApiKey(e.target.value)} + className="w-full px-3 py-1.5 rounded bg-surface-2 border border-border text-xs text-text-primary" + /> +
+
+ )} +
+ + {/* Action Buttons & Feedback */} +
+
+ + + +
+ + +
+ + {/* Test Connection Output */} + {testResult && ( +
+ {testResult.ok ? ( +
+ Connection Verified: + + Successfully generated {testResult.dimensions}-dim embedding in{" "} + {testResult.latencyMs}ms + {testResult.resolvedBaseUrl ? ` via ${testResult.resolvedBaseUrl}` : ""}. + +
+ ) : ( +
+ Connection Test Failed: + {testResult.error || "Unknown error"} +
+ )} +
+ )} + + {/* Clear Message */} + {clearMessage &&

{clearMessage}

} + + {/* Save Message */} + {semMessage && ( +

+ {semMessage.text} +

+ )} +
)}
- {validationError &&

{validationError}

} - {message && ( -

- {message.text} -

- )} -
- + + + {/* ── 2. Model Catalog Cache Card (Preserved Compatibility) ── */} + +
+
+

{t("modelCatalogCacheTtl")}

+

{t("modelCatalogCacheTtlDescription")}

+
+
+ + { + setCatalogTtl(event.target.value); + setCatalogMessage(null); + }} + onKeyDown={(event) => { + if (event.key === "Enter" && catalogDirty) void saveCatalogTtl(); + }} + className="w-32 px-3 py-1.5 rounded bg-surface-2 border border-border text-sm text-text-primary" + disabled={catalogLoading || catalogSaving} + /> + ms + + {catalogDirty && ( + + {t("modelCatalogCacheTtlCurrent", { value: savedCatalogTtl })} + + )} +
+ {catalogValidationError && ( +

{catalogValidationError}

+ )} + {catalogMessage && ( +

+ {catalogMessage.text} +

+ )} +
+ +
); } diff --git a/src/app/api/provider-models/route.ts b/src/app/api/provider-models/route.ts index beba1dbc1f..7b24e849a1 100644 --- a/src/app/api/provider-models/route.ts +++ b/src/app/api/provider-models/route.ts @@ -150,6 +150,9 @@ export async function POST(request) { // #9820: optional video-generation job preset (job/poll path). generationConfig, isFree, + dimensions, + supportedInputTypes, + modelType, } = validation.data; const model = await addCustomModel( @@ -166,7 +169,12 @@ export async function POST(request) { }, typeof supportsVision === "boolean" ? supportsVision : undefined, generationConfig, - typeof isFree === "boolean" ? isFree : undefined + typeof isFree === "boolean" ? isFree : undefined, + { + ...(typeof dimensions === "number" && dimensions > 0 ? { dimensions } : {}), + ...(Array.isArray(supportedInputTypes) ? { supportedInputTypes } : {}), + ...(typeof modelType === "string" ? { modelType } : {}), + } ); return Response.json({ model }); } catch (error) { diff --git a/src/app/api/settings/cache-config/embedding-options/route.ts b/src/app/api/settings/cache-config/embedding-options/route.ts new file mode 100644 index 0000000000..c6e420c1ec --- /dev/null +++ b/src/app/api/settings/cache-config/embedding-options/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from "next/server"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; +import { getEmbeddingOptions } from "../embeddingOptions"; + +export async function GET(request: Request) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const providers = await getEmbeddingOptions(); + return NextResponse.json({ providers }); + } catch (error: unknown) { + const message = sanitizeErrorMessage(error); + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/app/api/settings/cache-config/embeddingOptions.ts b/src/app/api/settings/cache-config/embeddingOptions.ts new file mode 100644 index 0000000000..fcbb7b14e2 --- /dev/null +++ b/src/app/api/settings/cache-config/embeddingOptions.ts @@ -0,0 +1,159 @@ +import { getProviderConnections } from "@/lib/db/providers"; +import { getSyncedAvailableModels, getCustomModels } from "@/lib/db/models"; +import { + EMBEDDING_PROVIDERS, + getEmbeddingProvider, +} from "@omniroute/open-sse/config/embeddingRegistry.ts"; + +export interface AvailableEmbeddingModelOption { + id: string; + rawId: string; + name: string; + dimensions?: number; + maxTokens?: number; + supportedInputTypes: string[]; +} + +export interface EmbeddingProviderOption { + id: string; + name: string; + hasConnection: boolean; + baseUrl?: string; + models: AvailableEmbeddingModelOption[]; +} + +function getProviderBaseUrl(providerSpecificData: unknown): string | undefined { + if (providerSpecificData && typeof providerSpecificData === "object") { + const data = providerSpecificData as Record; + if (typeof data.baseUrl === "string" && data.baseUrl.trim().length > 0) { + return data.baseUrl.trim(); + } + } + return undefined; +} + +export async function getEmbeddingOptions(): Promise { + const connections = await getProviderConnections().catch(() => []); + const connectionsByProvider = new Map(); + + for (const conn of connections) { + const p = conn.provider; + if (!p) continue; + const list = connectionsByProvider.get(p) || []; + list.push(conn); + connectionsByProvider.set(p, list); + } + + // Collect all candidate providers: configured connections + curated EMBEDDING_PROVIDERS + const candidateProviders = new Set([ + ...Object.keys(EMBEDDING_PROVIDERS), + ...connectionsByProvider.keys(), + ]); + + const providerOptions: EmbeddingProviderOption[] = []; + + for (const providerId of candidateProviders) { + const conns = connectionsByProvider.get(providerId) || []; + const activeConn = conns.find((c) => c.isActive !== false) || conns[0]; + const hasConnection = conns.length > 0; + + const curated = getEmbeddingProvider(providerId); + const configuredBaseUrl = activeConn + ? getProviderBaseUrl(activeConn.providerSpecificData) + : undefined; + const baseUrl = configuredBaseUrl || curated?.baseUrl; + + // Collect models for this provider + const modelsMap = new Map(); + + // 1. Add curated models from embedding registry + if (curated?.models) { + for (const m of curated.models) { + modelsMap.set(m.id, { + id: `${providerId}/${m.id}`, + rawId: m.id, + name: m.name || m.id, + dimensions: m.dimensions, + maxTokens: undefined, + supportedInputTypes: (m.modalities as string[]) || ["text"], + }); + } + } + + // 2. Add synced models from DB + try { + const synced = await getSyncedAvailableModels(providerId); + for (const sm of synced) { + const isEmbedding = + sm.modelType === "embedding" || + sm.apiFormat === "embeddings" || + sm.supportedEndpoints?.includes("embeddings") || + modelsMap.has(sm.id); + + if (isEmbedding) { + const existing = modelsMap.get(sm.id); + modelsMap.set(sm.id, { + id: `${providerId}/${sm.id}`, + rawId: sm.id, + name: sm.name || existing?.name || sm.id, + dimensions: sm.dimensions || existing?.dimensions, + maxTokens: sm.inputTokenLimit || existing?.maxTokens, + supportedInputTypes: sm.supportedInputTypes || + existing?.supportedInputTypes || ["text"], + }); + } + } + } catch { + // Fall through on DB error + } + + // 3. Add custom models from DB + try { + const custom = await getCustomModels(providerId); + if (Array.isArray(custom)) { + for (const cm of custom) { + const isEmbedding = + cm.modelType === "embedding" || + cm.apiFormat === "embeddings" || + (Array.isArray(cm.supportedEndpoints) && + cm.supportedEndpoints.includes("embeddings")) || + modelsMap.has(cm.id); + + if (isEmbedding) { + const existing = modelsMap.get(cm.id); + modelsMap.set(cm.id, { + id: `${providerId}/${cm.id}`, + rawId: cm.id, + name: cm.name || existing?.name || cm.id, + dimensions: cm.dimensions || existing?.dimensions, + maxTokens: cm.inputTokenLimit || existing?.maxTokens, + supportedInputTypes: cm.supportedInputTypes || + existing?.supportedInputTypes || ["text"], + }); + } + } + } + } catch { + // Fall through on DB error + } + + if (modelsMap.size > 0 || curated !== undefined) { + providerOptions.push({ + id: providerId, + name: activeConn?.name || (curated ? providerId : providerId), + hasConnection, + baseUrl, + models: Array.from(modelsMap.values()), + }); + } + } + + providerOptions.sort((a, b) => { + if (a.hasConnection !== b.hasConnection) { + return a.hasConnection ? -1 : 1; + } + return a.name.localeCompare(b.name); + }); + + return providerOptions; +} diff --git a/src/app/api/settings/cache-config/route.ts b/src/app/api/settings/cache-config/route.ts index fc5a6634aa..f4ee601800 100644 --- a/src/app/api/settings/cache-config/route.ts +++ b/src/app/api/settings/cache-config/route.ts @@ -8,11 +8,26 @@ import { getSettings, updateSettings } from "@/lib/db/settings"; import { isAuthenticated } from "@/shared/utils/apiAuth"; import { z } from "zod"; import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { resetSemanticCacheManager } from "@omniroute/open-sse/services/cache/semanticCacheManager.ts"; +import { ensureSemanticCacheDbBridge } from "@/lib/cache/semanticCacheDbBridge"; +import { getEmbeddingOptions } from "./embeddingOptions"; + +ensureSemanticCacheDbBridge(); const cacheConfigUpdateSchema = z.object({ semanticCacheEnabled: z.boolean().optional(), semanticCacheMaxSize: z.number().positive().optional(), semanticCacheTTL: z.number().positive().optional(), + semanticCacheBackend: z.enum(["memory", "redis"]).optional(), + semanticCacheThreshold: z.number().min(0).max(1).optional(), + semanticCacheEmbeddingProvider: z.string().trim().optional(), + semanticCacheEmbeddingModel: z.string().trim().optional(), + semanticCacheEmbeddingDimension: z.number().positive().nullable().optional(), + semanticCacheEmbeddingBaseUrl: z.string().trim().nullable().optional(), + semanticCacheEmbeddingApiKey: z.string().trim().nullable().optional(), + semanticCacheRedisUrl: z.string().trim().nullable().optional(), + semanticCacheRedisPrefix: z.string().trim().optional(), + semanticCacheRequireZeroTemp: z.boolean().optional(), promptCacheEnabled: z.boolean().optional(), promptCacheStrategy: z.enum(["auto", "system-only", "manual"]).optional(), alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(), @@ -24,6 +39,16 @@ const CACHE_CONFIG_KEYS = [ "semanticCacheEnabled", "semanticCacheMaxSize", "semanticCacheTTL", + "semanticCacheBackend", + "semanticCacheThreshold", + "semanticCacheEmbeddingProvider", + "semanticCacheEmbeddingModel", + "semanticCacheEmbeddingDimension", + "semanticCacheEmbeddingBaseUrl", + "semanticCacheEmbeddingApiKey", + "semanticCacheRedisUrl", + "semanticCacheRedisPrefix", + "semanticCacheRequireZeroTemp", "promptCacheEnabled", "promptCacheStrategy", "alwaysPreserveClientCache", @@ -33,8 +58,18 @@ const CACHE_CONFIG_KEYS = [ const DEFAULTS = { semanticCacheEnabled: true, - semanticCacheMaxSize: 100, + semanticCacheMaxSize: 1000, semanticCacheTTL: 1800000, + semanticCacheBackend: "memory", + semanticCacheThreshold: 0.8, + semanticCacheEmbeddingProvider: "lemonade", + semanticCacheEmbeddingModel: "harrier-oss-v1-0.6b", + semanticCacheEmbeddingDimension: 1024, + semanticCacheEmbeddingBaseUrl: "", + semanticCacheEmbeddingApiKey: "", + semanticCacheRedisUrl: "", + semanticCacheRedisPrefix: "omniroute:semcache:", + semanticCacheRequireZeroTemp: true, promptCacheEnabled: true, promptCacheStrategy: "auto", alwaysPreserveClientCache: "auto", @@ -55,7 +90,10 @@ export async function GET(request: NextRequest) { // idempotencyWindowMs is not part of the databaseSettings "cache" section — // it lives in the flat general settings (src/lib/db/settings.ts), which is // where src/lib/idempotencyLayer.ts actually reads it from. - const flatSettings = await getSettings(); + const [flatSettings, embeddingOptions] = await Promise.all([ + getSettings(), + getEmbeddingOptions(), + ]); const config: Record = {}; for (const key of CACHE_CONFIG_KEYS) { if (key === "idempotencyWindowMs") { @@ -64,6 +102,7 @@ export async function GET(request: NextRequest) { config[key] = (cache as Record)[key] ?? DEFAULTS[key]; } } + config.embeddingOptions = embeddingOptions; return NextResponse.json(config); } catch (error) { return NextResponse.json({ error: String(error) }, { status: 500 }); @@ -100,6 +139,36 @@ export async function PUT(request: NextRequest) { if (body.semanticCacheTTL !== undefined) { updates.semanticCacheTTL = body.semanticCacheTTL; } + if (body.semanticCacheBackend !== undefined) { + updates.semanticCacheBackend = body.semanticCacheBackend; + } + if (body.semanticCacheThreshold !== undefined) { + updates.semanticCacheThreshold = body.semanticCacheThreshold; + } + if (body.semanticCacheEmbeddingProvider !== undefined) { + updates.semanticCacheEmbeddingProvider = body.semanticCacheEmbeddingProvider; + } + if (body.semanticCacheEmbeddingModel !== undefined) { + updates.semanticCacheEmbeddingModel = body.semanticCacheEmbeddingModel; + } + if (body.semanticCacheEmbeddingDimension !== undefined) { + updates.semanticCacheEmbeddingDimension = body.semanticCacheEmbeddingDimension ?? undefined; + } + if (body.semanticCacheEmbeddingBaseUrl !== undefined) { + updates.semanticCacheEmbeddingBaseUrl = body.semanticCacheEmbeddingBaseUrl ?? undefined; + } + if (body.semanticCacheEmbeddingApiKey !== undefined) { + updates.semanticCacheEmbeddingApiKey = body.semanticCacheEmbeddingApiKey ?? undefined; + } + if (body.semanticCacheRedisUrl !== undefined) { + updates.semanticCacheRedisUrl = body.semanticCacheRedisUrl ?? undefined; + } + if (body.semanticCacheRedisPrefix !== undefined) { + updates.semanticCacheRedisPrefix = body.semanticCacheRedisPrefix; + } + if (body.semanticCacheRequireZeroTemp !== undefined) { + updates.semanticCacheRequireZeroTemp = body.semanticCacheRequireZeroTemp; + } if (body.promptCacheEnabled !== undefined) { updates.promptCacheEnabled = body.promptCacheEnabled; } @@ -117,6 +186,7 @@ export async function PUT(request: NextRequest) { // which bumps the model-catalog cache version so in-flight responses pick // up the fresh TTL — no separate version bump needed here. updateDatabaseSettings({ cache: updates }); + resetSemanticCacheManager(); // idempotencyWindowMs is not part of the databaseSettings "cache" section — // persist it through the flat general settings module instead (see GET). diff --git a/src/app/api/settings/cache-config/test-embedding/route.ts b/src/app/api/settings/cache-config/test-embedding/route.ts new file mode 100644 index 0000000000..5738742905 --- /dev/null +++ b/src/app/api/settings/cache-config/test-embedding/route.ts @@ -0,0 +1,76 @@ +import { NextResponse } from "next/server"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; +import { createDefaultEmbeddingGenerator } from "@omniroute/open-sse/services/cache/embeddingClient.ts"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; +import { z } from "zod"; +import { isValidationFailure, validateBody } from "@/shared/validation/helpers"; +import { resolveProviderConnectionDetails } from "@/lib/cache/semanticCacheDbBridge"; + +const testEmbeddingSchema = z.object({ + provider: z.string().trim().min(1), + model: z.string().trim().min(1), + baseUrl: z.string().trim().optional(), + apiKey: z.string().trim().optional(), + dimensions: z.number().positive().optional(), +}); + +export async function POST(request: Request) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let rawBody: unknown; + try { + rawBody = await request.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 }); + } + + const validation = validateBody(testEmbeddingSchema, rawBody); + if (isValidationFailure(validation)) { + return validation.response; + } + + const { provider, model, baseUrl, apiKey } = validation.data; + + // Resolve connection details from DB if not explicitly passed + const conn = resolveProviderConnectionDetails(provider); + const effectiveBaseUrl = baseUrl || conn.baseUrl; + const effectiveApiKey = apiKey || conn.apiKey; + + try { + const generator = createDefaultEmbeddingGenerator({ + embeddingProvider: provider, + embeddingModel: model, + embeddingBaseUrl: effectiveBaseUrl, + embeddingApiKey: effectiveApiKey, + }); + + const start = Date.now(); + const result = await generator("OmniRoute semantic cache live probe test"); + const latencyMs = Date.now() - start; + + if (!result || !Array.isArray(result.embedding)) { + return NextResponse.json( + { + ok: false, + error: "Failed to generate embedding (empty response or unsupported endpoint)", + }, + { status: 200 } + ); + } + + return NextResponse.json({ + ok: true, + latencyMs, + dimensions: result.embedding.length, + resolvedBaseUrl: effectiveBaseUrl, + }); + } catch (error: unknown) { + const message = sanitizeErrorMessage(error); + return NextResponse.json( + { ok: false, error: message }, + { status: 200 } // Return 200 with ok: false so the UI can display test error cleanly + ); + } +} diff --git a/src/app/api/v1/chat/completions/route.ts b/src/app/api/v1/chat/completions/route.ts index 8f6f9a1934..13d9359ce5 100644 --- a/src/app/api/v1/chat/completions/route.ts +++ b/src/app/api/v1/chat/completions/route.ts @@ -36,6 +36,7 @@ import { assertCommonChatGptWebModelAvailable, isCommonChatGptWebRetirementError, } from "@/shared/constants/chatgptWebRetirement"; +import { ensureSemanticCacheDbBridge } from "@/lib/cache/semanticCacheDbBridge"; let initPromise = null; @@ -48,6 +49,7 @@ const injectionGuard = createInjectionGuard({ logger: null }); */ function ensureInitialized() { if (!initPromise) { + ensureSemanticCacheDbBridge(); initPromise = Promise.resolve(initTranslators()).then(() => { console.log("[SSE] Translators initialized"); }); diff --git a/src/lib/api/modelTestRunner.ts b/src/lib/api/modelTestRunner.ts index c72248f55a..205b5b63e0 100644 --- a/src/lib/api/modelTestRunner.ts +++ b/src/lib/api/modelTestRunner.ts @@ -259,12 +259,15 @@ export function detectTestKind(modelStr: string, customModel: any, nodeApiType?: !isRerank && (apiFormat === "embeddings" || nodeType === "embeddings" || + customModel?.modelType === "embedding" || supportedEndpoints.includes("embeddings") || lowerModel.includes("embedding") || lowerModel.includes("bge-") || lowerModel.includes("text-embed") || lowerModel.includes("jina-clip") || - lowerModel.includes("colbert")); + lowerModel.includes("colbert") || + lowerModel.includes("harrier-") || + lowerModel.includes("nomic-embed")); return { isRerank, isEmbedding, isAudioTranscription }; } diff --git a/src/lib/cache/semanticCacheDbBridge.ts b/src/lib/cache/semanticCacheDbBridge.ts new file mode 100644 index 0000000000..b71cb82b7e --- /dev/null +++ b/src/lib/cache/semanticCacheDbBridge.ts @@ -0,0 +1,79 @@ +import { getDatabaseSettings } from "@/lib/db/databaseSettings"; +import { getDbInstance } from "@/lib/db/core"; +import { decryptConnectionFields } from "@/lib/db/encryption"; +import { registerSemanticCacheConfigResolver } from "@omniroute/open-sse/config/semanticCacheConfig.ts"; + +let registered = false; + +export function resolveProviderConnectionDetails(provider: string): { + baseUrl?: string; + apiKey?: string; +} { + if (!provider) return {}; + try { + const db = getDbInstance(); + const row = db + .prepare( + "SELECT * FROM provider_connections WHERE provider = ? AND is_active != 0 ORDER BY priority ASC, id ASC LIMIT 1" + ) + .get(provider) as Record | undefined; + + if (!row) return {}; + const decrypted = decryptConnectionFields(row); + let baseUrl: string | undefined; + if (decrypted.provider_specific_data) { + try { + const parsed = + typeof decrypted.provider_specific_data === "string" + ? JSON.parse(decrypted.provider_specific_data) + : decrypted.provider_specific_data; + if (typeof parsed?.baseUrl === "string" && parsed.baseUrl.trim()) { + baseUrl = parsed.baseUrl.trim(); + } + } catch { + // Ignore parse error + } + } + const rawKey = decrypted.api_key || decrypted.apiKey; + const apiKey = typeof rawKey === "string" && rawKey.trim() ? rawKey.trim() : undefined; + return { baseUrl, apiKey }; + } catch { + return {}; + } +} + +export function ensureSemanticCacheDbBridge(): void { + if (registered) return; + registered = true; + registerSemanticCacheConfigResolver(() => { + try { + const s = getDatabaseSettings().cache; + if (!s) return null; + + const conn = s.semanticCacheEmbeddingProvider + ? resolveProviderConnectionDetails(s.semanticCacheEmbeddingProvider) + : {}; + + const embeddingBaseUrl = s.semanticCacheEmbeddingBaseUrl || conn.baseUrl; + const embeddingApiKey = s.semanticCacheEmbeddingApiKey || conn.apiKey; + + return { + enabled: s.semanticCacheEnabled, + backend: s.semanticCacheBackend, + similarityThreshold: s.semanticCacheThreshold, + ttlMs: s.semanticCacheTTL, + maxEntries: s.semanticCacheMaxSize, + embeddingProvider: s.semanticCacheEmbeddingProvider, + embeddingModel: s.semanticCacheEmbeddingModel, + embeddingDimension: s.semanticCacheEmbeddingDimension, + embeddingBaseUrl, + embeddingApiKey, + redisUrl: s.semanticCacheRedisUrl, + redisPrefix: s.semanticCacheRedisPrefix, + requireZeroTemperature: s.semanticCacheRequireZeroTemp, + }; + } catch { + return null; + } + }); +} diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index 0a12729242..2487d0d3b3 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -37,6 +37,16 @@ const LEGACY_FLAT_KEYS: { semanticCacheEnabled: ["semanticCacheEnabled"], semanticCacheMaxSize: ["semanticCacheMaxSize"], semanticCacheTTL: ["semanticCacheTTL"], + semanticCacheBackend: ["semanticCacheBackend"], + semanticCacheThreshold: ["semanticCacheThreshold"], + semanticCacheEmbeddingProvider: ["semanticCacheEmbeddingProvider"], + semanticCacheEmbeddingModel: ["semanticCacheEmbeddingModel"], + semanticCacheEmbeddingDimension: ["semanticCacheEmbeddingDimension"], + semanticCacheEmbeddingBaseUrl: ["semanticCacheEmbeddingBaseUrl"], + semanticCacheEmbeddingApiKey: ["semanticCacheEmbeddingApiKey"], + semanticCacheRedisUrl: ["semanticCacheRedisUrl"], + semanticCacheRedisPrefix: ["semanticCacheRedisPrefix"], + semanticCacheRequireZeroTemp: ["semanticCacheRequireZeroTemp"], promptCacheEnabled: ["promptCacheEnabled"], promptCacheStrategy: ["promptCacheStrategy"], alwaysPreserveClientCache: ["alwaysPreserveClientCache"], @@ -294,7 +304,7 @@ export function updateDatabaseSettings( const sectionValues = nextSettings[section] as Record; for (const [key, value] of Object.entries(sectionValues)) { - insert.run(DATABASE_SETTINGS_NAMESPACE, `${section}.${key}`, JSON.stringify(value)); + insert.run(DATABASE_SETTINGS_NAMESPACE, `${section}.${key}`, JSON.stringify(value ?? null)); } } diff --git a/src/lib/db/models.ts b/src/lib/db/models.ts index 4ec8b75069..0a8e2a8328 100644 --- a/src/lib/db/models.ts +++ b/src/lib/db/models.ts @@ -207,7 +207,12 @@ export async function addCustomModel( // custom OpenAI-compatible video models. Persisted on the model row; the // /v1/videos/generations handler reads it back to pick the job/poll path. generationConfig?: { preset: string }, - isFree?: boolean + isFree?: boolean, + extraMeta?: { + dimensions?: number; + supportedInputTypes?: string[]; + modelType?: "chat" | "embedding" | "image" | "rerank"; + } ) { const db = getDbInstance(); const row = db @@ -235,6 +240,13 @@ export async function addCustomModel( ...(typeof supportsVision === "boolean" ? { supportsVision } : {}), ...(typeof isFree === "boolean" ? { isFree } : {}), ...(generationConfig && generationConfig.preset ? { generationConfig } : {}), + ...(typeof extraMeta?.dimensions === "number" && extraMeta.dimensions > 0 + ? { dimensions: extraMeta.dimensions } + : {}), + ...(Array.isArray(extraMeta?.supportedInputTypes) + ? { supportedInputTypes: extraMeta.supportedInputTypes } + : {}), + ...(typeof extraMeta?.modelType === "string" ? { modelType: extraMeta.modelType } : {}), }; models.push(model); db.prepare( diff --git a/src/lib/db/models/synced.ts b/src/lib/db/models/synced.ts index 92f2c4eb7a..505174d7d5 100644 --- a/src/lib/db/models/synced.ts +++ b/src/lib/db/models/synced.ts @@ -22,6 +22,9 @@ export interface SyncedAvailableModel { // #4264: image-input capability captured at sync time (e.g. OpenRouter // `architecture.input_modalities`/`modality`) so the catalog can surface vision. supportsVision?: boolean; + dimensions?: number; + supportedInputTypes?: string[]; + modelType?: "chat" | "embedding" | "image" | "rerank"; } export type SyncedAvailableModelInput = Omit & { @@ -87,6 +90,19 @@ function normalizeSyncedAvailableModel(model: unknown): SyncedAvailableModel | n ...(typeof record.supportsTools === "boolean" ? { supportsTools: record.supportsTools } : {}), ...(typeof record.supportsVideo === "boolean" ? { supportsVideo: record.supportsVideo } : {}), ...(record.supportsVision === true ? { supportsVision: true } : {}), + ...(typeof record.dimensions === "number" && record.dimensions > 0 + ? { dimensions: record.dimensions } + : {}), + ...(Array.isArray(record.supportedInputTypes) + ? { + supportedInputTypes: record.supportedInputTypes.filter( + (t): t is string => typeof t === "string" && t.length > 0 + ), + } + : {}), + ...(typeof record.modelType === "string" + ? { modelType: record.modelType as "chat" | "embedding" | "image" | "rerank" } + : {}), }; } diff --git a/src/lib/providerModels/modelDiscovery.ts b/src/lib/providerModels/modelDiscovery.ts index 9a5697c54a..dc446f5f7c 100644 --- a/src/lib/providerModels/modelDiscovery.ts +++ b/src/lib/providerModels/modelDiscovery.ts @@ -7,6 +7,7 @@ import { import { CANONICAL_EFFORT_VALUES } from "@/shared/reasoning/effortStandardization"; import { isObsoleteKiroModelAlias } from "@omniroute/open-sse/services/kiroModels.ts"; import { filterSelectableModels } from "@omniroute/open-sse/services/modelLifecycle.ts"; +import { getEmbeddingProvider } from "@omniroute/open-sse/config/embeddingRegistry.ts"; type JsonRecord = Record; @@ -255,6 +256,124 @@ export function isAutoFetchModelsEnabled(providerSpecificData: unknown): boolean return asRecord(providerSpecificData).autoFetchModels === true; } +const KNOWN_EMBEDDING_PREFIXES = [ + "text-embedding-", + "bge-", + "gte-", + "e5-", + "nomic-embed", + "all-minilm", + "embeddinggemma", + "jina-embeddings", + "jina-clip", + "cohere-embed", + "multilingual-e5", +]; + +const KNOWN_EMBEDDING_DIMENSIONS: Record = { + "harrier-oss-v1-0.6b": 1024, + "text-embedding-3-small": 1536, + "text-embedding-3-large": 3072, + "text-embedding-ada-002": 1536, + "bge-m3": 1024, + "bge-large-en-v1.5": 1024, + "bge-small-en-v1.5": 384, + "bge-base-en-v1.5": 768, + "nomic-embed-text": 768, + "all-minilm-l6-v2": 384, + embeddinggemma: 768, +}; + +export function detectModelModality( + record: JsonRecord, + providerId?: string +): { + isEmbedding: boolean; + isImage: boolean; + isRerank: boolean; + dimensions?: number; + supportedInputTypes: string[]; +} { + const rawId = toNonEmptyString(record.id) || toNonEmptyString(record.name) || ""; + const modelLeaf = rawId.toLowerCase().split("/").pop() || ""; + const rawLabels = Array.isArray(record.labels) + ? record.labels + .map((l) => (typeof l === "string" ? l.trim().toLowerCase() : "")) + .filter(Boolean) + : []; + const typeStr = toNonEmptyString(record.type)?.toLowerCase(); + const objStr = toNonEmptyString(record.object)?.toLowerCase(); + const caps = asRecord(record.capabilities); + const rawEndpoints = Array.isArray(record.supportedEndpoints) + ? record.supportedEndpoints.map((e) => (typeof e === "string" ? e.trim().toLowerCase() : "")) + : []; + + const registryProvider = providerId ? getEmbeddingProvider(providerId) : undefined; + const registryModel = registryProvider?.models.find( + (m) => m.id === modelLeaf || m.id === rawId || rawId.endsWith(`/${m.id}`) + ); + + const isRerank = + rawLabels.includes("reranking") || + rawLabels.includes("rerank") || + typeStr === "rerank" || + rawEndpoints.includes("rerank") || + modelLeaf.includes("rerank"); + + const isImage = + !isRerank && + (rawLabels.includes("image") || + rawLabels.includes("images") || + typeStr === "image" || + objStr === "image" || + rawEndpoints.includes("images") || + rawEndpoints.includes("image") || + modelLeaf.startsWith("gpt-image-") || + modelLeaf.startsWith("dall-e-") || + modelLeaf === "chatgpt-image-latest" || + modelLeaf.startsWith("flux-") || + modelLeaf.startsWith("sdxl-") || + modelLeaf.startsWith("stable-diffusion")); + + const isEmbedding = + !isRerank && + !isImage && + (rawLabels.includes("embeddings") || + rawLabels.includes("embedding") || + typeStr === "embedding" || + typeStr === "embeddings" || + objStr === "embedding" || + caps.embeddings === true || + caps.embedding === true || + rawEndpoints.includes("embeddings") || + rawEndpoints.includes("embedding") || + Boolean(registryModel) || + KNOWN_EMBEDDING_PREFIXES.some((prefix) => modelLeaf.includes(prefix))); + + const dimensions = firstPositiveNumber( + record.dimensions, + record.dimension, + record.embedding_dimension, + record.embedding_dimensions, + registryModel?.dimensions, + KNOWN_EMBEDDING_DIMENSIONS[modelLeaf] + ); + + const supportedInputTypes: string[] = Array.isArray(record.supportedInputTypes) + ? record.supportedInputTypes.filter((t): t is string => typeof t === "string" && t.length > 0) + : registryModel?.modalities + ? (registryModel.modalities as string[]) + : ["text"]; + + return { + isEmbedding, + isImage, + isRerank, + dimensions, + supportedInputTypes, + }; +} + export function normalizeDiscoveredModels( models: unknown, providerId?: string @@ -294,6 +413,16 @@ export function normalizeDiscoveredModels( toNonEmptyString(record.displayName) || toNonEmptyString(record.model) || id; + + const modality = detectModelModality(record, providerId); + const modelType = modality.isEmbedding + ? "embedding" + : modality.isRerank + ? "rerank" + : modality.isImage + ? "image" + : "chat"; + const supportedEndpoints = Array.isArray(record.supportedEndpoints) ? Array.from( new Set( @@ -302,7 +431,23 @@ export function normalizeDiscoveredModels( .filter((endpoint): endpoint is string => Boolean(endpoint)) ) ).sort() - : undefined; + : modality.isEmbedding + ? ["embeddings"] + : modality.isRerank + ? ["rerank"] + : modality.isImage + ? ["images"] + : undefined; + + const apiFormat = + toNonEmptyString(record.apiFormat) || + (modality.isEmbedding + ? "embeddings" + : modality.isRerank + ? "rerank" + : modality.isImage + ? "images-generations" + : undefined); const topProvider = asRecord(record.top_provider); @@ -314,6 +459,8 @@ export function normalizeDiscoveredModels( record.inputTokenLimit, record.context_length, record.contextLength, + record.max_context_window, + record.max_tokens, topProvider.context_length ); const outputTokenLimit = firstPositiveNumber( @@ -333,9 +480,7 @@ export function normalizeDiscoveredModels( id, name, source: "imported", - ...(toNonEmptyString(record.apiFormat) - ? { apiFormat: toNonEmptyString(record.apiFormat)! } - : {}), + ...(apiFormat ? { apiFormat } : {}), ...(toNonEmptyString(record.targetFormat) ? { targetFormat: toNonEmptyString(record.targetFormat)! } : {}), @@ -357,6 +502,13 @@ export function normalizeDiscoveredModels( ...(typeof record.supportsTools === "boolean" ? { supportsTools: record.supportsTools } : {}), ...(typeof record.supportsVideo === "boolean" ? { supportsVideo: record.supportsVideo } : {}), ...(supportsVision ? { supportsVision: true } : {}), + ...(typeof modality.dimensions === "number" && modality.dimensions > 0 + ? { dimensions: modality.dimensions } + : {}), + ...(modality.supportedInputTypes.length > 0 + ? { supportedInputTypes: modality.supportedInputTypes } + : {}), + modelType, }); } diff --git a/src/lib/semanticCache.ts b/src/lib/semanticCache.ts index 5e94b04910..5c05843f42 100644 --- a/src/lib/semanticCache.ts +++ b/src/lib/semanticCache.ts @@ -244,6 +244,27 @@ export function getCachedResponse(signature) { return null; } +/** + * Record a semantic cache hit: increments hit count for the entry in SQLite + * and increments global hit metrics (hits and tokens_saved). + */ +export function recordSemanticCacheHit(signature: string, tokensSaved = 0): void { + try { + const db = getDbInstance(); + if (signature) { + db.prepare( + "UPDATE semantic_cache SET hit_count = hit_count + 1 WHERE signature = ? OR prompt_hash = ?" + ).run(signature, signature.slice(0, 16)); + } + incrementMetric("hits"); + if (tokensSaved > 0) { + incrementMetric("tokens_saved", tokensSaved); + } + } catch { + // DB not available — fail open + } +} + /** * Store a response in cache. * @param {string} signature diff --git a/src/shared/validation/schemas/provider.ts b/src/shared/validation/schemas/provider.ts index c93e563ab7..cb1a9c8022 100644 --- a/src/shared/validation/schemas/provider.ts +++ b/src/shared/validation/schemas/provider.ts @@ -291,6 +291,9 @@ export const providerModelMutationSchema = z.object({ // the same flag flows through `getCustomVisionCapabilityFields()` in the /v1/models // catalog. `null` clears a manual override back to the id-based heuristic. supportsVision: z.boolean().nullable().optional(), + dimensions: z.number().int().positive().nullable().optional(), + supportedInputTypes: z.array(z.string()).optional(), + modelType: z.enum(["chat", "embedding", "image", "rerank"]).optional(), isFree: z.boolean().nullable().optional(), normalizeToolCallId: z.boolean().optional(), preserveOpenAIDeveloperRole: z.boolean().nullable().optional(), @@ -519,9 +522,7 @@ export const updateProviderConnectionSchema = z errorCode: z.union([z.string(), z.null()]).optional(), rateLimitedUntil: z.union([z.string(), z.null()]).optional(), lastTested: z.union([z.string(), z.null()]).optional(), - healthCheckInterval: z - .union([z.null(), z.coerce.number().int().min(0).max(1440)]) - .optional(), + healthCheckInterval: z.union([z.null(), z.coerce.number().int().min(0).max(1440)]).optional(), group: z.union([z.string().max(100), z.null()]).optional(), maxConcurrent: z.union([z.null(), z.coerce.number().int().min(0)]).optional(), // Per-window quota cutoffs. Map keys are window names (e.g. "window5h", diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts index 2b4820e198..3a5246b40d 100644 --- a/src/types/databaseSettings.ts +++ b/src/types/databaseSettings.ts @@ -32,6 +32,16 @@ export interface DatabaseSettings { semanticCacheEnabled: boolean; semanticCacheMaxSize: number; semanticCacheTTL: number; + semanticCacheBackend?: "memory" | "redis"; + semanticCacheThreshold?: number; + semanticCacheEmbeddingProvider?: string; + semanticCacheEmbeddingModel?: string; + semanticCacheEmbeddingDimension?: number; + semanticCacheEmbeddingBaseUrl?: string; + semanticCacheEmbeddingApiKey?: string; + semanticCacheRedisUrl?: string; + semanticCacheRedisPrefix?: string; + semanticCacheRequireZeroTemp?: boolean; promptCacheEnabled: boolean; promptCacheStrategy: "auto" | "system-only" | "manual"; alwaysPreserveClientCache: "auto" | "always" | "never"; @@ -99,8 +109,18 @@ export const DEFAULT_DATABASE_SETTINGS: Omit { + const putResponse = await cacheConfigRoute.PUT( + makeJsonRequest("PUT", { + semanticCacheEnabled: true, + semanticCacheBackend: "redis", + semanticCacheThreshold: 0.88, + semanticCacheEmbeddingProvider: "lemonade", + semanticCacheEmbeddingModel: "harrier-oss-v1-0.6b", + semanticCacheEmbeddingDimension: 1024, + semanticCacheRedisUrl: "redis://192.168.31.147:6379", + semanticCacheRequireZeroTemp: false, + }) as never + ); + assert.equal(putResponse.status, 200); + + const getResponse = await cacheConfigRoute.GET(makeJsonRequest("GET") as never); + const getBody = await getResponse.json(); + assert.equal(getBody.semanticCacheBackend, "redis"); + assert.equal(getBody.semanticCacheThreshold, 0.88); + assert.equal(getBody.semanticCacheEmbeddingProvider, "lemonade"); + assert.equal(getBody.semanticCacheEmbeddingModel, "harrier-oss-v1-0.6b"); + assert.equal(getBody.semanticCacheEmbeddingDimension, 1024); + assert.equal(getBody.semanticCacheRedisUrl, "redis://192.168.31.147:6379"); + assert.equal(getBody.semanticCacheRequireZeroTemp, false); + }); + + await t.test("embedding-options route returns candidate providers and models", async () => { + const embeddingOptionsRoute = + await import("../../src/app/api/settings/cache-config/embedding-options/route.ts"); + const response = await embeddingOptionsRoute.GET( + new Request("http://localhost/api/settings/cache-config/embedding-options") as never + ); + assert.equal(response.status, 200); + const body = await response.json(); + assert.ok(Array.isArray(body.providers)); + assert.ok(body.providers.length > 0); + + const lemonade = body.providers.find( + (p: { id: string; models: Array<{ rawId: string; dimensions?: number }> }) => + p.id === "lemonade" + ); + assert.ok(lemonade, "lemonade provider option should be returned"); + const harrier = lemonade.models.find( + (m: { rawId: string; dimensions?: number }) => m.rawId === "harrier-oss-v1-0.6b" + ); + assert.ok(harrier, "harrier-oss-v1-0.6b model should be present in lemonade models"); + assert.equal(harrier.dimensions, 1024); + }); }); diff --git a/tests/unit/model-embedding-discovery-and-cache.test.ts b/tests/unit/model-embedding-discovery-and-cache.test.ts new file mode 100644 index 0000000000..f2aaed8cc5 --- /dev/null +++ b/tests/unit/model-embedding-discovery-and-cache.test.ts @@ -0,0 +1,187 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + detectModelModality, + normalizeDiscoveredModels, +} from "@/lib/providerModels/modelDiscovery"; +import type { SyncedAvailableModel } from "@/lib/db/models"; +import { + getModelEndpointDecision, + isChatSelectableModel, + filterChatSelectableModels, +} from "../../open-sse/services/modelEndpointPolicy.ts"; +import { detectTestKind } from "@/lib/api/modelTestRunner"; + +test("detectModelModality flags Lemonade embeddings model and pulls dimensions and context length", () => { + // Lemonade verbatim /v1/models shape for harrier-oss-v1-0.6b + const lemonadeRecord = { + id: "harrier-oss-v1-0.6b", + object: "model", + owned_by: "lemonade", + labels: ["custom", "embeddings"], + context_length: 32768, + max_context_window: 32768, + }; + + const modality = detectModelModality(lemonadeRecord, "lemonade"); + assert.equal(modality.isEmbedding, true); + assert.equal(modality.isImage, false); + assert.equal(modality.isRerank, false); + assert.equal(modality.dimensions, 1024, "Should resolve 1024 dimensions from registry"); + assert.deepEqual(modality.supportedInputTypes, ["text"]); + + // Normalize discovered model + const synced = normalizeDiscoveredModels([lemonadeRecord], "lemonade"); + assert.equal(synced.length, 1); + const [model] = synced; + assert.equal(model.id, "harrier-oss-v1-0.6b"); + assert.equal(model.modelType, "embedding"); + assert.equal(model.apiFormat, "embeddings"); + assert.deepEqual(model.supportedEndpoints, ["embeddings"]); + assert.equal(model.inputTokenLimit, 32768); + assert.equal(model.dimensions, 1024); + assert.deepEqual(model.supportedInputTypes, ["text"]); +}); + +test("detectModelModality identifies reranking and image models from labels", () => { + const rerankRecord = { + id: "bge-reranker-large", + labels: ["custom", "reranking"], + }; + const rerankModality = detectModelModality(rerankRecord, "custom"); + assert.equal(rerankModality.isRerank, true); + assert.equal(rerankModality.isEmbedding, false); + + const imageRecord = { + id: "flux-1-schnell", + labels: ["image"], + }; + const imageModality = detectModelModality(imageRecord, "custom"); + assert.equal(imageModality.isImage, true); + assert.equal(imageModality.isEmbedding, false); +}); + +test("modelEndpointPolicy excludes embedding models from chat completions", () => { + // Upstream explicit endpoints with embeddings + assert.deepEqual(getModelEndpointDecision("lemonade", "harrier-oss-v1-0.6b", ["embeddings"]), { + kind: "embedding", + chatSelectable: false, + reason: "explicit-endpoints", + }); + + // OpenAI text-embedding-3-small provider policy + assert.deepEqual(getModelEndpointDecision("openai", "text-embedding-3-small"), { + kind: "embedding", + chatSelectable: false, + reason: "provider-policy", + }); + + // isChatSelectableModel returns false + assert.equal( + isChatSelectableModel("lemonade", { + id: "harrier-oss-v1-0.6b", + supportedEndpoints: ["embeddings"], + }), + false + ); + + // Filter removes embedding model from chat candidates + const filtered = filterChatSelectableModels("lemonade", [ + { id: "qwen2.5-coder-7b", supportedEndpoints: ["chat"] }, + { id: "harrier-oss-v1-0.6b", supportedEndpoints: ["embeddings"] }, + ]); + assert.deepEqual( + filtered.map((m) => m.id), + ["qwen2.5-coder-7b"] + ); +}); + +test("detectTestKind in modelTestRunner detects embedding test probe for harrier-oss-v1-0.6b", () => { + // Test with modelType flag + const result1 = detectTestKind("lemonade/harrier-oss-v1-0.6b", { + modelType: "embedding", + dimensions: 1024, + } as unknown as SyncedAvailableModel); + assert.equal(result1.isEmbedding, true); + assert.equal(result1.isRerank, false); + assert.equal(result1.isAudioTranscription, false); + + // Test with supportedEndpoints + const result2 = detectTestKind("lemonade/harrier-oss-v1-0.6b", { + supportedEndpoints: ["embeddings"], + } as unknown as SyncedAvailableModel); + assert.equal(result2.isEmbedding, true); + + // Test with apiFormat + const result3 = detectTestKind("lemonade/harrier-oss-v1-0.6b", { + apiFormat: "embeddings", + } as unknown as SyncedAvailableModel); + assert.equal(result3.isEmbedding, true); +}); + +test("test-embedding route validates inputs and generates embeddings via live Lemonade", async () => { + const testEmbeddingRoute = + await import("../../src/app/api/settings/cache-config/test-embedding/route.ts"); + + const req = new Request("http://localhost/api/settings/cache-config/test-embedding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: "lemonade", + model: "harrier-oss-v1-0.6b", + baseUrl: "http://192.168.31.147:13305/v1", + apiKey: "lemonade", + dimensions: 1024, + }), + }); + + const response = await testEmbeddingRoute.POST(req); + assert.equal(response.status, 200); + const data = await response.json(); + assert.equal(data.ok, true); + assert.equal(data.dimensions, 1024); + assert.ok(typeof data.latencyMs === "number" && data.latencyMs > 0); +}); + +test("test-embedding route automatically resolves connection details from DB when not passed", async () => { + const { getDbInstance } = await import("@/lib/db/core"); + const testEmbeddingRoute = + await import("../../src/app/api/settings/cache-config/test-embedding/route.ts"); + + const db = getDbInstance(); + db.prepare( + ` + INSERT OR REPLACE INTO provider_connections ( + id, provider, name, auth_type, api_key, provider_specific_data, is_active, created_at, updated_at + ) VALUES ( + 'test-conn-lemonade-1', + 'lemonade', + 'Lemonade Local Server', + 'apikey', + 'lemonade', + '{"baseUrl":"http://192.168.31.147:13305/"}', + 1, + CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP + ) + ` + ).run(); + + // Omit baseUrl and apiKey from payload + const req = new Request("http://localhost/api/settings/cache-config/test-embedding", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: "lemonade", + model: "harrier-oss-v1-0.6b", + dimensions: 1024, + }), + }); + + const response = await testEmbeddingRoute.POST(req); + assert.equal(response.status, 200); + const data = await response.json(); + assert.equal(data.ok, true, `Expected ok=true but got error: ${data.error}`); + assert.equal(data.dimensions, 1024); + assert.equal(data.resolvedBaseUrl, "http://192.168.31.147:13305/"); +});