diff --git a/open-sse/config/semanticCacheConfig.ts b/open-sse/config/semanticCacheConfig.ts index 03526f0403..241aadaedd 100644 --- a/open-sse/config/semanticCacheConfig.ts +++ b/open-sse/config/semanticCacheConfig.ts @@ -11,7 +11,13 @@ export type SemanticCacheBackend = "memory" | "redis"; export type SemanticCacheType = "direct" | "semantic" | "both"; export interface SemanticCacheConfig { - /** Master toggle for semantic caching. */ + /** + * Master toggle for the dual-layer (in-memory/Redis vector) manager. OFF by default: + * the legacy SQLite exact-match cache (`semanticCacheEnabled`) keeps working on its + * own; this layer is opt-in via the `semanticCacheVectorEnabled` setting or + * `OMNIROUTE_SEMANTIC_CACHE_ENABLED=true`, because enabling it calls an embedding + * endpoint on every cacheable request (#14159 re-land of #12630). + */ enabled: boolean; /** Storage and vector backend. Defaults to "memory". */ backend: SemanticCacheBackend; @@ -52,7 +58,7 @@ export interface SemanticCacheConfig { } export const DEFAULT_SEMANTIC_CACHE_CONFIG: SemanticCacheConfig = { - enabled: true, + enabled: false, backend: "memory", similarityThreshold: 0.8, ttlMs: 1800000, // 30 minutes diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index c67b2b4dd4..397dae6eaa 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -5577,6 +5577,9 @@ export async function handleChatCore({ headers: clientRawRequest?.headers, translatedResponse, model, + // The dual-layer manager scopes entries per provider (cacheByProvider); + // lookup passes the resolved provider, so the write must too (#14159). + provider, apiKeyId: apiKeyInfo?.id ?? undefined, usage, log, @@ -6081,6 +6084,7 @@ export async function handleChatCore({ body: bodyForCacheWrite, headers: clientRawRequest?.headers, model, + provider, apiKeyId: apiKeyInfo?.id ?? undefined, streamUsage, log, diff --git a/open-sse/handlers/chatCore/semanticCache.ts b/open-sse/handlers/chatCore/semanticCache.ts index 92e53e624c..803dc62770 100644 --- a/open-sse/handlers/chatCore/semanticCache.ts +++ b/open-sse/handlers/chatCore/semanticCache.ts @@ -106,7 +106,10 @@ export async function checkSemanticCache({ providerRequest: null, providerResponse: null, clientResponse: cached, - cacheSource: hitType === "semantic" ? "semantic_similarity" : "semantic", + // Both hit types are served without an upstream call; attemptLogging only + // knows "semantic" | "upstream", so a similarity hit must not fall through + // to "upstream" (#14159). The hit type is surfaced via the response headers. + cacheSource: "semantic", }); // Finalize by exact request id (#12910): a (model, provider, connectionId) // tuple can match the wrong in-flight request when connectionId is null or @@ -148,8 +151,12 @@ export async function checkSemanticCache({ const headers: Record = { "Content-Type": cachedSse ? "text/event-stream" : "application/json", - [OMNIROUTE_RESPONSE_HEADERS.cache]: - hitType === "semantic" ? "HIT (semantic)" : "HIT (exact)", + // Keep the legacy `HIT` value verbatim: consumers match it exactly + // (tests/unit/chatcore-semantic-cache.test.ts and the chat-route suites). + // A similarity hit is distinguished by X-OmniRoute-Cache-Similarity below. + [OMNIROUTE_RESPONSE_HEADERS.cache]: "HIT", + // Marker for latency measurement tools: this response served from cache + // has synthetic (near-zero) latency, not real upstream latency. [OMNIROUTE_RESPONSE_HEADERS.cacheLatency]: "synthetic", [OMNIROUTE_RESPONSE_HEADERS.savingsTokens]: String(tokensSaved), }; diff --git a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts index 566454c24e..8624b9ce95 100644 --- a/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts +++ b/open-sse/handlers/chatCore/streamingSemanticCacheStore.ts @@ -14,7 +14,7 @@ import { outputContractOf, setCachedResponse as defaultSetCachedResponse, isCacheableForWrite as defaultIsCacheableForWrite, - isTruncatedCompletion as defaultIsTruncatedCompletion, + isTruncatedStreamBody as defaultIsTruncatedStreamBody, } from "@/lib/semanticCache"; import { isSmallEnoughForSemanticCache as defaultIsSmallEnough } from "../../utils/estimateSize.ts"; import { getSemanticCacheManager } from "../../services/cache/semanticCacheManager.ts"; @@ -31,7 +31,7 @@ type CacheBody = { export interface StreamingSemanticCacheStoreDeps { isCacheableForWrite: typeof defaultIsCacheableForWrite; /** Optional so pre-existing callers/tests with partial deps keep working. */ - isTruncatedCompletion?: typeof defaultIsTruncatedCompletion; + isTruncatedStreamBody?: typeof defaultIsTruncatedStreamBody; isSmallEnoughForSemanticCache: typeof defaultIsSmallEnough; generateSignature: typeof defaultGenerateSignature; setCachedResponse: typeof defaultSetCachedResponse; @@ -39,7 +39,7 @@ export interface StreamingSemanticCacheStoreDeps { const DEFAULT_DEPS: StreamingSemanticCacheStoreDeps = { isCacheableForWrite: defaultIsCacheableForWrite, - isTruncatedCompletion: defaultIsTruncatedCompletion, + isTruncatedStreamBody: defaultIsTruncatedStreamBody, isSmallEnoughForSemanticCache: defaultIsSmallEnough, generateSignature: defaultGenerateSignature, setCachedResponse: defaultSetCachedResponse, @@ -112,7 +112,7 @@ export function storeStreamingSemanticCacheResponse( args.streamStatus !== 200 || !args.streamResponseBody || !deps.isCacheableForWrite(args.body, args.headers) || - (deps.isTruncatedCompletion ?? defaultIsTruncatedCompletion)(args.streamResponseBody) + (deps.isTruncatedStreamBody ?? defaultIsTruncatedStreamBody)(args.streamResponseBody) ) { return; } diff --git a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx index 0bbb40298a..517ce82c28 100644 --- a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx @@ -28,6 +28,7 @@ interface CacheConfigResponse { semanticCacheEnabled?: boolean; semanticCacheMaxSize?: number; semanticCacheTTL?: number; + semanticCacheVectorEnabled?: boolean; semanticCacheBackend?: "memory" | "redis"; semanticCacheThreshold?: number; semanticCacheEmbeddingProvider?: string; @@ -58,6 +59,8 @@ export default function CacheSettingsTab() { // Semantic Cache State const [semEnabled, setSemEnabled] = useState(true); + // Vector-similarity layer is opt-in (#14159): off unless the operator turns it on. + const [semVectorEnabled, setSemVectorEnabled] = useState(false); const [semBackend, setSemBackend] = useState<"memory" | "redis">("memory"); const [semThreshold, setSemThreshold] = useState(0.8); const [semTtlMinutes, setSemTtlMinutes] = useState(30); @@ -113,6 +116,9 @@ export default function CacheSettingsTab() { if (config.semanticCacheEnabled !== undefined) { setSemEnabled(config.semanticCacheEnabled); } + if (config.semanticCacheVectorEnabled !== undefined) { + setSemVectorEnabled(config.semanticCacheVectorEnabled); + } if (config.semanticCacheBackend === "redis" || config.semanticCacheBackend === "memory") { setSemBackend(config.semanticCacheBackend); } @@ -250,6 +256,7 @@ export default function CacheSettingsTab() { const payload = { semanticCacheEnabled: semEnabled, + semanticCacheVectorEnabled: semVectorEnabled, semanticCacheBackend: semBackend, semanticCacheThreshold: Number(semThreshold), semanticCacheTTL: semTtlMinutes * 60000, @@ -340,8 +347,8 @@ export default function CacheSettingsTab() {

- Local vector-similarity cache. Reuses high-confidence matching responses to cut - latency and upstream token costs. + Exact-match response cache with an optional vector-similarity layer. Reuses matching + responses to cut latency and upstream token costs.

+ {/* Vector-similarity layer opt-in (default off) */} +
+
+

+ Vector Similarity Layer (embeddings) +

+

+ Off by default. When on, every cacheable request is embedded via the provider + below so near-duplicate prompts can reuse a cached answer. Exact-match caching + keeps working without it. +

+
+ +
+ {/* Provider & Model Selection Row */}
diff --git a/src/app/api/settings/cache-config/route.ts b/src/app/api/settings/cache-config/route.ts index bf6f440325..ff6336066b 100644 --- a/src/app/api/settings/cache-config/route.ts +++ b/src/app/api/settings/cache-config/route.ts @@ -18,6 +18,7 @@ const cacheConfigUpdateSchema = z.object({ semanticCacheEnabled: z.boolean().optional(), semanticCacheMaxSize: z.number().positive().optional(), semanticCacheTTL: z.number().positive().optional(), + semanticCacheVectorEnabled: z.boolean().optional(), semanticCacheBackend: z.enum(["memory", "redis"]).optional(), semanticCacheThreshold: z.number().min(0).max(1).optional(), semanticCacheEmbeddingProvider: z.string().trim().optional(), @@ -39,6 +40,7 @@ const CACHE_CONFIG_KEYS = [ "semanticCacheEnabled", "semanticCacheMaxSize", "semanticCacheTTL", + "semanticCacheVectorEnabled", "semanticCacheBackend", "semanticCacheThreshold", "semanticCacheEmbeddingProvider", @@ -60,6 +62,7 @@ const DEFAULTS = { semanticCacheEnabled: true, semanticCacheMaxSize: 1000, semanticCacheTTL: 1800000, + semanticCacheVectorEnabled: false, semanticCacheBackend: "memory", semanticCacheThreshold: 0.8, semanticCacheEmbeddingProvider: "lemonade", @@ -143,6 +146,9 @@ export async function PUT(request: NextRequest) { if (body.semanticCacheTTL !== undefined) { updates.semanticCacheTTL = body.semanticCacheTTL; } + if (body.semanticCacheVectorEnabled !== undefined) { + updates.semanticCacheVectorEnabled = body.semanticCacheVectorEnabled; + } if (body.semanticCacheBackend !== undefined) { updates.semanticCacheBackend = body.semanticCacheBackend; } diff --git a/src/app/api/settings/cache-config/test-embedding/route.ts b/src/app/api/settings/cache-config/test-embedding/route.ts index 5738742905..756081a26b 100644 --- a/src/app/api/settings/cache-config/test-embedding/route.ts +++ b/src/app/api/settings/cache-config/test-embedding/route.ts @@ -28,7 +28,10 @@ export async function POST(request: Request) { const validation = validateBody(testEmbeddingSchema, rawBody); if (isValidationFailure(validation)) { - return validation.response; + // `validateBody()` returns `{ success, error }` — there is no `.response` + // (that shape belongs to `validatedJsonBody()`); returning `undefined` here + // crashed the route on any invalid payload (TS2339 caught by api-typecheck). + return NextResponse.json({ error: validation.error }, { status: 400 }); } const { provider, model, baseUrl, apiKey } = validation.data; diff --git a/src/lib/cache/semanticCacheDbBridge.ts b/src/lib/cache/semanticCacheDbBridge.ts index b71cb82b7e..feac98a7c8 100644 --- a/src/lib/cache/semanticCacheDbBridge.ts +++ b/src/lib/cache/semanticCacheDbBridge.ts @@ -58,7 +58,10 @@ export function ensureSemanticCacheDbBridge(): void { const embeddingApiKey = s.semanticCacheEmbeddingApiKey || conn.apiKey; return { - enabled: s.semanticCacheEnabled, + // The vector layer is opt-in (#14159): it only runs when the operator turned + // on BOTH the master semantic-cache toggle and the vector-layer toggle. With + // it off, chatCore behaves exactly like the legacy SQLite exact-match cache. + enabled: s.semanticCacheEnabled !== false && s.semanticCacheVectorEnabled === true, backend: s.semanticCacheBackend, similarityThreshold: s.semanticCacheThreshold, ttlMs: s.semanticCacheTTL, diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index a3dd2aac82..8cf49347b2 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -37,6 +37,7 @@ const LEGACY_FLAT_KEYS: { semanticCacheEnabled: ["semanticCacheEnabled"], semanticCacheMaxSize: ["semanticCacheMaxSize"], semanticCacheTTL: ["semanticCacheTTL"], + semanticCacheVectorEnabled: ["semanticCacheVectorEnabled"], semanticCacheBackend: ["semanticCacheBackend"], semanticCacheThreshold: ["semanticCacheThreshold"], semanticCacheEmbeddingProvider: ["semanticCacheEmbeddingProvider"], diff --git a/src/lib/providerModels/modelDiscovery.ts b/src/lib/providerModels/modelDiscovery.ts index 304436bcb8..415700d381 100644 --- a/src/lib/providerModels/modelDiscovery.ts +++ b/src/lib/providerModels/modelDiscovery.ts @@ -424,6 +424,15 @@ const KNOWN_EMBEDDING_PREFIXES = [ "multilingual-e5", ]; +/** Mirrors CHAT_ENDPOINTS in open-sse/services/modelEndpointPolicy.ts. */ +const CHAT_ENDPOINT_HINTS = new Set([ + "chat", + "chat-completions", + "chat/completions", + "messages", + "responses", +]); + const KNOWN_EMBEDDING_DIMENSIONS: Record = { "harrier-oss-v1-0.6b": 1024, "text-embedding-3-small": 1536, @@ -467,14 +476,22 @@ export function detectModelModality( (m) => m.id === modelLeaf || m.id === rawId || rawId.endsWith(`/${m.id}`) ); + // An explicit chat endpoint is authoritative: a model that upstream says serves + // chat (e.g. `supportedEndpoints: ["chat", "embeddings"]`) must never be + // downgraded to embedding/rerank/image by the id/label heuristics below — + // that would drop it from the chat catalog (#14159 re-land of #12630). + const hasChatEndpoint = rawEndpoints.some((endpoint) => CHAT_ENDPOINT_HINTS.has(endpoint)); + const isRerank = - rawLabels.includes("reranking") || - rawLabels.includes("rerank") || - typeStr === "rerank" || - rawEndpoints.includes("rerank") || - modelLeaf.includes("rerank"); + !hasChatEndpoint && + (rawLabels.includes("reranking") || + rawLabels.includes("rerank") || + typeStr === "rerank" || + rawEndpoints.includes("rerank") || + modelLeaf.includes("rerank")); const isImage = + !hasChatEndpoint && !isRerank && (rawLabels.includes("image") || rawLabels.includes("images") || @@ -490,6 +507,7 @@ export function detectModelModality( modelLeaf.startsWith("stable-diffusion")); const isEmbedding = + !hasChatEndpoint && !isRerank && !isImage && (rawLabels.includes("embeddings") || @@ -569,13 +587,17 @@ export function normalizeDiscoveredModels( id; const modality = detectModelModality(record, providerId); + // Only non-chat modalities are stamped on the synced row. Chat models keep the + // tip's exact shape (no `modelType`/`supportedInputTypes` defaults) so the + // import-mode diff stays stable and existing catalog snapshots do not churn. const modelType = modality.isEmbedding ? "embedding" : modality.isRerank ? "rerank" : modality.isImage ? "image" - : "chat"; + : undefined; + const explicitInputTypes = Array.isArray(record.supportedInputTypes); const supportedEndpoints = Array.isArray(record.supportedEndpoints) ? Array.from( @@ -689,10 +711,10 @@ export function normalizeDiscoveredModels( ...(typeof modality.dimensions === "number" && modality.dimensions > 0 ? { dimensions: modality.dimensions } : {}), - ...(modality.supportedInputTypes.length > 0 + ...((modelType || explicitInputTypes) && modality.supportedInputTypes.length > 0 ? { supportedInputTypes: modality.supportedInputTypes } : {}), - modelType, + ...(modelType ? { modelType } : {}), }); } diff --git a/src/lib/semanticCache.ts b/src/lib/semanticCache.ts index baa670e3ab..0913c5a22b 100644 --- a/src/lib/semanticCache.ts +++ b/src/lib/semanticCache.ts @@ -548,6 +548,10 @@ export function isTruncatedCompletion(response: unknown): boolean { * disables caching. */ export function isTruncatedStreamBody(streamBody: unknown): boolean { + // chatCore hands the streaming store the *assembled* body (an object with + // `choices[].finish_reason`), not raw SSE text — so the object shape must be + // checked too or the streaming guard is a no-op in production (#14159). + if (streamBody && typeof streamBody === "object") return isTruncatedCompletion(streamBody); if (typeof streamBody !== "string" || streamBody.length === 0) return false; for (const line of streamBody.split("\n")) { const trimmed = line.trim(); diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts index 14143bed2f..5daa41f196 100644 --- a/src/types/databaseSettings.ts +++ b/src/types/databaseSettings.ts @@ -32,6 +32,12 @@ export interface DatabaseSettings { semanticCacheEnabled: boolean; semanticCacheMaxSize: number; semanticCacheTTL: number; + /** + * Opt-in for the dual-layer vector-similarity cache (#14159). Off by default: + * it makes an embedding call per cacheable request, so it must never be on + * for an operator who only enabled the legacy exact-match cache. + */ + semanticCacheVectorEnabled?: boolean; semanticCacheBackend?: "memory" | "redis"; semanticCacheThreshold?: number; semanticCacheEmbeddingProvider?: string; @@ -120,6 +126,7 @@ export const DEFAULT_DATABASE_SETTINGS: Omit non-null result"); assert.equal(result.success, true, "HIT result.success is true"); const res = result.response as Response; - assert.equal( - res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), - "HIT (exact)", - "X-OmniRoute-Cache: HIT (exact)" - ); + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT", "X-OmniRoute-Cache: HIT"); assert.equal( res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cacheHit), "true", @@ -293,7 +289,7 @@ test("checkSemanticCache returns a streaming SSE HIT (text/event-stream) when st "text/event-stream", "streaming HIT -> text/event-stream" ); - assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT (exact)"); + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT"); const bodyText = await res.text(); assert.ok(bodyText.includes("data: "), "SSE body contains data frames"); assert.ok(bodyText.includes("streamed cached answer"), "SSE body carries the cached content"); @@ -328,7 +324,7 @@ test("checkSemanticCache HITs even when the cached body has no usage (cost falls assert.ok(result, "HIT with no usage -> non-null result"); assert.equal(result.success, true); const res = result.response as Response; - assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT (exact)"); + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT"); // cachedUsage resolves to undefined -> cachedCost = 0 -> the zero-cost sentinel header. assert.equal( res.headers.get(OMNIROUTE_RESPONSE_HEADERS.responseCost), @@ -379,7 +375,7 @@ test("checkSemanticCache HIT bills 0 incremental cost and reports the original c assert.ok(result, "HIT -> non-null result"); const res = result.response as Response; - assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT (exact)"); + assert.equal(res.headers.get(OMNIROUTE_RESPONSE_HEADERS.cache), "HIT"); // Incremental cost billed to the client on a HIT is 0 (no upstream call happened). assert.equal( res.headers.get(OMNIROUTE_RESPONSE_HEADERS.responseCost), @@ -509,13 +505,10 @@ test("checkSemanticCache HIT includes X-OmniRoute-Cache-Latency: synthetic heade ); }); -test("checkSemanticCache HIT finalizes the exact pending request by id (#12910)", async () => { +test("checkSemanticCache HIT finalizes the exact pending request by id", async () => { clearCache(); - const { - clearPendingRequests, - getPendingById, - trackPendingRequest: trackPending, - } = await import("../../src/lib/usage/usageHistory.ts"); + const { clearPendingRequests, getPendingById, trackPendingRequest } = + await import("../../src/lib/usage/usageHistory.ts"); const { getCompletedDetails } = await import("../../src/lib/usage/completedRequestDetails.ts"); clearPendingRequests(); try { @@ -530,7 +523,7 @@ test("checkSemanticCache HIT finalizes the exact pending request by id (#12910)" ], usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 }, }; - const pendingId = trackPending("gpt-4o", "openai", "account-a", true); + const pendingId = trackPendingRequest("gpt-4o", "openai", "account-a", true); assert.ok(pendingId); const { args } = makeHitArgs({ body: { diff --git a/tests/unit/semantic-cache-no-truncated-writes.test.ts b/tests/unit/semantic-cache-no-truncated-writes.test.ts index 47750e4107..aa33d8a02d 100644 --- a/tests/unit/semantic-cache-no-truncated-writes.test.ts +++ b/tests/unit/semantic-cache-no-truncated-writes.test.ts @@ -4,7 +4,7 @@ // a mid-sentence answer that no retry can clear (only a cache flush). // Observed live on OmniRoute against github/gemini-3.5-flash: temperature:0 returned // finish_reason "length" at 93 completion tokens on every call, while the same request -// with x-omniroute-no-cache:true returned a complete 239-token response. (#12885) +// with x-omniroute-no-cache:true returned a complete 239-token response. import { test } from "node:test"; import assert from "node:assert/strict"; @@ -12,14 +12,15 @@ const { storeSemanticCacheResponse } = await import("../../open-sse/handlers/chatCore/semanticCacheStore.ts"); const { storeStreamingSemanticCacheResponse } = await import("../../open-sse/handlers/chatCore/streamingSemanticCacheStore.ts"); -// Real truncation predicate — only the cache backend and the temperature/size +// Real truncation predicates — only the cache backend and the temperature/size // gates are stubbed, so these tests exercise the actual detection logic. -const { isTruncatedCompletion } = await import("../../src/lib/semanticCache.ts"); +const { isTruncatedCompletion, isTruncatedStreamBody } = await import("@/lib/semanticCache"); function deps(stored: unknown[]) { return { isCacheableForWrite: () => true, isTruncatedCompletion, + isTruncatedStreamBody, isSmallEnoughForSemanticCache: () => true, generateSignature: () => "sig", setCachedResponse: (_s: unknown, _m: string, r: unknown, t: number) => stored.push({ r, t }), @@ -63,6 +64,28 @@ test("still caches a complete non-streaming response", () => { }); test("does not cache a streaming response truncated by max_tokens", () => { + const stored: unknown[] = []; + storeStreamingSemanticCacheResponse( + { + enabled: true, + streamStatus: 200, + streamResponseBody: + 'data: {"choices":[{"finish_reason":"length","delta":{"content":"partial"}}]}\n\ndata: [DONE]\n\n', + body: { messages: [{ role: "user", content: "hi" }], temperature: 0 }, + headers: undefined, + model: "gemini-3.5-flash", + streamUsage: { prompt_tokens: 10, completion_tokens: 93 }, + } as never, + deps(stored) + ); + assert.equal(stored.length, 0, "truncated streaming response must not be cached"); +}); + +// #14159 (re-land of #12630): chatCore hands the streaming store the *assembled* +// body (an object with `choices[].finish_reason`), not raw SSE text. The guard must +// therefore also work on that shape — otherwise the streaming half of #12885 is a +// no-op in production. The SSE-string case above is kept as-is (tip contract). +test("does not cache an assembled streaming body truncated by max_tokens", () => { const stored: unknown[] = []; storeStreamingSemanticCacheResponse( { @@ -78,10 +101,10 @@ test("does not cache a streaming response truncated by max_tokens", () => { }, deps(stored) ); - assert.equal(stored.length, 0, "truncated streaming response must not be cached"); + assert.equal(stored.length, 0, "truncated assembled streaming body must not be cached"); }); -test("still caches a complete streaming response", () => { +test("still caches a complete assembled streaming body", () => { const stored: unknown[] = []; storeStreamingSemanticCacheResponse( { diff --git a/tests/unit/semantic-cache-vector-layer-opt-in.test.ts b/tests/unit/semantic-cache-vector-layer-opt-in.test.ts new file mode 100644 index 0000000000..32ba957392 --- /dev/null +++ b/tests/unit/semantic-cache-vector-layer-opt-in.test.ts @@ -0,0 +1,94 @@ +// Regression guard for #14159 (re-land of #12630, dual-layer semantic cache). +// +// The original PR wired the new vector-similarity layer to the pre-existing +// `semanticCacheEnabled` toggle (default ON), so every installation started +// calling an embedding endpoint (lemonade @ localhost:13305 by default) on every +// temperature=0 request — two extra `fetch()`es per chat call, visible as +// `fetchCalls.length === 3` in tests/unit/chat-combo-live-test.test.ts. +// +// Contract enforced here: +// 1. The manager config is OFF by default (no env, no DB override). +// 2. The DB bridge only enables the manager when BOTH the master toggle and the +// new `semanticCacheVectorEnabled` opt-in are on. +// 3. A model whose upstream record declares a chat endpoint is never downgraded +// to embedding/rerank/image by the modality heuristics, and chat models keep +// the tip's row shape (no `modelType`/`supportedInputTypes` defaults). +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-semcache-optin-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; +delete process.env.OMNIROUTE_SEMANTIC_CACHE_ENABLED; + +const core = await import("../../src/lib/db/core.ts"); +const { DEFAULT_SEMANTIC_CACHE_CONFIG, resolveSemanticCacheConfig } = + await import("../../open-sse/config/semanticCacheConfig.ts"); +const { ensureSemanticCacheDbBridge } = + await import("../../src/lib/cache/semanticCacheDbBridge.ts"); +const { getDatabaseSettings, updateDatabaseSettings } = + await import("../../src/lib/db/databaseSettings.ts"); +const { DEFAULT_DATABASE_SETTINGS } = await import("../../src/types/databaseSettings.ts"); +const { detectModelModality, normalizeDiscoveredModels } = + await import("../../src/lib/providerModels/modelDiscovery.ts"); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("dual-layer manager is off by default (no env, no DB override)", () => { + assert.equal(DEFAULT_SEMANTIC_CACHE_CONFIG.enabled, false); + assert.equal(DEFAULT_DATABASE_SETTINGS.cache.semanticCacheVectorEnabled, false); + // Legacy exact-match cache default is untouched (tip contract). + assert.equal(DEFAULT_DATABASE_SETTINGS.cache.semanticCacheEnabled, true); +}); + +test("DB bridge enables the vector layer only when master + vector toggles are both on", () => { + ensureSemanticCacheDbBridge(); + + // Fresh install: master on (legacy default), vector opt-in absent → manager off. + assert.equal(getDatabaseSettings().cache.semanticCacheEnabled, true); + assert.equal(resolveSemanticCacheConfig().enabled, false); + + updateDatabaseSettings({ cache: { semanticCacheVectorEnabled: true } }); + assert.equal(resolveSemanticCacheConfig().enabled, true); + + // Master off wins even when the vector opt-in is on. + updateDatabaseSettings({ cache: { semanticCacheEnabled: false } }); + assert.equal(resolveSemanticCacheConfig().enabled, false); +}); + +test("an explicit chat endpoint vetoes the embedding/rerank/image modality heuristics", () => { + const modality = detectModelModality( + { id: "gemini-custom-preview", supportedEndpoints: ["chat", "embeddings"] }, + "gemini" + ); + assert.equal(modality.isEmbedding, false); + assert.equal(modality.isRerank, false); + assert.equal(modality.isImage, false); + + const [chatModel] = normalizeDiscoveredModels( + [{ id: "gemini-custom-preview", supportedEndpoints: ["chat", "embeddings"] }], + "gemini" + ); + assert.equal(chatModel.modelType, undefined, "chat models carry no modelType stamp"); + assert.equal(chatModel.apiFormat, undefined, "chat models carry no synthesized apiFormat"); + assert.equal( + chatModel.supportedInputTypes, + undefined, + "chat models carry no supportedInputTypes default" + ); + + // A pure embedding record still gets its modality metadata. + const [embeddingModel] = normalizeDiscoveredModels( + [{ id: "text-embedding-3-small", supportedEndpoints: ["embeddings"] }], + "openai" + ); + assert.equal(embeddingModel.modelType, "embedding"); + assert.equal(embeddingModel.apiFormat, "embeddings"); + assert.deepEqual(embeddingModel.supportedInputTypes, ["text"]); +});