From 678048505109d6795db459082e7a7e44097756f2 Mon Sep 17 00:00:00 2001 From: oyi77 Date: Tue, 31 Mar 2026 02:41:30 +0700 Subject: [PATCH] feat(cache): persistent metrics, cache entry browser, settings UI, MCP tools, prefix analyzer Implements remaining features from #813: Phase 1 - Persistent Metrics: - Add cache_metrics table for persistent hit/miss tracking - Semantic cache stats now survive server restarts Phase 2 - Cache Entry Browser: - /api/cache/entries endpoint with search, pagination, delete - CacheEntriesTab component for browsing cached entries Phase 3 - Settings UI: - CacheSettingsTab for semantic/prompt cache configuration - /api/settings/cache-config endpoint Phase 4 - Prefix Analyzer: - src/lib/promptCache/prefixAnalyzer.ts for intelligent caching - Analyzes message arrays to find stable prefixes Phase 5 - Provider Support: - Added deepseek to CACHING_PROVIDERS Phase 6 - MCP Tools: - omniroute_cache_stats tool - omniroute_cache_flush tool Phase 7 - Retention: - cleanOldMetrics() for auto-purge of old entries Closes #813 --- open-sse/mcp-server/schemas/index.ts | 6 + open-sse/mcp-server/schemas/tools.ts | 67 +- open-sse/utils/cacheControlPolicy.ts | 7 +- .../cache/components/CacheEntriesTab.tsx | 174 +++++ src/app/(dashboard)/dashboard/cache/page.tsx | 598 ++++++++++-------- .../settings/components/CacheSettingsTab.tsx | 191 ++++++ .../(dashboard)/dashboard/settings/page.tsx | 2 + src/app/api/cache/entries/route.ts | 95 +++ src/app/api/settings/cache-config/route.ts | 73 +++ src/i18n/messages/en.json | 24 +- src/lib/promptCache/index.ts | 1 + src/lib/promptCache/prefixAnalyzer.ts | 77 +++ src/lib/semanticCache.ts | 85 ++- 13 files changed, 1094 insertions(+), 306 deletions(-) create mode 100644 src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx create mode 100644 src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx create mode 100644 src/app/api/cache/entries/route.ts create mode 100644 src/app/api/settings/cache-config/route.ts create mode 100644 src/lib/promptCache/index.ts create mode 100644 src/lib/promptCache/prefixAnalyzer.ts diff --git a/open-sse/mcp-server/schemas/index.ts b/open-sse/mcp-server/schemas/index.ts index 99a947f3ff..ed1a5c9ba6 100644 --- a/open-sse/mcp-server/schemas/index.ts +++ b/open-sse/mcp-server/schemas/index.ts @@ -60,6 +60,12 @@ export { getSessionSnapshotInput, getSessionSnapshotOutput, getSessionSnapshotTool, + cacheStatsInput, + cacheStatsOutput, + cacheStatsTool, + cacheFlushInput, + cacheFlushOutput, + cacheFlushTool, } from "./tools.ts"; // A2A schemas diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index 1557c0a136..46d03f325a 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -806,11 +806,73 @@ export const syncPricingTool: McpToolDefinition = { + name: "omniroute_cache_stats", + description: + "Returns cache statistics including semantic cache hit rate, prompt cache metrics by provider, and idempotency layer stats.", + inputSchema: cacheStatsInput, + outputSchema: cacheStatsOutput, + scopes: ["read:cache"], + auditLevel: "basic", + phase: 2, + sourceEndpoints: ["/api/cache"], +}; + +export const cacheFlushInput = z.object({ + signature: z.string().optional().describe("Specific cache signature to invalidate"), + model: z.string().optional().describe("Invalidate all entries for a specific model"), +}); + +export const cacheFlushOutput = z.object({ + ok: z.boolean(), + invalidated: z.number().optional(), + scope: z.string().optional(), +}); + +export const cacheFlushTool: McpToolDefinition = { + name: "omniroute_cache_flush", + description: + "Flush cache entries. Provide signature to invalidate a single entry, model to invalidate all entries for a model, or omit both to clear all.", + inputSchema: cacheFlushInput, + outputSchema: cacheFlushOutput, + scopes: ["write:cache"], + auditLevel: "full", + phase: 2, + sourceEndpoints: ["/api/cache"], +}; + // ============ Tool Registry ============ /** All MCP tool definitions, ordered by phase then name */ export const MCP_TOOLS = [ - // Phase 1: Essential getHealthTool, listCombosTool, getComboMetricsTool, @@ -819,7 +881,6 @@ export const MCP_TOOLS = [ routeRequestTool, costReportTool, listModelsCatalogTool, - // Phase 2: Advanced simulateRouteTool, setBudgetGuardTool, setRoutingStrategyTool, @@ -830,6 +891,8 @@ export const MCP_TOOLS = [ explainRouteTool, getSessionSnapshotTool, syncPricingTool, + cacheStatsTool, + cacheFlushTool, ] as const; /** Essential tools only (Phase 1) */ diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts index af501d7495..3f18e2cf6d 100644 --- a/open-sse/utils/cacheControlPolicy.ts +++ b/open-sse/utils/cacheControlPolicy.ts @@ -72,12 +72,7 @@ const DETERMINISTIC_STRATEGIES: Set = new Set(["priority", /** * Providers that support prompt caching */ -const CACHING_PROVIDERS = new Set([ - "claude", - "anthropic", - "zai", - "qwen", // Alibaba Qwen Coding Plan International -]); +const CACHING_PROVIDERS = new Set(["claude", "anthropic", "zai", "qwen", "deepseek"]); /** * Detect if the client is Claude Code or another caching-aware client diff --git a/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx b/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx new file mode 100644 index 0000000000..66792dcbfa --- /dev/null +++ b/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx @@ -0,0 +1,174 @@ +"use client"; + +import { useState, useEffect, useCallback } from "react"; +import { Button } from "@/shared/components"; +import { useTranslations } from "next-intl"; + +interface CacheEntry { + id: string; + signature: string; + model: string; + hit_count: number; + tokens_saved: number; + created_at: string; + expires_at: string; +} + +interface Pagination { + page: number; + limit: number; + total: number; + totalPages: number; +} + +export default function CacheEntriesTab() { + const t = useTranslations("cache"); + const [entries, setEntries] = useState([]); + const [pagination, setPagination] = useState({ + page: 1, + limit: 20, + total: 0, + totalPages: 0, + }); + const [loading, setLoading] = useState(true); + const [search, setSearch] = useState(""); + const [deleting, setDeleting] = useState(null); + + const fetchEntries = useCallback( + async (page = 1) => { + setLoading(true); + try { + const params = new URLSearchParams({ page: String(page), limit: String(pagination.limit) }); + if (search) params.set("search", search); + + const res = await fetch(`/api/cache/entries?${params}`); + if (res.ok) { + const data = await res.json(); + setEntries(data.entries); + setPagination(data.pagination); + } + } catch { + // ignore + } finally { + setLoading(false); + } + }, + [search, pagination.limit] + ); + + useEffect(() => { + fetchEntries(); + }, [fetchEntries]); + + const handleDelete = async (signature: string) => { + setDeleting(signature); + try { + await fetch(`/api/cache/entries?signature=${encodeURIComponent(signature)}`, { + method: "DELETE", + }); + await fetchEntries(pagination.page); + } finally { + setDeleting(null); + } + }; + + const formatDate = (dateStr: string) => { + return new Date(dateStr).toLocaleString(); + }; + + return ( +
+
+ setSearch(e.target.value)} + onKeyDown={(e) => e.key === "Enter" && fetchEntries()} + className="flex-1 px-3 py-2 text-sm rounded-lg border border-border bg-surface text-text-main placeholder:text-text-muted" + /> + +
+ + {loading ? ( +
{t("loading")}
+ ) : entries.length === 0 ? ( +
{t("noEntries")}
+ ) : ( + <> +
+ + + + + + + + + + + + + + {entries.map((entry) => ( + + + + + + + + + + ))} + +
{t("signature")}{t("model")}{t("hits")}{t("tokensSaved")}{t("created")}{t("expires")}{t("actions")}
+ {entry.signature.slice(0, 12)}... + {entry.model}{entry.hit_count} + {entry.tokens_saved.toLocaleString()} + + {formatDate(entry.created_at)} + + {formatDate(entry.expires_at)} + + +
+
+ + {/* Pagination */} + {pagination.totalPages > 1 && ( +
+ + + {pagination.page} / {pagination.totalPages} + + +
+ )} + + )} +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/cache/page.tsx b/src/app/(dashboard)/dashboard/cache/page.tsx index 34007450e2..24064e8403 100644 --- a/src/app/(dashboard)/dashboard/cache/page.tsx +++ b/src/app/(dashboard)/dashboard/cache/page.tsx @@ -4,6 +4,7 @@ import { useState, useEffect, useCallback } from "react"; import { Card, Button, EmptyState } from "@/shared/components"; import { useNotificationStore } from "@/store/notificationStore"; import { useTranslations } from "next-intl"; +import CacheEntriesTab from "./components/CacheEntriesTab"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -138,6 +139,7 @@ export default function CachePage() { const [stats, setStats] = useState(null); const [loading, setLoading] = useState(true); const [clearing, setClearing] = useState(false); + const [activeTab, setActiveTab] = useState<"overview" | "entries">("overview"); const notify = useNotificationStore(); const fetchStats = useCallback(async () => { @@ -226,296 +228,334 @@ export default function CachePage() { - {/* Loading skeleton */} - {loading && ( -
+ + +
- {/* Error / empty state */} - {!loading && !stats && ( - void fetchStats()} - /> - )} + {/* Entries tab */} + {activeTab === "entries" && } - {/* Main content */} - {!loading && stats && ( + {/* Overview tab content */} + {activeTab === "overview" && ( <> - {/* Stats grid */} -
- - - - -
- - {/* Hit rate + breakdown */} - -
-
-

{t("performance")}

- - {t("autoRefresh", { seconds: REFRESH_INTERVAL_SECONDS })} - -
- -
-
-
- {sc?.hits ?? 0} -
-
{t("hits")}
-
-
-
- {sc?.misses ?? 0} -
-
{t("misses")}
-
-
-
{totalRequests}
-
{t("total")}
-
-
+ {/* Loading skeleton */} + {loading && ( +
+ {Array.from({ length: 4 }).map((_, i) => ( +
+ ))}
- - - {/* Prompt Cache Stats */} - {pc && ( - -
-
- -

{t("promptCache")}

-
- -
-
-
- {pc.requestsWithCacheControl.toLocaleString()} -
-
{t("cachedRequests")}
-
-
-
- {promptCacheHitRate.toFixed(1)}% -
-
{t("cacheHitRate")}
-
-
-
- {pc.totalCachedTokens.toLocaleString()} -
-
{t("cachedTokens")}
-
-
-
- {pc.totalCacheCreationTokens.toLocaleString()} -
-
{t("cacheCreationTokens")}
-
-
- - {providerEntries.length > 0 && ( -
-

{t("byProvider")}

-
- - - - - - - - - - - - {providerEntries.map(([provider, data]) => ( - - - - - - - - ))} - -
{t("provider")}{t("requests")}{t("inputTokens")}{t("cachedTokensCol")}{t("cacheCreation")}
{provider} - {data.requests.toLocaleString()} - - {data.inputTokens.toLocaleString()} - - {data.cachedTokens.toLocaleString()} - - {data.cacheCreationTokens.toLocaleString()} -
-
-
- )} -
- )} - {/* Cache Trend (24h) */} - {trend.length > 0 && ( - -
-
- -

{t("trend24h")}

-
-
- {trend.map((point) => { - const height = Math.max(4, (point.requests / maxTrendRequests) * 100); - const cachedHeight = - point.requests > 0 - ? Math.max(2, (point.cachedRequests / point.requests) * height) - : 0; - const hour = new Date(point.timestamp).toLocaleTimeString([], { - hour: "2-digit", - minute: "2-digit", - hour12: false, - }); - return ( -
-
- {hour}: {point.requests} {t("requests").toLowerCase()},{" "} - {point.cachedRequests} {t("cached").toLowerCase()} -
-
-
-
-
- - {hour.split(":")[0]} - + {/* Error / empty state */} + {!loading && !stats && ( + void fetchStats()} + /> + )} + + {/* Main content */} + {!loading && stats && ( + <> + {/* Stats grid */} +
+ + + + +
+ + {/* Hit rate + breakdown */} + +
+
+

{t("performance")}

+ + {t("autoRefresh", { seconds: REFRESH_INTERVAL_SECONDS })} + +
+ +
+
+
+ {sc?.hits ?? 0}
- ); - })} -
-
-
-
- {t("total")} -
-
-
- {t("cached")} +
{t("hits")}
+
+
+
+ {sc?.misses ?? 0} +
+
{t("misses")}
+
+
+
{totalRequests}
+
{t("total")}
+
-
- + + + {/* Prompt Cache Stats */} + {pc && ( + +
+
+ +

{t("promptCache")}

+
+ +
+
+
+ {pc.requestsWithCacheControl.toLocaleString()} +
+
{t("cachedRequests")}
+
+
+
+ {promptCacheHitRate.toFixed(1)}% +
+
{t("cacheHitRate")}
+
+
+
+ {pc.totalCachedTokens.toLocaleString()} +
+
{t("cachedTokens")}
+
+
+
+ {pc.totalCacheCreationTokens.toLocaleString()} +
+
+ {t("cacheCreationTokens")} +
+
+
+ + {providerEntries.length > 0 && ( +
+

+ {t("byProvider")} +

+
+ + + + + + + + + + + + {providerEntries.map(([provider, data]) => ( + + + + + + + + ))} + +
{t("provider")}{t("requests")}{t("inputTokens")}{t("cachedTokensCol")}{t("cacheCreation")}
{provider} + {data.requests.toLocaleString()} + + {data.inputTokens.toLocaleString()} + + {data.cachedTokens.toLocaleString()} + + {data.cacheCreationTokens.toLocaleString()} +
+
+
+ )} +
+ + )} + + {/* Cache Trend (24h) */} + {trend.length > 0 && ( + +
+
+ +

{t("trend24h")}

+
+
+ {trend.map((point) => { + const height = Math.max(4, (point.requests / maxTrendRequests) * 100); + const cachedHeight = + point.requests > 0 + ? Math.max(2, (point.cachedRequests / point.requests) * height) + : 0; + const hour = new Date(point.timestamp).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + hour12: false, + }); + return ( +
+
+ {hour}: {point.requests} {t("requests").toLowerCase()},{" "} + {point.cachedRequests} {t("cached").toLowerCase()} +
+
+
+
+
+ + {hour.split(":")[0]} + +
+ ); + })} +
+
+
+
+ {t("total")} +
+
+
+ {t("cached")} +
+
+
+ + )} + + {/* Cache behavior */} + +
+

{t("behavior")}

+
+ {t("behaviorDeterministic")} + + {t.rich("behaviorBypass", { + header: () => ( + + X-OmniRoute-No-Cache: true + + ), + })} + + {t("behaviorTwoTier")} + + {t.rich("behaviorTtl", { + envVar: () => ( + + SEMANTIC_CACHE_TTL_MS + + ), + })} + +
+
+ + + {/* Idempotency */} + +
+
+ +

{t("idempotency")}

+
+
+
+
+ {idp?.activeKeys ?? 0} +
+
{t("activeDedupKeys")}
+
+
+
+ {idp ? `${(idp.windowMs / 1000).toFixed(0)}s` : "—"} +
+
{t("dedupWindow")}
+
+
+
+ + )} - - {/* Cache behavior */} - -
-

{t("behavior")}

-
- {t("behaviorDeterministic")} - - {t.rich("behaviorBypass", { - header: () => ( - - X-OmniRoute-No-Cache: true - - ), - })} - - {t("behaviorTwoTier")} - - {t.rich("behaviorTtl", { - envVar: () => ( - - SEMANTIC_CACHE_TTL_MS - - ), - })} - -
-
- - - {/* Idempotency */} - -
-
- -

{t("idempotency")}

-
-
-
-
{idp?.activeKeys ?? 0}
-
{t("activeDedupKeys")}
-
-
-
- {idp ? `${(idp.windowMs / 1000).toFixed(0)}s` : "—"} -
-
{t("dedupWindow")}
-
-
-
- )}
diff --git a/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx new file mode 100644 index 0000000000..633592a462 --- /dev/null +++ b/src/app/(dashboard)/dashboard/settings/components/CacheSettingsTab.tsx @@ -0,0 +1,191 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Card, Button } from "@/shared/components"; +import { useTranslations } from "next-intl"; + +interface CacheConfig { + semanticCacheEnabled: boolean; + semanticCacheMaxSize: number; + semanticCacheTTL: number; + promptCacheEnabled: boolean; + promptCacheStrategy: "auto" | "system-only" | "manual"; + alwaysPreserveClientCache: "auto" | "always" | "never"; +} + +export default function CacheSettingsTab() { + const t = useTranslations("settings"); + const [config, setConfig] = useState({ + semanticCacheEnabled: true, + semanticCacheMaxSize: 100, + semanticCacheTTL: 1800000, + promptCacheEnabled: true, + promptCacheStrategy: "auto", + alwaysPreserveClientCache: "auto", + }); + const [saving, setSaving] = useState(false); + const [loading, setLoading] = useState(true); + + useEffect(() => { + fetch("/api/settings/cache-config") + .then((r) => (r.ok ? r.json() : null)) + .then((data) => { + if (data) setConfig(data); + }) + .catch(() => {}) + .finally(() => setLoading(false)); + }, []); + + const handleSave = async () => { + setSaving(true); + try { + await fetch("/api/settings/cache-config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(config), + }); + } finally { + setSaving(false); + } + }; + + if (loading) { + return ( + +

{t("loading")}

+ + ); + } + + return ( + +

+ cached + {t("cacheSettings")} +

+ +
+ {/* Semantic Cache */} +
+

{t("semanticCache")}

+ + + + + + +
+ + {/* Prompt Cache */} +
+

{t("promptCache")}

+ + + + + + +
+ + {/* Save */} +
+ +
+
+ + ); +} diff --git a/src/app/(dashboard)/dashboard/settings/page.tsx b/src/app/(dashboard)/dashboard/settings/page.tsx index 7aa69f112c..9d0b261022 100644 --- a/src/app/(dashboard)/dashboard/settings/page.tsx +++ b/src/app/(dashboard)/dashboard/settings/page.tsx @@ -18,6 +18,7 @@ import ModelAliasesTab from "./components/ModelAliasesTab"; import BackgroundDegradationTab from "./components/BackgroundDegradationTab"; import CacheStatsCard from "./components/CacheStatsCard"; +import CacheSettingsTab from "./components/CacheSettingsTab"; import ResilienceTab from "./components/ResilienceTab"; const tabs = [ @@ -89,6 +90,7 @@ export default function SettingsPage() { +
)} diff --git a/src/app/api/cache/entries/route.ts b/src/app/api/cache/entries/route.ts new file mode 100644 index 0000000000..26f30b02a9 --- /dev/null +++ b/src/app/api/cache/entries/route.ts @@ -0,0 +1,95 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getDbInstance } from "@/lib/db/core"; + +interface CacheEntry { + id: string; + signature: string; + model: string; + hit_count: number; + tokens_saved: number; + created_at: string; + expires_at: string; +} + +export async function GET(req: NextRequest) { + try { + const { searchParams } = new URL(req.url); + const page = Math.max(1, parseInt(searchParams.get("page") || "1", 10)); + const limit = Math.min(100, Math.max(1, parseInt(searchParams.get("limit") || "20", 10))); + const search = searchParams.get("search") || ""; + const model = searchParams.get("model") || ""; + const sortBy = searchParams.get("sortBy") || "created_at"; + const sortOrder = searchParams.get("sortOrder") || "desc"; + + const db = getDbInstance(); + const offset = (page - 1) * limit; + + const conditions: string[] = []; + const params: unknown[] = []; + + if (search) { + conditions.push("(signature LIKE ? OR model LIKE ?)"); + params.push(`%${search}%`, `%${search}%`); + } + + if (model) { + conditions.push("model = ?"); + params.push(model); + } + + const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const validSortColumns = ["created_at", "expires_at", "hit_count", "tokens_saved", "model"]; + const orderBy = validSortColumns.includes(sortBy) ? sortBy : "created_at"; + const order = sortOrder === "asc" ? "ASC" : "DESC"; + + const countRow = db + .prepare(`SELECT COUNT(*) as total FROM semantic_cache ${whereClause}`) + .get(...params) as { total: number }; + + const entries = db + .prepare( + `SELECT id, signature, model, hit_count, tokens_saved, created_at, expires_at + FROM semantic_cache ${whereClause} + ORDER BY ${orderBy} ${order} + LIMIT ? OFFSET ?` + ) + .all(...params, limit, offset) as CacheEntry[]; + + return NextResponse.json({ + entries, + pagination: { + page, + limit, + total: countRow?.total || 0, + totalPages: Math.ceil((countRow?.total || 0) / limit), + }, + }); + } catch (error) { + return NextResponse.json({ error: String(error) }, { status: 500 }); + } +} + +export async function DELETE(req: NextRequest) { + try { + const { searchParams } = new URL(req.url); + const signature = searchParams.get("signature"); + const model = searchParams.get("model"); + + const db = getDbInstance(); + + if (signature) { + db.prepare("DELETE FROM semantic_cache WHERE signature = ?").run(signature); + return NextResponse.json({ ok: true, deleted: 1 }); + } + + if (model) { + const result = db.prepare("DELETE FROM semantic_cache WHERE model = ?").run(model); + return NextResponse.json({ ok: true, deleted: result.changes }); + } + + return NextResponse.json({ error: "Provide signature or model parameter" }, { status: 400 }); + } catch (error) { + return NextResponse.json({ error: String(error) }, { status: 500 }); + } +} diff --git a/src/app/api/settings/cache-config/route.ts b/src/app/api/settings/cache-config/route.ts new file mode 100644 index 0000000000..a3fecff41f --- /dev/null +++ b/src/app/api/settings/cache-config/route.ts @@ -0,0 +1,73 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getSettings, updateSettings } from "@/lib/localDb"; +import { isAuthenticated } from "@/shared/utils/apiAuth"; + +const CACHE_CONFIG_KEYS = [ + "semanticCacheEnabled", + "semanticCacheMaxSize", + "semanticCacheTTL", + "promptCacheEnabled", + "promptCacheStrategy", + "alwaysPreserveClientCache", +] as const; + +const DEFAULTS = { + semanticCacheEnabled: true, + semanticCacheMaxSize: 100, + semanticCacheTTL: 1800000, + promptCacheEnabled: true, + promptCacheStrategy: "auto", + alwaysPreserveClientCache: "auto", +}; + +export async function GET(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const settings = await getSettings(); + const config: Record = {}; + for (const key of CACHE_CONFIG_KEYS) { + config[key] = settings[key] ?? DEFAULTS[key]; + } + return NextResponse.json(config); + } catch (error) { + return NextResponse.json({ error: String(error) }, { status: 500 }); + } +} + +export async function PUT(request: NextRequest) { + if (!(await isAuthenticated(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + try { + const body = await request.json(); + const updates: Record = {}; + + if (typeof body.semanticCacheEnabled === "boolean") { + updates.semanticCacheEnabled = body.semanticCacheEnabled; + } + if (typeof body.semanticCacheMaxSize === "number" && body.semanticCacheMaxSize > 0) { + updates.semanticCacheMaxSize = body.semanticCacheMaxSize; + } + if (typeof body.semanticCacheTTL === "number" && body.semanticCacheTTL > 0) { + updates.semanticCacheTTL = body.semanticCacheTTL; + } + if (typeof body.promptCacheEnabled === "boolean") { + updates.promptCacheEnabled = body.promptCacheEnabled; + } + if (["auto", "system-only", "manual"].includes(body.promptCacheStrategy)) { + updates.promptCacheStrategy = body.promptCacheStrategy; + } + if (["auto", "always", "never"].includes(body.alwaysPreserveClientCache)) { + updates.alwaysPreserveClientCache = body.alwaysPreserveClientCache; + } + + await updateSettings(updates); + return NextResponse.json({ ok: true }); + } catch (error) { + return NextResponse.json({ error: String(error) }, { status: 500 }); + } +} diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index 745da07ac3..43463d7cba 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1712,6 +1712,17 @@ "cacheMisses": "Cache Misses", "hitRate": "Hit Rate", "cacheEntries": "Cache Entries", + "cacheSettings": "Cache Settings", + "semanticCache": "Semantic Cache", + "maxEntries": "Max Entries", + "ttlMinutes": "TTL (minutes)", + "promptCache": "Prompt Cache", + "strategy": "Strategy", + "preserveClientCache": "Preserve Client Cache", + "enabled": "Enabled", + "loading": "Loading...", + "saving": "Saving...", + "save": "Save", "circuitBreaker": "Circuit Breaker", "retryPolicy": "Retry Policy", "maxRetries": "Max Retries", @@ -2933,6 +2944,17 @@ "cachedTokensCol": "Cached", "cacheCreation": "Creation", "trend24h": "Cache Trend (24h)", - "cached": "Cached" + "cached": "Cached", + "overview": "Overview", + "entries": "Entries", + "searchEntries": "Search entries...", + "search": "Search", + "loading": "Loading...", + "noEntries": "No cache entries found", + "signature": "Signature", + "model": "Model", + "created": "Created", + "expires": "Expires", + "actions": "Actions" } } diff --git a/src/lib/promptCache/index.ts b/src/lib/promptCache/index.ts new file mode 100644 index 0000000000..f4ee1fd501 --- /dev/null +++ b/src/lib/promptCache/index.ts @@ -0,0 +1 @@ +export { analyzePrefix, shouldInjectCacheControl } from "./prefixAnalyzer"; diff --git a/src/lib/promptCache/prefixAnalyzer.ts b/src/lib/promptCache/prefixAnalyzer.ts new file mode 100644 index 0000000000..43f622c87f --- /dev/null +++ b/src/lib/promptCache/prefixAnalyzer.ts @@ -0,0 +1,77 @@ +import crypto from "crypto"; + +interface Message { + role: string; + content: string | unknown[]; +} + +interface PrefixAnalysis { + prefixEndIdx: number; + prefixHash: string; + prefixTokens: number; + prefixType: "system_only" | "system_and_tools" | "system_tools_history"; + confidence: number; +} + +function normalizeContent(content: string | unknown[]): string { + if (typeof content === "string") return content; + return JSON.stringify(content); +} + +function estimateTokens(text: string): number { + return Math.ceil(text.length / 4); +} + +export function analyzePrefix(messages: Message[]): PrefixAnalysis { + if (!Array.isArray(messages) || messages.length === 0) { + return { + prefixEndIdx: -1, + prefixHash: "", + prefixTokens: 0, + prefixType: "system_only", + confidence: 0, + }; + } + + let prefixEndIdx = -1; + let prefixType: PrefixAnalysis["prefixType"] = "system_only"; + let confidence = 0.5; + + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + const role = msg.role || "user"; + + if (role === "system") { + prefixEndIdx = i; + prefixType = "system_only"; + confidence = 0.9; + } else if (role === "tool" || (role === "assistant" && Array.isArray(msg.content))) { + prefixEndIdx = i; + prefixType = "system_and_tools"; + confidence = 0.8; + } else if (role === "assistant") { + prefixEndIdx = i; + prefixType = "system_tools_history"; + confidence = 0.7; + } else { + break; + } + } + + const prefixMessages = messages.slice(0, prefixEndIdx + 1); + const prefixText = prefixMessages.map((m) => normalizeContent(m.content)).join("\n"); + const prefixHash = crypto.createHash("sha256").update(prefixText).digest("hex"); + const prefixTokens = estimateTokens(prefixText); + + return { + prefixEndIdx, + prefixHash, + prefixTokens, + prefixType, + confidence, + }; +} + +export function shouldInjectCacheControl(analysis: PrefixAnalysis, minTokens = 1024): boolean { + return analysis.prefixTokens >= minTokens && analysis.confidence >= 0.7; +} diff --git a/src/lib/semanticCache.ts b/src/lib/semanticCache.ts index d20e4b560e..369d7a0d68 100644 --- a/src/lib/semanticCache.ts +++ b/src/lib/semanticCache.ts @@ -29,6 +29,45 @@ function toNumber(value: unknown, fallback = 0): number { return fallback; } +function ensureCacheMetricsTable() { + try { + const db = getDbInstance(); + db.prepare( + `CREATE TABLE IF NOT EXISTS cache_metrics ( + key TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + )` + ).run(); + db.prepare( + `INSERT OR IGNORE INTO cache_metrics (key, value) VALUES ('hits', 0), ('misses', 0), ('tokens_saved', 0)` + ).run(); + } catch { + // DB not available + } +} + +function incrementMetric(metric: "hits" | "misses" | "tokens_saved", amount = 1) { + try { + const db = getDbInstance(); + db.prepare( + `UPDATE cache_metrics SET value = value + ?, updated_at = datetime('now') WHERE key = ?` + ).run(amount, metric); + } catch { + // DB not available — fall back to in-memory + } +} + +function getMetricValue(metric: string): number { + try { + const db = getDbInstance(); + const row = db.prepare(`SELECT value FROM cache_metrics WHERE key = ?`).get(metric); + return row ? toNumber(asRecord(row).value, 0) : 0; + } catch { + return 0; + } +} + function getHeaderValue( headers: { get?: (name: string) => string | null } | Record | null | undefined, name: string @@ -51,7 +90,6 @@ function getHeaderValue( // ─── Singleton ───────────────── let memoryCache: LRUCache | null = null; -let stats = { hits: 0, misses: 0, tokensSaved: 0 }; function getMemoryCache() { if (!memoryCache) { @@ -60,6 +98,7 @@ function getMemoryCache() { maxBytes: parseInt(process.env.SEMANTIC_CACHE_MAX_BYTES || String(4 * 1024 * 1024), 10), defaultTTL: parseInt(process.env.SEMANTIC_CACHE_TTL_MS || "1800000", 10), }); + ensureCacheMetricsTable(); } return memoryCache; } @@ -108,8 +147,8 @@ export function getCachedResponse(signature) { // 1. Check memory cache const memResult = getMemoryCache().get(signature); if (memResult) { - stats.hits++; - stats.tokensSaved += memResult.tokensSaved || 0; + incrementMetric("hits"); + incrementMetric("tokens_saved", memResult.tokensSaved || 0); return memResult.response; } @@ -126,7 +165,7 @@ export function getCachedResponse(signature) { const record = asRecord(row); const responsePayload = typeof record.response === "string" ? record.response : null; if (!responsePayload) { - stats.misses++; + incrementMetric("misses"); return null; } const parsed = JSON.parse(responsePayload); @@ -141,15 +180,15 @@ export function getCachedResponse(signature) { signature ); - stats.hits++; - stats.tokensSaved += tokensSaved; + incrementMetric("hits"); + incrementMetric("tokens_saved", tokensSaved); return parsed; } } catch { // DB not available — fail open } - stats.misses++; + incrementMetric("misses"); return null; } @@ -280,6 +319,17 @@ export function stopAutoCleanup(): void { } } +export function cleanOldMetrics(retentionDays = 90): number { + try { + const db = getDbInstance(); + const cutoff = new Date(Date.now() - retentionDays * 86400000).toISOString(); + const result = db.prepare("DELETE FROM semantic_cache WHERE created_at < ?").run(cutoff); + return result.changes || 0; + } catch { + return 0; + } +} + /** * Clear all cache entries. */ @@ -288,17 +338,12 @@ export function clearCache() { try { const db = getDbInstance(); db.prepare("DELETE FROM semantic_cache").run(); + db.prepare("UPDATE cache_metrics SET value = 0").run(); } catch { // DB not available } - stats = { hits: 0, misses: 0, tokensSaved: 0 }; } -// ─── Stats ───────────────── - -/** - * Get cache statistics. - */ export function getCacheStats() { const memStats = getMemoryCache().getStats(); let dbSize = 0; @@ -312,14 +357,18 @@ export function getCacheStats() { // DB not available } - const total = stats.hits + stats.misses; + const hits = getMetricValue("hits"); + const misses = getMetricValue("misses"); + const tokensSaved = getMetricValue("tokens_saved"); + const total = hits + misses; + return { memoryEntries: memStats.size, dbEntries: dbSize, - hits: stats.hits, - misses: stats.misses, - hitRate: total > 0 ? ((stats.hits / total) * 100).toFixed(1) : "0.0", - tokensSaved: stats.tokensSaved, + hits, + misses, + hitRate: total > 0 ? ((hits / total) * 100).toFixed(1) : "0.0", + tokensSaved, }; }