feat(cache): persistent metrics, cache entry browser, settings UI, MCP tools, prefix analyzer

Implements remaining features from #813:

Phase 1 - Persistent Metrics:
- Add cache_metrics table for persistent hit/miss tracking
- Semantic cache stats now survive server restarts

Phase 2 - Cache Entry Browser:
- /api/cache/entries endpoint with search, pagination, delete
- CacheEntriesTab component for browsing cached entries

Phase 3 - Settings UI:
- CacheSettingsTab for semantic/prompt cache configuration
- /api/settings/cache-config endpoint

Phase 4 - Prefix Analyzer:
- src/lib/promptCache/prefixAnalyzer.ts for intelligent caching
- Analyzes message arrays to find stable prefixes

Phase 5 - Provider Support:
- Added deepseek to CACHING_PROVIDERS

Phase 6 - MCP Tools:
- omniroute_cache_stats tool
- omniroute_cache_flush tool

Phase 7 - Retention:
- cleanOldMetrics() for auto-purge of old entries

Closes #813
This commit is contained in:
oyi77
2026-03-31 02:41:30 +07:00
committed by Diego Rodrigues de Sa e Souza
parent d043e7a242
commit 6780485051
13 changed files with 1094 additions and 306 deletions

View File

@@ -60,6 +60,12 @@ export {
getSessionSnapshotInput,
getSessionSnapshotOutput,
getSessionSnapshotTool,
cacheStatsInput,
cacheStatsOutput,
cacheStatsTool,
cacheFlushInput,
cacheFlushOutput,
cacheFlushTool,
} from "./tools.ts";
// A2A schemas

View File

@@ -806,11 +806,73 @@ export const syncPricingTool: McpToolDefinition<typeof syncPricingInput, typeof
sourceEndpoints: ["/api/pricing/sync"],
};
// ============ Cache Tools ============
export const cacheStatsInput = z.object({}).describe("No parameters required");
export const cacheStatsOutput = z.object({
semanticCache: z.object({
memoryEntries: z.number(),
dbEntries: z.number(),
hits: z.number(),
misses: z.number(),
hitRate: z.string(),
tokensSaved: z.number(),
}),
promptCache: z
.object({
totalRequests: z.number(),
requestsWithCacheControl: z.number(),
totalCachedTokens: z.number(),
totalCacheCreationTokens: z.number(),
estimatedCostSaved: z.number(),
})
.nullable(),
idempotency: z.object({
activeKeys: z.number(),
windowMs: z.number(),
}),
});
export const cacheStatsTool: McpToolDefinition<typeof cacheStatsInput, typeof cacheStatsOutput> = {
name: "omniroute_cache_stats",
description:
"Returns cache statistics including semantic cache hit rate, prompt cache metrics by provider, and idempotency layer stats.",
inputSchema: cacheStatsInput,
outputSchema: cacheStatsOutput,
scopes: ["read:cache"],
auditLevel: "basic",
phase: 2,
sourceEndpoints: ["/api/cache"],
};
export const cacheFlushInput = z.object({
signature: z.string().optional().describe("Specific cache signature to invalidate"),
model: z.string().optional().describe("Invalidate all entries for a specific model"),
});
export const cacheFlushOutput = z.object({
ok: z.boolean(),
invalidated: z.number().optional(),
scope: z.string().optional(),
});
export const cacheFlushTool: McpToolDefinition<typeof cacheFlushInput, typeof cacheFlushOutput> = {
name: "omniroute_cache_flush",
description:
"Flush cache entries. Provide signature to invalidate a single entry, model to invalidate all entries for a model, or omit both to clear all.",
inputSchema: cacheFlushInput,
outputSchema: cacheFlushOutput,
scopes: ["write:cache"],
auditLevel: "full",
phase: 2,
sourceEndpoints: ["/api/cache"],
};
// ============ Tool Registry ============
/** All MCP tool definitions, ordered by phase then name */
export const MCP_TOOLS = [
// Phase 1: Essential
getHealthTool,
listCombosTool,
getComboMetricsTool,
@@ -819,7 +881,6 @@ export const MCP_TOOLS = [
routeRequestTool,
costReportTool,
listModelsCatalogTool,
// Phase 2: Advanced
simulateRouteTool,
setBudgetGuardTool,
setRoutingStrategyTool,
@@ -830,6 +891,8 @@ export const MCP_TOOLS = [
explainRouteTool,
getSessionSnapshotTool,
syncPricingTool,
cacheStatsTool,
cacheFlushTool,
] as const;
/** Essential tools only (Phase 1) */

View File

@@ -72,12 +72,7 @@ const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set(["priority",
/**
* Providers that support prompt caching
*/
const CACHING_PROVIDERS = new Set([
"claude",
"anthropic",
"zai",
"qwen", // Alibaba Qwen Coding Plan International
]);
const CACHING_PROVIDERS = new Set(["claude", "anthropic", "zai", "qwen", "deepseek"]);
/**
* Detect if the client is Claude Code or another caching-aware client

View File

@@ -0,0 +1,174 @@
"use client";
import { useState, useEffect, useCallback } from "react";
import { Button } from "@/shared/components";
import { useTranslations } from "next-intl";
interface CacheEntry {
id: string;
signature: string;
model: string;
hit_count: number;
tokens_saved: number;
created_at: string;
expires_at: string;
}
interface Pagination {
page: number;
limit: number;
total: number;
totalPages: number;
}
export default function CacheEntriesTab() {
const t = useTranslations("cache");
const [entries, setEntries] = useState<CacheEntry[]>([]);
const [pagination, setPagination] = useState<Pagination>({
page: 1,
limit: 20,
total: 0,
totalPages: 0,
});
const [loading, setLoading] = useState(true);
const [search, setSearch] = useState("");
const [deleting, setDeleting] = useState<string | null>(null);
const fetchEntries = useCallback(
async (page = 1) => {
setLoading(true);
try {
const params = new URLSearchParams({ page: String(page), limit: String(pagination.limit) });
if (search) params.set("search", search);
const res = await fetch(`/api/cache/entries?${params}`);
if (res.ok) {
const data = await res.json();
setEntries(data.entries);
setPagination(data.pagination);
}
} catch {
// ignore
} finally {
setLoading(false);
}
},
[search, pagination.limit]
);
useEffect(() => {
fetchEntries();
}, [fetchEntries]);
const handleDelete = async (signature: string) => {
setDeleting(signature);
try {
await fetch(`/api/cache/entries?signature=${encodeURIComponent(signature)}`, {
method: "DELETE",
});
await fetchEntries(pagination.page);
} finally {
setDeleting(null);
}
};
const formatDate = (dateStr: string) => {
return new Date(dateStr).toLocaleString();
};
return (
<div className="flex flex-col gap-4">
<div className="flex items-center gap-3">
<input
type="text"
placeholder={t("searchEntries")}
value={search}
onChange={(e) => setSearch(e.target.value)}
onKeyDown={(e) => e.key === "Enter" && fetchEntries()}
className="flex-1 px-3 py-2 text-sm rounded-lg border border-border bg-surface text-text-main placeholder:text-text-muted"
/>
<Button variant="secondary" size="sm" onClick={() => fetchEntries()}>
{t("search")}
</Button>
</div>
{loading ? (
<div className="text-sm text-text-muted">{t("loading")}</div>
) : entries.length === 0 ? (
<div className="text-sm text-text-muted text-center py-8">{t("noEntries")}</div>
) : (
<>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-text-muted border-b border-border/30">
<th className="pb-2 pr-4">{t("signature")}</th>
<th className="pb-2 pr-4">{t("model")}</th>
<th className="pb-2 pr-4">{t("hits")}</th>
<th className="pb-2 pr-4">{t("tokensSaved")}</th>
<th className="pb-2 pr-4">{t("created")}</th>
<th className="pb-2 pr-4">{t("expires")}</th>
<th className="pb-2">{t("actions")}</th>
</tr>
</thead>
<tbody>
{entries.map((entry) => (
<tr key={entry.id} className="border-b border-border/20">
<td className="py-2 pr-4 font-mono text-xs">
{entry.signature.slice(0, 12)}...
</td>
<td className="py-2 pr-4">{entry.model}</td>
<td className="py-2 pr-4 tabular-nums">{entry.hit_count}</td>
<td className="py-2 pr-4 tabular-nums text-green-500">
{entry.tokens_saved.toLocaleString()}
</td>
<td className="py-2 pr-4 text-xs text-text-muted">
{formatDate(entry.created_at)}
</td>
<td className="py-2 pr-4 text-xs text-text-muted">
{formatDate(entry.expires_at)}
</td>
<td className="py-2">
<button
onClick={() => handleDelete(entry.signature)}
disabled={deleting === entry.signature}
className="text-xs text-red-400 hover:text-red-300 disabled:opacity-50"
>
{deleting === entry.signature ? "..." : "🗑️"}
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
{/* Pagination */}
{pagination.totalPages > 1 && (
<div className="flex items-center justify-center gap-2 pt-2">
<Button
variant="secondary"
size="sm"
onClick={() => fetchEntries(pagination.page - 1)}
disabled={pagination.page <= 1}
>
</Button>
<span className="text-sm text-text-muted">
{pagination.page} / {pagination.totalPages}
</span>
<Button
variant="secondary"
size="sm"
onClick={() => fetchEntries(pagination.page + 1)}
disabled={pagination.page >= pagination.totalPages}
>
</Button>
</div>
)}
</>
)}
</div>
);
}

View File

@@ -4,6 +4,7 @@ import { useState, useEffect, useCallback } from "react";
import { Card, Button, EmptyState } from "@/shared/components";
import { useNotificationStore } from "@/store/notificationStore";
import { useTranslations } from "next-intl";
import CacheEntriesTab from "./components/CacheEntriesTab";
// ─── Types ───────────────────────────────────────────────────────────────────
@@ -138,6 +139,7 @@ export default function CachePage() {
const [stats, setStats] = useState<CacheStats | null>(null);
const [loading, setLoading] = useState(true);
const [clearing, setClearing] = useState(false);
const [activeTab, setActiveTab] = useState<"overview" | "entries">("overview");
const notify = useNotificationStore();
const fetchStats = useCallback(async () => {
@@ -226,296 +228,334 @@ export default function CachePage() {
</div>
</div>
{/* Loading skeleton */}
{loading && (
<div
className="grid grid-cols-2 md:grid-cols-4 gap-4"
aria-busy="true"
aria-label="Loading cache statistics"
{/* Tab navigation */}
<div className="flex gap-1 p-1 rounded-lg bg-black/5 dark:bg-white/5 w-fit">
<button
onClick={() => setActiveTab("overview")}
className={`px-4 py-2 rounded-md text-sm font-medium transition-all ${
activeTab === "overview"
? "bg-white dark:bg-white/10 text-text-main shadow-sm"
: "text-text-muted hover:text-text-main"
}`}
>
{Array.from({ length: 4 }).map((_, i) => (
<div key={i} className="h-24 rounded-xl bg-surface-raised animate-pulse" />
))}
</div>
)}
{t("overview")}
</button>
<button
onClick={() => setActiveTab("entries")}
className={`px-4 py-2 rounded-md text-sm font-medium transition-all ${
activeTab === "entries"
? "bg-white dark:bg-white/10 text-text-main shadow-sm"
: "text-text-muted hover:text-text-main"
}`}
>
{t("entries")}
</button>
</div>
{/* Error / empty state */}
{!loading && !stats && (
<EmptyState
icon="cached"
title={t("unavailable")}
description={t("unavailableDesc")}
actionLabel={t("refresh")}
onAction={() => void fetchStats()}
/>
)}
{/* Entries tab */}
{activeTab === "entries" && <CacheEntriesTab />}
{/* Main content */}
{!loading && stats && (
{/* Overview tab content */}
{activeTab === "overview" && (
<>
{/* Stats grid */}
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
<StatCard
icon="memory"
label={t("memoryEntries")}
value={sc?.memoryEntries ?? 0}
sub={t("memoryEntriesSub")}
/>
<StatCard
icon="storage"
label={t("dbEntries")}
value={sc?.dbEntries ?? 0}
sub={t("dbEntriesSub")}
/>
<StatCard
icon="trending_up"
label={t("cacheHits")}
value={sc?.hits ?? 0}
sub={t("cacheHitsSub", { total: totalRequests })}
valueClass="text-green-500"
/>
<StatCard
icon="token"
label={t("tokensSaved")}
value={(sc?.tokensSaved ?? 0).toLocaleString()}
sub={t("tokensSavedSub")}
valueClass="text-blue-400"
/>
</div>
{/* Hit rate + breakdown */}
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center justify-between">
<h2 className="font-medium text-sm">{t("performance")}</h2>
<span className="text-xs text-text-muted">
{t("autoRefresh", { seconds: REFRESH_INTERVAL_SECONDS })}
</span>
</div>
<HitRateBar hitRate={hitRate} label={t("hitRate")} />
<div className="grid grid-cols-3 gap-4 pt-3 border-t border-border/30 text-center">
<div>
<div className="text-lg font-semibold tabular-nums text-green-500">
{sc?.hits ?? 0}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("hits")}</div>
</div>
<div>
<div className="text-lg font-semibold tabular-nums text-red-400">
{sc?.misses ?? 0}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("misses")}</div>
</div>
<div>
<div className="text-lg font-semibold tabular-nums">{totalRequests}</div>
<div className="text-xs text-text-muted mt-0.5">{t("total")}</div>
</div>
</div>
{/* Loading skeleton */}
{loading && (
<div
className="grid grid-cols-2 md:grid-cols-4 gap-4"
aria-busy="true"
aria-label="Loading cache statistics"
>
{Array.from({ length: 4 }).map((_, i) => (
<div key={i} className="h-24 rounded-xl bg-surface-raised animate-pulse" />
))}
</div>
</Card>
{/* Prompt Cache Stats */}
{pc && (
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
bolt
</span>
<h2 className="font-medium text-sm">{t("promptCache")}</h2>
</div>
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">
{pc.requestsWithCacheControl.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cachedRequests")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-green-500">
{promptCacheHitRate.toFixed(1)}%
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cacheHitRate")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-blue-400">
{pc.totalCachedTokens.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cachedTokens")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-purple-400">
{pc.totalCacheCreationTokens.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cacheCreationTokens")}</div>
</div>
</div>
{providerEntries.length > 0 && (
<div className="pt-3 border-t border-border/30">
<h3 className="text-xs font-medium text-text-muted mb-3">{t("byProvider")}</h3>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-text-muted border-b border-border/30">
<th className="pb-2 pr-4">{t("provider")}</th>
<th className="pb-2 pr-4">{t("requests")}</th>
<th className="pb-2 pr-4">{t("inputTokens")}</th>
<th className="pb-2 pr-4">{t("cachedTokensCol")}</th>
<th className="pb-2">{t("cacheCreation")}</th>
</tr>
</thead>
<tbody>
{providerEntries.map(([provider, data]) => (
<tr key={provider} className="border-b border-border/20">
<td className="py-2 pr-4 font-medium">{provider}</td>
<td className="py-2 pr-4 tabular-nums">
{data.requests.toLocaleString()}
</td>
<td className="py-2 pr-4 tabular-nums">
{data.inputTokens.toLocaleString()}
</td>
<td className="py-2 pr-4 tabular-nums text-green-500">
{data.cachedTokens.toLocaleString()}
</td>
<td className="py-2 tabular-nums text-purple-400">
{data.cacheCreationTokens.toLocaleString()}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
</Card>
)}
{/* Cache Trend (24h) */}
{trend.length > 0 && (
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
timeline
</span>
<h2 className="font-medium text-sm">{t("trend24h")}</h2>
</div>
<div className="flex items-end gap-1 h-32">
{trend.map((point) => {
const height = Math.max(4, (point.requests / maxTrendRequests) * 100);
const cachedHeight =
point.requests > 0
? Math.max(2, (point.cachedRequests / point.requests) * height)
: 0;
const hour = new Date(point.timestamp).toLocaleTimeString([], {
hour: "2-digit",
minute: "2-digit",
hour12: false,
});
return (
<div
key={point.timestamp}
className="flex-1 flex flex-col items-center gap-1 group relative"
>
<div className="absolute bottom-full mb-1 hidden group-hover:block bg-surface-raised border border-border rounded px-2 py-1 text-xs whitespace-nowrap z-10">
{hour}: {point.requests} {t("requests").toLowerCase()},{" "}
{point.cachedRequests} {t("cached").toLowerCase()}
</div>
<div className="w-full flex flex-col justify-end h-full gap-px">
<div
className="w-full bg-green-500/30 rounded-t"
style={{ height: `${cachedHeight}%` }}
/>
<div
className="w-full bg-text-muted/20 rounded-t"
style={{ height: `${height - cachedHeight}%` }}
/>
</div>
<span className="text-[10px] text-text-muted truncate w-full text-center">
{hour.split(":")[0]}
</span>
{/* Error / empty state */}
{!loading && !stats && (
<EmptyState
icon="cached"
title={t("unavailable")}
description={t("unavailableDesc")}
actionLabel={t("refresh")}
onAction={() => void fetchStats()}
/>
)}
{/* Main content */}
{!loading && stats && (
<>
{/* Stats grid */}
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
<StatCard
icon="memory"
label={t("memoryEntries")}
value={sc?.memoryEntries ?? 0}
sub={t("memoryEntriesSub")}
/>
<StatCard
icon="storage"
label={t("dbEntries")}
value={sc?.dbEntries ?? 0}
sub={t("dbEntriesSub")}
/>
<StatCard
icon="trending_up"
label={t("cacheHits")}
value={sc?.hits ?? 0}
sub={t("cacheHitsSub", { total: totalRequests })}
valueClass="text-green-500"
/>
<StatCard
icon="token"
label={t("tokensSaved")}
value={(sc?.tokensSaved ?? 0).toLocaleString()}
sub={t("tokensSavedSub")}
valueClass="text-blue-400"
/>
</div>
{/* Hit rate + breakdown */}
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center justify-between">
<h2 className="font-medium text-sm">{t("performance")}</h2>
<span className="text-xs text-text-muted">
{t("autoRefresh", { seconds: REFRESH_INTERVAL_SECONDS })}
</span>
</div>
<HitRateBar hitRate={hitRate} label={t("hitRate")} />
<div className="grid grid-cols-3 gap-4 pt-3 border-t border-border/30 text-center">
<div>
<div className="text-lg font-semibold tabular-nums text-green-500">
{sc?.hits ?? 0}
</div>
);
})}
</div>
<div className="flex items-center gap-4 text-xs text-text-muted">
<div className="flex items-center gap-1.5">
<div className="w-3 h-3 rounded bg-text-muted/20" />
<span>{t("total")}</span>
</div>
<div className="flex items-center gap-1.5">
<div className="w-3 h-3 rounded bg-green-500/30" />
<span>{t("cached")}</span>
<div className="text-xs text-text-muted mt-0.5">{t("hits")}</div>
</div>
<div>
<div className="text-lg font-semibold tabular-nums text-red-400">
{sc?.misses ?? 0}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("misses")}</div>
</div>
<div>
<div className="text-lg font-semibold tabular-nums">{totalRequests}</div>
<div className="text-xs text-text-muted mt-0.5">{t("total")}</div>
</div>
</div>
</div>
</div>
</Card>
</Card>
{/* Prompt Cache Stats */}
{pc && (
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
bolt
</span>
<h2 className="font-medium text-sm">{t("promptCache")}</h2>
</div>
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">
{pc.requestsWithCacheControl.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cachedRequests")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-green-500">
{promptCacheHitRate.toFixed(1)}%
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cacheHitRate")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-blue-400">
{pc.totalCachedTokens.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("cachedTokens")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums text-purple-400">
{pc.totalCacheCreationTokens.toLocaleString()}
</div>
<div className="text-xs text-text-muted mt-0.5">
{t("cacheCreationTokens")}
</div>
</div>
</div>
{providerEntries.length > 0 && (
<div className="pt-3 border-t border-border/30">
<h3 className="text-xs font-medium text-text-muted mb-3">
{t("byProvider")}
</h3>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead>
<tr className="text-left text-xs text-text-muted border-b border-border/30">
<th className="pb-2 pr-4">{t("provider")}</th>
<th className="pb-2 pr-4">{t("requests")}</th>
<th className="pb-2 pr-4">{t("inputTokens")}</th>
<th className="pb-2 pr-4">{t("cachedTokensCol")}</th>
<th className="pb-2">{t("cacheCreation")}</th>
</tr>
</thead>
<tbody>
{providerEntries.map(([provider, data]) => (
<tr key={provider} className="border-b border-border/20">
<td className="py-2 pr-4 font-medium">{provider}</td>
<td className="py-2 pr-4 tabular-nums">
{data.requests.toLocaleString()}
</td>
<td className="py-2 pr-4 tabular-nums">
{data.inputTokens.toLocaleString()}
</td>
<td className="py-2 pr-4 tabular-nums text-green-500">
{data.cachedTokens.toLocaleString()}
</td>
<td className="py-2 tabular-nums text-purple-400">
{data.cacheCreationTokens.toLocaleString()}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
)}
</div>
</Card>
)}
{/* Cache Trend (24h) */}
{trend.length > 0 && (
<Card>
<div className="p-5 flex flex-col gap-4">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
timeline
</span>
<h2 className="font-medium text-sm">{t("trend24h")}</h2>
</div>
<div className="flex items-end gap-1 h-32">
{trend.map((point) => {
const height = Math.max(4, (point.requests / maxTrendRequests) * 100);
const cachedHeight =
point.requests > 0
? Math.max(2, (point.cachedRequests / point.requests) * height)
: 0;
const hour = new Date(point.timestamp).toLocaleTimeString([], {
hour: "2-digit",
minute: "2-digit",
hour12: false,
});
return (
<div
key={point.timestamp}
className="flex-1 flex flex-col items-center gap-1 group relative"
>
<div className="absolute bottom-full mb-1 hidden group-hover:block bg-surface-raised border border-border rounded px-2 py-1 text-xs whitespace-nowrap z-10">
{hour}: {point.requests} {t("requests").toLowerCase()},{" "}
{point.cachedRequests} {t("cached").toLowerCase()}
</div>
<div className="w-full flex flex-col justify-end h-full gap-px">
<div
className="w-full bg-green-500/30 rounded-t"
style={{ height: `${cachedHeight}%` }}
/>
<div
className="w-full bg-text-muted/20 rounded-t"
style={{ height: `${height - cachedHeight}%` }}
/>
</div>
<span className="text-[10px] text-text-muted truncate w-full text-center">
{hour.split(":")[0]}
</span>
</div>
);
})}
</div>
<div className="flex items-center gap-4 text-xs text-text-muted">
<div className="flex items-center gap-1.5">
<div className="w-3 h-3 rounded bg-text-muted/20" />
<span>{t("total")}</span>
</div>
<div className="flex items-center gap-1.5">
<div className="w-3 h-3 rounded bg-green-500/30" />
<span>{t("cached")}</span>
</div>
</div>
</div>
</Card>
)}
{/* Cache behavior */}
<Card>
<div className="p-5 flex flex-col gap-3">
<h2 className="font-medium text-sm">{t("behavior")}</h2>
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
<InfoRow icon="info">{t("behaviorDeterministic")}</InfoRow>
<InfoRow icon="info">
{t.rich("behaviorBypass", {
header: () => (
<code className="bg-surface px-1 py-0.5 rounded text-xs font-mono">
X-OmniRoute-No-Cache: true
</code>
),
})}
</InfoRow>
<InfoRow icon="info">{t("behaviorTwoTier")}</InfoRow>
<InfoRow icon="info">
{t.rich("behaviorTtl", {
envVar: () => (
<code className="bg-surface px-1 py-0.5 rounded text-xs font-mono">
SEMANTIC_CACHE_TTL_MS
</code>
),
})}
</InfoRow>
</div>
</div>
</Card>
{/* Idempotency */}
<Card>
<div className="p-5 flex flex-col gap-3">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
fingerprint
</span>
<h2 className="font-medium text-sm">{t("idempotency")}</h2>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">
{idp?.activeKeys ?? 0}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("activeDedupKeys")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">
{idp ? `${(idp.windowMs / 1000).toFixed(0)}s` : "—"}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("dedupWindow")}</div>
</div>
</div>
</div>
</Card>
</>
)}
{/* Cache behavior */}
<Card>
<div className="p-5 flex flex-col gap-3">
<h2 className="font-medium text-sm">{t("behavior")}</h2>
<div className="grid grid-cols-1 md:grid-cols-2 gap-3">
<InfoRow icon="info">{t("behaviorDeterministic")}</InfoRow>
<InfoRow icon="info">
{t.rich("behaviorBypass", {
header: () => (
<code className="bg-surface px-1 py-0.5 rounded text-xs font-mono">
X-OmniRoute-No-Cache: true
</code>
),
})}
</InfoRow>
<InfoRow icon="info">{t("behaviorTwoTier")}</InfoRow>
<InfoRow icon="info">
{t.rich("behaviorTtl", {
envVar: () => (
<code className="bg-surface px-1 py-0.5 rounded text-xs font-mono">
SEMANTIC_CACHE_TTL_MS
</code>
),
})}
</InfoRow>
</div>
</div>
</Card>
{/* Idempotency */}
<Card>
<div className="p-5 flex flex-col gap-3">
<div className="flex items-center gap-2">
<span
className="material-symbols-outlined text-base text-text-muted"
aria-hidden="true"
>
fingerprint
</span>
<h2 className="font-medium text-sm">{t("idempotency")}</h2>
</div>
<div className="grid grid-cols-2 gap-4">
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">{idp?.activeKeys ?? 0}</div>
<div className="text-xs text-text-muted mt-0.5">{t("activeDedupKeys")}</div>
</div>
<div className="p-3 rounded-lg bg-surface/50">
<div className="text-lg font-semibold tabular-nums">
{idp ? `${(idp.windowMs / 1000).toFixed(0)}s` : "—"}
</div>
<div className="text-xs text-text-muted mt-0.5">{t("dedupWindow")}</div>
</div>
</div>
</div>
</Card>
</>
)}
</div>

View File

@@ -0,0 +1,191 @@
"use client";
import { useState, useEffect } from "react";
import { Card, Button } from "@/shared/components";
import { useTranslations } from "next-intl";
interface CacheConfig {
semanticCacheEnabled: boolean;
semanticCacheMaxSize: number;
semanticCacheTTL: number;
promptCacheEnabled: boolean;
promptCacheStrategy: "auto" | "system-only" | "manual";
alwaysPreserveClientCache: "auto" | "always" | "never";
}
export default function CacheSettingsTab() {
const t = useTranslations("settings");
const [config, setConfig] = useState<CacheConfig>({
semanticCacheEnabled: true,
semanticCacheMaxSize: 100,
semanticCacheTTL: 1800000,
promptCacheEnabled: true,
promptCacheStrategy: "auto",
alwaysPreserveClientCache: "auto",
});
const [saving, setSaving] = useState(false);
const [loading, setLoading] = useState(true);
useEffect(() => {
fetch("/api/settings/cache-config")
.then((r) => (r.ok ? r.json() : null))
.then((data) => {
if (data) setConfig(data);
})
.catch(() => {})
.finally(() => setLoading(false));
}, []);
const handleSave = async () => {
setSaving(true);
try {
await fetch("/api/settings/cache-config", {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(config),
});
} finally {
setSaving(false);
}
};
if (loading) {
return (
<Card className="p-6">
<p className="text-sm text-text-muted">{t("loading")}</p>
</Card>
);
}
return (
<Card className="p-6">
<h3 className="text-lg font-semibold text-text-main flex items-center gap-2 mb-4">
<span className="material-symbols-outlined text-[20px]">cached</span>
{t("cacheSettings")}
</h3>
<div className="space-y-6">
{/* Semantic Cache */}
<div className="space-y-3">
<h4 className="text-sm font-medium text-text-main">{t("semanticCache")}</h4>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("enabled")}</span>
<button
onClick={() =>
setConfig((c) => ({ ...c, semanticCacheEnabled: !c.semanticCacheEnabled }))
}
className={`relative w-10 h-5 rounded-full transition-colors ${
config.semanticCacheEnabled ? "bg-green-500" : "bg-border"
}`}
>
<span
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
config.semanticCacheEnabled ? "left-5" : "left-0.5"
}`}
/>
</button>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("maxEntries")}</span>
<input
type="number"
min={1}
max={1000}
value={config.semanticCacheMaxSize}
onChange={(e) =>
setConfig((c) => ({ ...c, semanticCacheMaxSize: parseInt(e.target.value) || 100 }))
}
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
/>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("ttlMinutes")}</span>
<input
type="number"
min={1}
max={1440}
value={Math.round(config.semanticCacheTTL / 60000)}
onChange={(e) =>
setConfig((c) => ({
...c,
semanticCacheTTL: (parseInt(e.target.value) || 30) * 60000,
}))
}
className="w-24 px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
/>
</label>
</div>
{/* Prompt Cache */}
<div className="space-y-3 pt-4 border-t border-border/30">
<h4 className="text-sm font-medium text-text-main">{t("promptCache")}</h4>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("enabled")}</span>
<button
onClick={() =>
setConfig((c) => ({ ...c, promptCacheEnabled: !c.promptCacheEnabled }))
}
className={`relative w-10 h-5 rounded-full transition-colors ${
config.promptCacheEnabled ? "bg-green-500" : "bg-border"
}`}
>
<span
className={`absolute top-0.5 w-4 h-4 rounded-full bg-white transition-transform ${
config.promptCacheEnabled ? "left-5" : "left-0.5"
}`}
/>
</button>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("strategy")}</span>
<select
value={config.promptCacheStrategy}
onChange={(e) =>
setConfig((c) => ({
...c,
promptCacheStrategy: e.target.value as CacheConfig["promptCacheStrategy"],
}))
}
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
>
<option value="auto">Auto</option>
<option value="system-only">System Only</option>
<option value="manual">Manual</option>
</select>
</label>
<label className="flex items-center justify-between">
<span className="text-sm text-text-muted">{t("preserveClientCache")}</span>
<select
value={config.alwaysPreserveClientCache}
onChange={(e) =>
setConfig((c) => ({
...c,
alwaysPreserveClientCache: e.target
.value as CacheConfig["alwaysPreserveClientCache"],
}))
}
className="px-2 py-1 text-sm rounded border border-border bg-surface text-text-main"
>
<option value="auto">Auto</option>
<option value="always">Always</option>
<option value="never">Never</option>
</select>
</label>
</div>
{/* Save */}
<div className="pt-4 border-t border-border/30">
<Button onClick={handleSave} disabled={saving} size="sm">
{saving ? t("saving") : t("save")}
</Button>
</div>
</div>
</Card>
);
}

View File

@@ -18,6 +18,7 @@ import ModelAliasesTab from "./components/ModelAliasesTab";
import BackgroundDegradationTab from "./components/BackgroundDegradationTab";
import CacheStatsCard from "./components/CacheStatsCard";
import CacheSettingsTab from "./components/CacheSettingsTab";
import ResilienceTab from "./components/ResilienceTab";
const tabs = [
@@ -89,6 +90,7 @@ export default function SettingsPage() {
<CodexServiceTierTab />
<SystemPromptTab />
<CacheStatsCard />
<CacheSettingsTab />
</div>
)}

95
src/app/api/cache/entries/route.ts vendored Normal file
View File

@@ -0,0 +1,95 @@
import { NextRequest, NextResponse } from "next/server";
import { getDbInstance } from "@/lib/db/core";
interface CacheEntry {
id: string;
signature: string;
model: string;
hit_count: number;
tokens_saved: number;
created_at: string;
expires_at: string;
}
export async function GET(req: NextRequest) {
try {
const { searchParams } = new URL(req.url);
const page = Math.max(1, parseInt(searchParams.get("page") || "1", 10));
const limit = Math.min(100, Math.max(1, parseInt(searchParams.get("limit") || "20", 10)));
const search = searchParams.get("search") || "";
const model = searchParams.get("model") || "";
const sortBy = searchParams.get("sortBy") || "created_at";
const sortOrder = searchParams.get("sortOrder") || "desc";
const db = getDbInstance();
const offset = (page - 1) * limit;
const conditions: string[] = [];
const params: unknown[] = [];
if (search) {
conditions.push("(signature LIKE ? OR model LIKE ?)");
params.push(`%${search}%`, `%${search}%`);
}
if (model) {
conditions.push("model = ?");
params.push(model);
}
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
const validSortColumns = ["created_at", "expires_at", "hit_count", "tokens_saved", "model"];
const orderBy = validSortColumns.includes(sortBy) ? sortBy : "created_at";
const order = sortOrder === "asc" ? "ASC" : "DESC";
const countRow = db
.prepare(`SELECT COUNT(*) as total FROM semantic_cache ${whereClause}`)
.get(...params) as { total: number };
const entries = db
.prepare(
`SELECT id, signature, model, hit_count, tokens_saved, created_at, expires_at
FROM semantic_cache ${whereClause}
ORDER BY ${orderBy} ${order}
LIMIT ? OFFSET ?`
)
.all(...params, limit, offset) as CacheEntry[];
return NextResponse.json({
entries,
pagination: {
page,
limit,
total: countRow?.total || 0,
totalPages: Math.ceil((countRow?.total || 0) / limit),
},
});
} catch (error) {
return NextResponse.json({ error: String(error) }, { status: 500 });
}
}
export async function DELETE(req: NextRequest) {
try {
const { searchParams } = new URL(req.url);
const signature = searchParams.get("signature");
const model = searchParams.get("model");
const db = getDbInstance();
if (signature) {
db.prepare("DELETE FROM semantic_cache WHERE signature = ?").run(signature);
return NextResponse.json({ ok: true, deleted: 1 });
}
if (model) {
const result = db.prepare("DELETE FROM semantic_cache WHERE model = ?").run(model);
return NextResponse.json({ ok: true, deleted: result.changes });
}
return NextResponse.json({ error: "Provide signature or model parameter" }, { status: 400 });
} catch (error) {
return NextResponse.json({ error: String(error) }, { status: 500 });
}
}

View File

@@ -0,0 +1,73 @@
import { NextRequest, NextResponse } from "next/server";
import { getSettings, updateSettings } from "@/lib/localDb";
import { isAuthenticated } from "@/shared/utils/apiAuth";
const CACHE_CONFIG_KEYS = [
"semanticCacheEnabled",
"semanticCacheMaxSize",
"semanticCacheTTL",
"promptCacheEnabled",
"promptCacheStrategy",
"alwaysPreserveClientCache",
] as const;
const DEFAULTS = {
semanticCacheEnabled: true,
semanticCacheMaxSize: 100,
semanticCacheTTL: 1800000,
promptCacheEnabled: true,
promptCacheStrategy: "auto",
alwaysPreserveClientCache: "auto",
};
export async function GET(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const settings = await getSettings();
const config: Record<string, unknown> = {};
for (const key of CACHE_CONFIG_KEYS) {
config[key] = settings[key] ?? DEFAULTS[key];
}
return NextResponse.json(config);
} catch (error) {
return NextResponse.json({ error: String(error) }, { status: 500 });
}
}
export async function PUT(request: NextRequest) {
if (!(await isAuthenticated(request))) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
try {
const body = await request.json();
const updates: Record<string, unknown> = {};
if (typeof body.semanticCacheEnabled === "boolean") {
updates.semanticCacheEnabled = body.semanticCacheEnabled;
}
if (typeof body.semanticCacheMaxSize === "number" && body.semanticCacheMaxSize > 0) {
updates.semanticCacheMaxSize = body.semanticCacheMaxSize;
}
if (typeof body.semanticCacheTTL === "number" && body.semanticCacheTTL > 0) {
updates.semanticCacheTTL = body.semanticCacheTTL;
}
if (typeof body.promptCacheEnabled === "boolean") {
updates.promptCacheEnabled = body.promptCacheEnabled;
}
if (["auto", "system-only", "manual"].includes(body.promptCacheStrategy)) {
updates.promptCacheStrategy = body.promptCacheStrategy;
}
if (["auto", "always", "never"].includes(body.alwaysPreserveClientCache)) {
updates.alwaysPreserveClientCache = body.alwaysPreserveClientCache;
}
await updateSettings(updates);
return NextResponse.json({ ok: true });
} catch (error) {
return NextResponse.json({ error: String(error) }, { status: 500 });
}
}

View File

@@ -1712,6 +1712,17 @@
"cacheMisses": "Cache Misses",
"hitRate": "Hit Rate",
"cacheEntries": "Cache Entries",
"cacheSettings": "Cache Settings",
"semanticCache": "Semantic Cache",
"maxEntries": "Max Entries",
"ttlMinutes": "TTL (minutes)",
"promptCache": "Prompt Cache",
"strategy": "Strategy",
"preserveClientCache": "Preserve Client Cache",
"enabled": "Enabled",
"loading": "Loading...",
"saving": "Saving...",
"save": "Save",
"circuitBreaker": "Circuit Breaker",
"retryPolicy": "Retry Policy",
"maxRetries": "Max Retries",
@@ -2933,6 +2944,17 @@
"cachedTokensCol": "Cached",
"cacheCreation": "Creation",
"trend24h": "Cache Trend (24h)",
"cached": "Cached"
"cached": "Cached",
"overview": "Overview",
"entries": "Entries",
"searchEntries": "Search entries...",
"search": "Search",
"loading": "Loading...",
"noEntries": "No cache entries found",
"signature": "Signature",
"model": "Model",
"created": "Created",
"expires": "Expires",
"actions": "Actions"
}
}

View File

@@ -0,0 +1 @@
export { analyzePrefix, shouldInjectCacheControl } from "./prefixAnalyzer";

View File

@@ -0,0 +1,77 @@
import crypto from "crypto";
interface Message {
role: string;
content: string | unknown[];
}
interface PrefixAnalysis {
prefixEndIdx: number;
prefixHash: string;
prefixTokens: number;
prefixType: "system_only" | "system_and_tools" | "system_tools_history";
confidence: number;
}
function normalizeContent(content: string | unknown[]): string {
if (typeof content === "string") return content;
return JSON.stringify(content);
}
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}
export function analyzePrefix(messages: Message[]): PrefixAnalysis {
if (!Array.isArray(messages) || messages.length === 0) {
return {
prefixEndIdx: -1,
prefixHash: "",
prefixTokens: 0,
prefixType: "system_only",
confidence: 0,
};
}
let prefixEndIdx = -1;
let prefixType: PrefixAnalysis["prefixType"] = "system_only";
let confidence = 0.5;
for (let i = 0; i < messages.length; i++) {
const msg = messages[i];
const role = msg.role || "user";
if (role === "system") {
prefixEndIdx = i;
prefixType = "system_only";
confidence = 0.9;
} else if (role === "tool" || (role === "assistant" && Array.isArray(msg.content))) {
prefixEndIdx = i;
prefixType = "system_and_tools";
confidence = 0.8;
} else if (role === "assistant") {
prefixEndIdx = i;
prefixType = "system_tools_history";
confidence = 0.7;
} else {
break;
}
}
const prefixMessages = messages.slice(0, prefixEndIdx + 1);
const prefixText = prefixMessages.map((m) => normalizeContent(m.content)).join("\n");
const prefixHash = crypto.createHash("sha256").update(prefixText).digest("hex");
const prefixTokens = estimateTokens(prefixText);
return {
prefixEndIdx,
prefixHash,
prefixTokens,
prefixType,
confidence,
};
}
export function shouldInjectCacheControl(analysis: PrefixAnalysis, minTokens = 1024): boolean {
return analysis.prefixTokens >= minTokens && analysis.confidence >= 0.7;
}

View File

@@ -29,6 +29,45 @@ function toNumber(value: unknown, fallback = 0): number {
return fallback;
}
function ensureCacheMetricsTable() {
try {
const db = getDbInstance();
db.prepare(
`CREATE TABLE IF NOT EXISTS cache_metrics (
key TEXT PRIMARY KEY,
value INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)`
).run();
db.prepare(
`INSERT OR IGNORE INTO cache_metrics (key, value) VALUES ('hits', 0), ('misses', 0), ('tokens_saved', 0)`
).run();
} catch {
// DB not available
}
}
function incrementMetric(metric: "hits" | "misses" | "tokens_saved", amount = 1) {
try {
const db = getDbInstance();
db.prepare(
`UPDATE cache_metrics SET value = value + ?, updated_at = datetime('now') WHERE key = ?`
).run(amount, metric);
} catch {
// DB not available — fall back to in-memory
}
}
function getMetricValue(metric: string): number {
try {
const db = getDbInstance();
const row = db.prepare(`SELECT value FROM cache_metrics WHERE key = ?`).get(metric);
return row ? toNumber(asRecord(row).value, 0) : 0;
} catch {
return 0;
}
}
function getHeaderValue(
headers: { get?: (name: string) => string | null } | Record<string, unknown> | null | undefined,
name: string
@@ -51,7 +90,6 @@ function getHeaderValue(
// ─── Singleton ─────────────────
let memoryCache: LRUCache | null = null;
let stats = { hits: 0, misses: 0, tokensSaved: 0 };
function getMemoryCache() {
if (!memoryCache) {
@@ -60,6 +98,7 @@ function getMemoryCache() {
maxBytes: parseInt(process.env.SEMANTIC_CACHE_MAX_BYTES || String(4 * 1024 * 1024), 10),
defaultTTL: parseInt(process.env.SEMANTIC_CACHE_TTL_MS || "1800000", 10),
});
ensureCacheMetricsTable();
}
return memoryCache;
}
@@ -108,8 +147,8 @@ export function getCachedResponse(signature) {
// 1. Check memory cache
const memResult = getMemoryCache().get(signature);
if (memResult) {
stats.hits++;
stats.tokensSaved += memResult.tokensSaved || 0;
incrementMetric("hits");
incrementMetric("tokens_saved", memResult.tokensSaved || 0);
return memResult.response;
}
@@ -126,7 +165,7 @@ export function getCachedResponse(signature) {
const record = asRecord(row);
const responsePayload = typeof record.response === "string" ? record.response : null;
if (!responsePayload) {
stats.misses++;
incrementMetric("misses");
return null;
}
const parsed = JSON.parse(responsePayload);
@@ -141,15 +180,15 @@ export function getCachedResponse(signature) {
signature
);
stats.hits++;
stats.tokensSaved += tokensSaved;
incrementMetric("hits");
incrementMetric("tokens_saved", tokensSaved);
return parsed;
}
} catch {
// DB not available — fail open
}
stats.misses++;
incrementMetric("misses");
return null;
}
@@ -280,6 +319,17 @@ export function stopAutoCleanup(): void {
}
}
export function cleanOldMetrics(retentionDays = 90): number {
try {
const db = getDbInstance();
const cutoff = new Date(Date.now() - retentionDays * 86400000).toISOString();
const result = db.prepare("DELETE FROM semantic_cache WHERE created_at < ?").run(cutoff);
return result.changes || 0;
} catch {
return 0;
}
}
/**
* Clear all cache entries.
*/
@@ -288,17 +338,12 @@ export function clearCache() {
try {
const db = getDbInstance();
db.prepare("DELETE FROM semantic_cache").run();
db.prepare("UPDATE cache_metrics SET value = 0").run();
} catch {
// DB not available
}
stats = { hits: 0, misses: 0, tokensSaved: 0 };
}
// ─── Stats ─────────────────
/**
* Get cache statistics.
*/
export function getCacheStats() {
const memStats = getMemoryCache().getStats();
let dbSize = 0;
@@ -312,14 +357,18 @@ export function getCacheStats() {
// DB not available
}
const total = stats.hits + stats.misses;
const hits = getMetricValue("hits");
const misses = getMetricValue("misses");
const tokensSaved = getMetricValue("tokens_saved");
const total = hits + misses;
return {
memoryEntries: memStats.size,
dbEntries: dbSize,
hits: stats.hits,
misses: stats.misses,
hitRate: total > 0 ? ((stats.hits / total) * 100).toFixed(1) : "0.0",
tokensSaved: stats.tokensSaved,
hits,
misses,
hitRate: total > 0 ? ((hits / total) * 100).toFixed(1) : "0.0",
tokensSaved,
};
}