diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 2021e3a61b..3d907b03b3 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -51,12 +51,13 @@ import { getModelPreserveOpenAIDeveloperRole, getModelUpstreamExtraHeaders, getUpstreamProxyConfig, + getCachedSettings, } from "@/lib/localDb"; import { getExecutor } from "../executors/index.ts"; -import { getCacheControlSettings } from "@/lib/cacheControlSettings"; import { shouldPreserveCacheControl, providerSupportsCaching, + type CacheControlMode, } from "../utils/cacheControlPolicy.ts"; import { getCacheMetrics } from "@/lib/db/settings.ts"; @@ -810,9 +811,16 @@ export async function handleChatCore({ } const stream = resolveStreamFlag(body?.stream, acceptHeader); + const runtimeSettings = await getCachedSettings().catch(() => ({}) as Record); + const semanticCacheEnabled = runtimeSettings.semanticCacheEnabled !== false; + const cacheControlMode = + runtimeSettings.alwaysPreserveClientCache === "always" || + runtimeSettings.alwaysPreserveClientCache === "never" + ? (runtimeSettings.alwaysPreserveClientCache as CacheControlMode) + : "auto"; // ── Phase 9.1: Semantic cache check (non-streaming, temp=0 only) ── - if (isCacheable(body, clientRawRequest?.headers)) { + if (semanticCacheEnabled && isCacheable(body, clientRawRequest?.headers)) { const signature = generateSignature(model, body.messages, body.temperature, body.top_p); const cached = getCachedResponse(signature); if (cached) { @@ -948,8 +956,6 @@ export async function handleChatCore({ let ccSessionId: string | null = null; // Determine if we should preserve client-side cache_control headers - // Fetch settings from DB to get user preference - const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const); const preserveCacheControl = shouldPreserveCacheControl({ userAgent, isCombo, @@ -2269,7 +2275,7 @@ export async function handleChatCore({ } // ── Phase 9.1: Cache store (non-streaming, temp=0) ── - if (isCacheable(body, clientRawRequest?.headers)) { + if (semanticCacheEnabled && isCacheable(body, clientRawRequest?.headers)) { const signature = generateSignature(model, body.messages, body.temperature, body.top_p); const tokensSaved = usage?.prompt_tokens + usage?.completion_tokens || 0; setCachedResponse(signature, model, translatedResponse, tokensSaved); diff --git a/open-sse/mcp-server/schemas/tools.ts b/open-sse/mcp-server/schemas/tools.ts index 99d15a20ce..8a65141ab5 100644 --- a/open-sse/mcp-server/schemas/tools.ts +++ b/open-sse/mcp-server/schemas/tools.ts @@ -884,8 +884,10 @@ export const cacheStatsOutput = z.object({ .object({ totalRequests: z.number(), requestsWithCacheControl: z.number(), + totalInputTokens: z.number(), totalCachedTokens: z.number(), totalCacheCreationTokens: z.number(), + tokensSaved: z.number(), estimatedCostSaved: z.number(), }) .nullable(), @@ -893,6 +895,11 @@ export const cacheStatsOutput = z.object({ activeKeys: z.number(), windowMs: z.number(), }), + config: z + .object({ + semanticCacheEnabled: z.boolean(), + }) + .optional(), }); export const cacheStatsTool: McpToolDefinition = { diff --git a/src/app/(dashboard)/dashboard/cache/__tests__/CachePage.test.tsx b/src/app/(dashboard)/dashboard/cache/__tests__/CachePage.test.tsx new file mode 100644 index 0000000000..d9529fd56f --- /dev/null +++ b/src/app/(dashboard)/dashboard/cache/__tests__/CachePage.test.tsx @@ -0,0 +1,210 @@ +// @vitest-environment jsdom +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen, waitFor, cleanup, fireEvent } from "@testing-library/react"; +import "@testing-library/jest-dom/vitest"; +import React from "react"; +import CachePage from "../page"; + +const notifications = { + success: vi.fn(), + error: vi.fn(), +}; + +vi.mock("next-intl", () => ({ + useTranslations: (namespace?: string) => { + const getMessage = (key: string) => { + const fullKey = namespace ? `${namespace}.${key}` : key; + const messages: Record = { + "cache.title": "Cache Management", + "cache.description": + "Monitor provider prompt cache efficiency and local semantic response reuse.", + "cache.refresh": "Refresh", + "cache.promptCache": "Prompt Cache (Provider-Side)", + "cache.promptCacheSectionDesc": "Prompt cache section", + "cache.lastUpdated": "Last updated", + "cache.withCacheControl": "With Cache Control", + "cache.cacheRateDesc": "of total requests", + "cache.cacheReuseRatio": "Cache Reuse Ratio", + "cache.cacheReuseRatioDesc": "Cache read tokens / Total input tokens", + "cache.cachedTokens": "Cache Read Tokens", + "cache.cachedTokensRead": "Read from cache", + "cache.estCostSaved": "Est. Cost Saved", + "cache.cacheCreationTokens": "Cache Write Tokens", + "cache.cacheRate": "Cache Rate", + "cache.requests": "Requests", + "cache.inputTokens": "Input Tokens", + "cache.tokensSaved": "Tokens Saved", + "cache.trend24h": "Cache Trend (24h)", + "cache.cached": "Cached", + "cache.byProvider": "Breakdown by Provider", + "cache.providerCacheRateDesc": "Provider cache rate description", + "cache.cachedTokensCol": "Cache Read", + "cache.cacheCreation": "Cache Write", + "cache.cacheCreationWrite": "Written to cache", + "cache.inputTokens": "Total Input Tokens", + "cache.semanticCache": "Semantic Cache", + "cache.semanticCacheSectionDesc": "Semantic cache section", + "cache.semanticCacheDisabledDesc": "Semantic cache is disabled.", + "cache.memoryEntries": "Memory Entries", + "cache.memoryEntriesSub": "In-memory LRU", + "cache.dbEntries": "DB Entries", + "cache.dbEntriesSub": "Persisted (SQLite)", + "cache.cacheHits": "Cache Hits", + "cache.cacheHitsSub": "of {total} total", + "cache.tokensSavedSub": "Estimated from hits", + "cache.performance": "Cache Performance", + "cache.autoRefresh": "Auto-refreshes every {seconds}s", + "cache.hitRate": "Hit Rate", + "cache.hits": "Hits", + "cache.misses": "Misses", + "cache.total": "Total", + "cache.behavior": "Cache Behavior", + "cache.behaviorDeterministic": "Only non-streaming requests with temperature=0 are cached.", + "cache.behaviorTwoTier": "Two-tier storage: in-memory LRU + SQLite.", + "cache.behaviorTtl": "TTL via {envVar}.", + "cache.idempotency": "Idempotency Layer", + "cache.activeDedupKeys": "Active Dedup Keys", + "cache.dedupWindow": "Dedup Window", + "cache.entries": "Entries", + "cache.semanticEntriesDesc": "Semantic entries only.", + "cache.searchEntries": "Search entries...", + "cache.search": "Search", + "cache.loading": "Loading...", + "cache.noEntries": "No cache entries found", + "cache.clearAll": "Clear Semantic Cache", + "settings.enabled": "Enabled", + disabled: "Disabled", + }; + + return messages[fullKey] ?? key; + }; + + const translate = (key: string, values?: Record) => { + let message = getMessage(key); + if (values) { + for (const [name, value] of Object.entries(values)) { + message = message.replace(`{${name}}`, String(value)); + } + } + return message; + }; + + translate.rich = (key: string, values?: Record React.ReactNode>) => { + if (key === "behaviorBypass") { + return <>Bypass with header {values?.header?.()}.; + } + if (key === "behaviorTtl") { + return <>TTL via {values?.envVar?.()}.; + } + return translate(key); + }; + + return translate; + }, +})); + +vi.mock("@/store/notificationStore", () => ({ + useNotificationStore: () => notifications, +})); + +describe("CachePage", () => { + const fetchMock = vi.fn(); + + beforeEach(() => { + fetchMock.mockReset(); + notifications.success.mockReset(); + notifications.error.mockReset(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + it("switches between prompt and semantic cache views", async () => { + fetchMock.mockImplementation(async (input: string) => { + if (input.startsWith("/api/cache/entries")) { + return { + ok: true, + json: async () => ({ + entries: [], + pagination: { page: 1, limit: 20, total: 0, totalPages: 0 }, + }), + }; + } + + return { + ok: true, + json: async () => ({ + semanticCache: { + memoryEntries: 0, + dbEntries: 0, + hits: 0, + misses: 3, + hitRate: "0.0", + tokensSaved: 0, + }, + promptCache: { + totalRequests: 10, + requestsWithCacheControl: 6, + totalInputTokens: 1000, + totalCachedTokens: 550, + totalCacheCreationTokens: 180, + tokensSaved: 550, + estimatedCostSaved: 0.12, + byProvider: { + claude: { + requests: 6, + totalRequests: 10, + cachedRequests: 6, + inputTokens: 1000, + cachedTokens: 550, + cacheCreationTokens: 180, + }, + }, + byStrategy: {}, + lastUpdated: "2026-04-11T05:00:00.000Z", + }, + trend: [ + { + timestamp: "2026-04-11T05:00:00.000Z", + requests: 10, + cachedRequests: 6, + inputTokens: 1000, + cachedTokens: 550, + cacheCreationTokens: 180, + }, + ], + idempotency: { + activeKeys: 2, + windowMs: 5000, + }, + config: { + semanticCacheEnabled: false, + }, + }), + }; + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Prompt Cache (Provider-Side)")).toBeInTheDocument(); + }); + + expect(screen.getByText("Breakdown by Provider")).toBeInTheDocument(); + expect(screen.getByText("claude")).toBeInTheDocument(); + expect(screen.getAllByText("60.0%").length).toBeGreaterThan(0); + expect(screen.queryByText("Semantic cache is disabled.")).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Semantic Cache" })); + + await waitFor(() => { + expect(screen.getByText("Semantic cache is disabled.")).toBeInTheDocument(); + }); + + expect(screen.getByText("Entries")).toBeInTheDocument(); + expect(screen.getByText("Active Dedup Keys")).toBeInTheDocument(); + }); +}); diff --git a/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx b/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx index 323fb6300d..0ceb58ec49 100644 --- a/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx +++ b/src/app/(dashboard)/dashboard/cache/components/CacheEntriesTab.tsx @@ -33,10 +33,12 @@ export default function CacheEntriesTab() { const [loading, setLoading] = useState(true); const [search, setSearch] = useState(""); const [deleting, setDeleting] = useState(null); + const [error, setError] = useState(null); const fetchEntries = useCallback( async (page = 1) => { setLoading(true); + setError(null); try { const params = new URLSearchParams({ page: String(page), limit: String(pagination.limit) }); if (search) params.set("search", search); @@ -46,14 +48,18 @@ export default function CacheEntriesTab() { const data = await res.json(); setEntries(data.entries); setPagination(data.pagination); + } else { + setEntries([]); + setError(t("entriesLoadError")); } } catch { - // ignore + setEntries([]); + setError(t("entriesLoadError")); } finally { setLoading(false); } }, - [search, pagination.limit] + [pagination.limit, search, t] ); useEffect(() => { @@ -94,6 +100,13 @@ export default function CacheEntriesTab() { {loading ? (
{t("loading")}
+ ) : error ? ( +
+
{error}
+ +
) : entries.length === 0 ? (
{t("noEntries")}
) : ( diff --git a/src/app/(dashboard)/dashboard/cache/page.tsx b/src/app/(dashboard)/dashboard/cache/page.tsx index 3ed8993f2f..5b828e1b42 100644 --- a/src/app/(dashboard)/dashboard/cache/page.tsx +++ b/src/app/(dashboard)/dashboard/cache/page.tsx @@ -1,13 +1,11 @@ "use client"; -import { useState, useEffect, useCallback } from "react"; +import { useState, useEffect, useCallback, type ReactNode } from "react"; import { Card, Button, EmptyState } from "@/shared/components"; import { useNotificationStore } from "@/store/notificationStore"; import { useTranslations } from "next-intl"; import CacheEntriesTab from "./components/CacheEntriesTab"; -// ─── Types ─────────────────────────────────────────────────────────────────── - interface SemanticCacheStats { memoryEntries: number; dbEntries: number; @@ -19,6 +17,8 @@ interface SemanticCacheStats { interface PromptCacheProviderStats { requests: number; + totalRequests?: number; + cachedRequests?: number; inputTokens: number; cachedTokens: number; cacheCreationTokens: number; @@ -51,75 +51,145 @@ interface CacheTrendPoint { cacheCreationTokens: number; } +interface CacheConfig { + semanticCacheEnabled: boolean; +} + interface CacheStats { semanticCache: SemanticCacheStats; promptCache: PromptCacheMetrics | null; trend: CacheTrendPoint[]; idempotency: IdempotencyStats; + config?: CacheConfig; } -// ─── Sub-components ────────────────────────────────────────────────────────── +type CacheView = "prompt" | "semantic"; function StatCard({ icon, label, value, sub, - valueClass = "text-text", + accent = "text-text-main", + size = "default", }: { icon: string; label: string; value: string | number; sub?: string; - valueClass?: string; + accent?: string; + size?: "default" | "hero"; }) { - return ( -
-
- - {label} -
-
{value}
- {sub &&
{sub}
} -
- ); -} - -function HitRateBar({ hitRate, label }: { hitRate: number; label: string }) { - const colorClass = hitRate >= 70 ? "bg-green-500" : hitRate >= 40 ? "bg-amber-400" : "bg-red-500"; - const textClass = - hitRate >= 70 ? "text-green-500" : hitRate >= 40 ? "text-amber-400" : "text-red-500"; + const isHero = size === "hero"; return (
-
- {label} - {hitRate.toFixed(1)}% +
+ + {label}
-
-
+
+ {value}
+ {sub &&
{sub}
}
); } -function InfoRow({ icon, children }: { icon: string; children: React.ReactNode }) { +function SectionBadge({ + icon, + children, + tone = "neutral", +}: { + icon: string; + children: ReactNode; + tone?: "neutral" | "green" | "amber"; +}) { + const toneClass = + tone === "green" + ? "border-green-500/20 bg-green-500/10 text-green-300" + : tone === "amber" + ? "border-amber-400/20 bg-amber-400/10 text-amber-300" + : "border-border/40 bg-surface/50 text-text-muted"; + + return ( + + + {children} + + ); +} + +function LinearMeter({ + label, + value, + tone = "green", + helper, +}: { + label: string; + value: number; + tone?: "green" | "blue" | "amber"; + helper?: string; +}) { + const colorClass = + tone === "blue" ? "bg-blue-400" : tone === "amber" ? "bg-amber-400" : "bg-emerald-500"; + + return ( +
+
+ {label} + {value.toFixed(1)}% +
+
+
+
+ {helper &&
{helper}
} +
+ ); +} + +function DetailStat({ + label, + value, + accent = "text-text-main", +}: { + label: string; + value: string; + accent?: string; +}) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function InfoRow({ icon, children }: { icon: string; children: ReactNode }) { return (