mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 04:12:10 +03:00
feat: add cache control settings and token-based metrics
Settings: - Add `alwaysPreserveClientCache` setting with modes: auto/always/never - UI toggle in Dashboard > Settings > Routing tab - Auto mode preserves cache_control for Claude Code clients with deterministic routing Metrics: - Track prompt cache token usage (input, cached, creation) - Display cache reuse ratio (cached/input tokens) - Breakdown by provider and routing strategy - Shows tokens saved and estimated cost savings API Endpoints: - GET /api/settings/cache-metrics - retrieve metrics - DELETE /api/settings/cache-metrics - reset metrics Files: - open-sse/utils/cacheControlPolicy.ts: CacheControlMetrics interface, trackCacheMetrics, updateCacheTokenMetrics - open-sse/handlers/chatCore.ts: Track cache tokens from provider responses - src/lib/db/settings.ts: Database functions for metrics persistence - src/lib/cacheControlSettings.ts: Cached settings accessor - src/app/(dashboard)/dashboard/settings/components/CacheStatsCard.tsx: Metrics dashboard UI - tests/unit/*.test.mjs: Unit tests (41 tests pass) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
tombii
@@ -2,7 +2,6 @@ import { getCorsOrigin } from "../utils/cors.ts";
|
||||
import { detectFormatFromEndpoint, getTargetFormat } from "../services/provider.ts";
|
||||
import { translateRequest, needsTranslation } from "../translator/index.ts";
|
||||
import { FORMATS } from "../translator/formats.ts";
|
||||
import { shouldPreserveCacheControl } from "../utils/cacheControlPolicy.ts";
|
||||
import {
|
||||
createSSETransformStreamWithLogger,
|
||||
createPassthroughStreamWithLogger,
|
||||
@@ -43,6 +42,14 @@ import {
|
||||
getModelUpstreamExtraHeaders,
|
||||
} from "@/lib/localDb";
|
||||
import { getExecutor } from "../executors/index.ts";
|
||||
import { getCacheControlSettings } from "@/lib/cacheControlSettings";
|
||||
import {
|
||||
shouldPreserveCacheControl,
|
||||
trackCacheMetrics,
|
||||
recordCacheHit,
|
||||
type CacheControlMetrics,
|
||||
} from "../utils/cacheControlPolicy.ts";
|
||||
import { getCacheMetrics, updateCacheMetrics } from "@/lib/db/settings.ts";
|
||||
|
||||
import {
|
||||
parseCodexQuotaHeaders,
|
||||
@@ -311,6 +318,7 @@ function attachLogMeta(
|
||||
* @param {string} options.comboName - Combo name if this is a combo request
|
||||
* @param {string} options.comboStrategy - Combo routing strategy (e.g., 'priority', 'cost-optimized')
|
||||
* @param {boolean} options.isCombo - Whether this request is from a combo
|
||||
* @param {string} options.connectionId - Connection ID for settings lookup
|
||||
*/
|
||||
export async function handleChatCore({
|
||||
body,
|
||||
@@ -683,11 +691,35 @@ export async function handleChatCore({
|
||||
const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE;
|
||||
|
||||
// Determine if we should preserve client-side cache_control headers
|
||||
// Fetch settings from DB to get user preference
|
||||
const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const);
|
||||
const preserveCacheControl = shouldPreserveCacheControl({
|
||||
userAgent,
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider: provider,
|
||||
settings: { alwaysPreserveClientCache: cacheControlMode },
|
||||
});
|
||||
|
||||
// Track cache metrics for this request
|
||||
let currentMetrics = await getCacheMetrics().catch(() => ({
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
}));
|
||||
|
||||
currentMetrics = trackCacheMetrics({
|
||||
preserved: preserveCacheControl,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
metrics: currentMetrics,
|
||||
});
|
||||
|
||||
if (preserveCacheControl) {
|
||||
@@ -1429,6 +1461,30 @@ export async function handleChatCore({
|
||||
const msg = `[${new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit" })}] 📊 [USAGE] ${provider.toUpperCase()} | in=${getLoggedInputTokens(usage)} | out=${getLoggedOutputTokens(usage)}${connectionId ? ` | account=${connectionId.slice(0, 8)}...` : ""}`;
|
||||
console.log(`${COLORS.green}${msg}${COLORS.reset}`);
|
||||
|
||||
// Track cache token metrics
|
||||
const inputTokens = usage.prompt_tokens || 0;
|
||||
const cachedTokens = toPositiveNumber(
|
||||
usage.cache_read_input_tokens ??
|
||||
usage.cached_tokens ??
|
||||
(usage as any).prompt_tokens_details?.cached_tokens
|
||||
);
|
||||
const cacheCreationTokens = toPositiveNumber(
|
||||
usage.cache_creation_input_tokens ??
|
||||
(usage as any).prompt_tokens_details?.cache_creation_tokens
|
||||
);
|
||||
|
||||
if (cachedTokens > 0 || cacheCreationTokens > 0) {
|
||||
currentMetrics = updateCacheTokenMetrics({
|
||||
metrics: currentMetrics,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved: 0, // Will be calculated based on pricing
|
||||
});
|
||||
}
|
||||
|
||||
saveRequestUsage({
|
||||
provider: provider || "unknown",
|
||||
model: model || "unknown",
|
||||
@@ -1536,6 +1592,11 @@ export async function handleChatCore({
|
||||
claudeCacheUsageMeta: cacheUsageLogMeta,
|
||||
});
|
||||
|
||||
// Persist cache metrics to database
|
||||
updateCacheMetrics(currentMetrics).catch((err) => {
|
||||
log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`);
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
response: new Response(JSON.stringify(translatedResponse), {
|
||||
@@ -1574,6 +1635,33 @@ export async function handleChatCore({
|
||||
clientPayload,
|
||||
}) => {
|
||||
const cacheUsageLogMeta = buildCacheUsageLogMeta(streamUsage);
|
||||
|
||||
// Track cache token metrics for streaming responses
|
||||
if (streamUsage && typeof streamUsage === "object") {
|
||||
const inputTokens = streamUsage.prompt_tokens || 0;
|
||||
const cachedTokens = toPositiveNumber(
|
||||
streamUsage.cache_read_input_tokens ??
|
||||
streamUsage.cached_tokens ??
|
||||
(streamUsage as any).prompt_tokens_details?.cached_tokens
|
||||
);
|
||||
const cacheCreationTokens = toPositiveNumber(
|
||||
streamUsage.cache_creation_input_tokens ??
|
||||
(streamUsage as any).prompt_tokens_details?.cache_creation_tokens
|
||||
);
|
||||
|
||||
if (cachedTokens > 0 || cacheCreationTokens > 0) {
|
||||
currentMetrics = updateCacheTokenMetrics({
|
||||
metrics: currentMetrics,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
persistAttemptLogs({
|
||||
status: streamStatus || 200,
|
||||
tokens: streamUsage || {},
|
||||
@@ -1585,6 +1673,11 @@ export async function handleChatCore({
|
||||
claudeCacheUsageMeta: cacheUsageLogMeta,
|
||||
});
|
||||
|
||||
// Persist cache metrics to database
|
||||
updateCacheMetrics(currentMetrics).catch((err) => {
|
||||
log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`);
|
||||
});
|
||||
|
||||
if (apiKeyInfo?.id && streamUsage) {
|
||||
calculateCost(provider, model, streamUsage)
|
||||
.then((estimatedCost) => {
|
||||
|
||||
@@ -12,13 +12,62 @@
|
||||
|
||||
import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies";
|
||||
|
||||
/**
|
||||
* Cache control preservation modes
|
||||
*/
|
||||
export type CacheControlMode = "auto" | "always" | "never";
|
||||
|
||||
/**
|
||||
* Cache control settings from the database
|
||||
*/
|
||||
export interface CacheControlSettings {
|
||||
alwaysPreserveClientCache?: CacheControlMode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache metrics for tracking effectiveness
|
||||
*/
|
||||
export interface CacheControlMetrics {
|
||||
// Totals
|
||||
totalRequests: number;
|
||||
requestsWithCacheControl: number;
|
||||
|
||||
// Token counts
|
||||
totalInputTokens: number;
|
||||
totalCachedTokens: number;
|
||||
totalCacheCreationTokens: number;
|
||||
|
||||
// Savings
|
||||
tokensSaved: number;
|
||||
estimatedCostSaved: number;
|
||||
|
||||
// Breakdowns
|
||||
byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Routing strategies that are deterministic (same request → same provider)
|
||||
*/
|
||||
const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set([
|
||||
"priority",
|
||||
"cost-optimized",
|
||||
]);
|
||||
const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set(["priority", "cost-optimized"]);
|
||||
|
||||
/**
|
||||
* Providers that support prompt caching
|
||||
@@ -55,7 +104,9 @@ export function providerSupportsCaching(provider: string | null | undefined): bo
|
||||
/**
|
||||
* Check if a routing strategy is deterministic
|
||||
*/
|
||||
export function isDeterministicStrategy(strategy: RoutingStrategyValue | null | undefined): boolean {
|
||||
export function isDeterministicStrategy(
|
||||
strategy: RoutingStrategyValue | null | undefined
|
||||
): boolean {
|
||||
if (!strategy) return false;
|
||||
return DETERMINISTIC_STRATEGIES.has(strategy);
|
||||
}
|
||||
@@ -67,6 +118,7 @@ export function isDeterministicStrategy(strategy: RoutingStrategyValue | null |
|
||||
* @param isCombo - Whether this is a combo model
|
||||
* @param comboStrategy - The combo's routing strategy (if applicable)
|
||||
* @param targetProvider - The target provider for the request
|
||||
* @param settings - Cache control settings from database (optional)
|
||||
* @returns true if cache_control should be preserved, false if OmniRoute should manage it
|
||||
*/
|
||||
export function shouldPreserveCacheControl({
|
||||
@@ -74,12 +126,23 @@ export function shouldPreserveCacheControl({
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider,
|
||||
settings,
|
||||
}: {
|
||||
userAgent: string | null | undefined;
|
||||
isCombo: boolean;
|
||||
comboStrategy?: RoutingStrategyValue | null;
|
||||
targetProvider: string | null | undefined;
|
||||
settings?: CacheControlSettings;
|
||||
}): boolean {
|
||||
// User override takes precedence
|
||||
if (settings?.alwaysPreserveClientCache === "always") {
|
||||
return true;
|
||||
}
|
||||
if (settings?.alwaysPreserveClientCache === "never") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Auto mode: use automatic detection (existing logic)
|
||||
// Must be a caching-aware client
|
||||
if (!isClaudeCodeClient(userAgent)) {
|
||||
return false;
|
||||
@@ -98,3 +161,145 @@ export function shouldPreserveCacheControl({
|
||||
// Combo: only preserve if strategy is deterministic
|
||||
return isDeterministicStrategy(comboStrategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Track cache control metrics for a request
|
||||
*/
|
||||
export function trackCacheMetrics({
|
||||
preserved,
|
||||
provider,
|
||||
strategy,
|
||||
metrics,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
}: {
|
||||
preserved: boolean;
|
||||
provider: string;
|
||||
strategy: string | null | undefined;
|
||||
metrics: CacheControlMetrics;
|
||||
inputTokens?: number;
|
||||
cachedTokens?: number;
|
||||
cacheCreationTokens?: number;
|
||||
}): CacheControlMetrics {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// Initialize metrics if empty
|
||||
if (!metrics) {
|
||||
metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
// Increment total requests
|
||||
metrics.totalRequests++;
|
||||
|
||||
// Track token counts
|
||||
const input = inputTokens || 0;
|
||||
const cached = cachedTokens || 0;
|
||||
const creation = cacheCreationTokens || 0;
|
||||
|
||||
metrics.totalInputTokens += input;
|
||||
metrics.totalCachedTokens += cached;
|
||||
metrics.totalCacheCreationTokens += creation;
|
||||
|
||||
// Calculate tokens saved (cached tokens are reused, not charged)
|
||||
if (cached > 0) {
|
||||
metrics.tokensSaved += cached;
|
||||
}
|
||||
|
||||
// Only track requests where cache_control was preserved
|
||||
if (preserved) {
|
||||
metrics.requestsWithCacheControl++;
|
||||
|
||||
// Initialize provider tracking
|
||||
if (!metrics.byProvider[provider]) {
|
||||
metrics.byProvider[provider] = {
|
||||
requests: 0,
|
||||
inputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
};
|
||||
}
|
||||
metrics.byProvider[provider].requests++;
|
||||
metrics.byProvider[provider].inputTokens += input;
|
||||
metrics.byProvider[provider].cachedTokens += cached;
|
||||
metrics.byProvider[provider].cacheCreationTokens += creation;
|
||||
|
||||
// Initialize strategy tracking
|
||||
if (strategy && !metrics.byStrategy[strategy]) {
|
||||
metrics.byStrategy[strategy] = {
|
||||
requests: 0,
|
||||
inputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
};
|
||||
}
|
||||
if (strategy) {
|
||||
metrics.byStrategy[strategy].requests++;
|
||||
metrics.byStrategy[strategy].inputTokens += input;
|
||||
metrics.byStrategy[strategy].cachedTokens += cached;
|
||||
metrics.byStrategy[strategy].cacheCreationTokens += creation;
|
||||
}
|
||||
}
|
||||
|
||||
metrics.lastUpdated = now;
|
||||
return metrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record cache token usage and update metrics
|
||||
*/
|
||||
export function updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider,
|
||||
strategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved,
|
||||
}: {
|
||||
metrics: CacheControlMetrics;
|
||||
provider: string;
|
||||
strategy: string | null | undefined;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
costSaved?: number;
|
||||
}): CacheControlMetrics {
|
||||
metrics.totalCachedTokens += cachedTokens;
|
||||
metrics.totalCacheCreationTokens += cacheCreationTokens;
|
||||
metrics.totalInputTokens += inputTokens;
|
||||
|
||||
// Cached tokens are reused (saved), creation tokens are new cache writes
|
||||
metrics.tokensSaved += cachedTokens;
|
||||
if (costSaved !== undefined) {
|
||||
metrics.estimatedCostSaved += costSaved;
|
||||
}
|
||||
|
||||
// Update provider tracking
|
||||
if (metrics.byProvider[provider]) {
|
||||
metrics.byProvider[provider].cachedTokens += cachedTokens;
|
||||
metrics.byProvider[provider].cacheCreationTokens += cacheCreationTokens;
|
||||
metrics.byProvider[provider].inputTokens += inputTokens;
|
||||
}
|
||||
|
||||
// Update strategy tracking
|
||||
if (strategy && metrics.byStrategy[strategy]) {
|
||||
metrics.byStrategy[strategy].cachedTokens += cachedTokens;
|
||||
metrics.byStrategy[strategy].cacheCreationTokens += cacheCreationTokens;
|
||||
metrics.byStrategy[strategy].inputTokens += inputTokens;
|
||||
}
|
||||
|
||||
metrics.lastUpdated = new Date().toISOString();
|
||||
return metrics;
|
||||
}
|
||||
|
||||
@@ -4,69 +4,190 @@ import { useState, useEffect } from "react";
|
||||
import { Card } from "@/shared/components";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
interface CacheMetrics {
|
||||
totalRequests: number;
|
||||
requestsWithCacheControl: number;
|
||||
totalInputTokens: number;
|
||||
totalCachedTokens: number;
|
||||
totalCacheCreationTokens: number;
|
||||
tokensSaved: number;
|
||||
estimatedCostSaved: number;
|
||||
byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
export default function CacheStatsCard() {
|
||||
const [cache, setCache] = useState(null);
|
||||
const [flushing, setFlushing] = useState(false);
|
||||
const [metrics, setMetrics] = useState<CacheMetrics | null>(null);
|
||||
const [resetting, setResetting] = useState(false);
|
||||
const t = useTranslations("settings");
|
||||
|
||||
const fetchStats = () => {
|
||||
fetch("/api/cache/stats")
|
||||
const fetchMetrics = () => {
|
||||
fetch("/api/settings/cache-metrics")
|
||||
.then((r) => r.json())
|
||||
.then(setCache)
|
||||
.then(setMetrics)
|
||||
.catch(() => {});
|
||||
};
|
||||
|
||||
useEffect(fetchStats, []);
|
||||
useEffect(fetchMetrics, []);
|
||||
|
||||
const handleFlush = async () => {
|
||||
setFlushing(true);
|
||||
const handleReset = async () => {
|
||||
setResetting(true);
|
||||
try {
|
||||
await fetch("/api/cache/stats", { method: "DELETE" });
|
||||
fetchStats();
|
||||
await fetch("/api/settings/cache-metrics", { method: "DELETE" });
|
||||
fetchMetrics();
|
||||
} finally {
|
||||
setFlushing(false);
|
||||
setResetting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const cacheHitRate =
|
||||
metrics && metrics.totalInputTokens > 0
|
||||
? (metrics.totalCachedTokens / metrics.totalInputTokens) * 100
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<Card className="p-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h3 className="text-lg font-semibold text-text-main flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[20px]">cached</span>
|
||||
{t("promptCache")}
|
||||
<span className="material-symbols-outlined text-[20px]">insights</span>
|
||||
Prompt Cache Metrics
|
||||
</h3>
|
||||
<button
|
||||
onClick={handleFlush}
|
||||
disabled={flushing}
|
||||
onClick={handleReset}
|
||||
disabled={resetting}
|
||||
className="px-3 py-1.5 text-xs rounded-lg bg-red-500/10 text-red-400 hover:bg-red-500/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
{flushing ? t("flushing") : t("flushCache")}
|
||||
{resetting ? "Resetting..." : "Reset Metrics"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{cache ? (
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">{t("size")}</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{cache.size}/{cache.maxSize}
|
||||
</p>
|
||||
{metrics ? (
|
||||
<div className="space-y-4">
|
||||
{/* Overview Stats */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Total Requests</p>
|
||||
<p className="font-mono text-lg text-text-main">{metrics.totalRequests}</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">With Cache Control</p>
|
||||
<p className="font-mono text-lg text-text-main">{metrics.requestsWithCacheControl}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("hitRate")}</p>
|
||||
<p className="font-mono text-lg text-text-main">{cache.hitRate?.toFixed(1) ?? 0}%</p>
|
||||
|
||||
{/* Token Stats */}
|
||||
<div className="grid grid-cols-3 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Input Tokens</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{metrics.totalInputTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Cached Tokens (Read)</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.totalCachedTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Cache Creation (Write)</p>
|
||||
<p className="font-mono text-lg text-blue-400">
|
||||
{metrics.totalCacheCreationTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("hits")}</p>
|
||||
<p className="font-mono text-text-main">{cache.hits ?? 0}</p>
|
||||
|
||||
{/* Cache Ratio */}
|
||||
<div className="rounded-lg bg-surface/50 border border-border/30 p-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-text-main">Cache Reuse Ratio</p>
|
||||
<p className="text-xs text-text-muted">Cached tokens / Total input tokens</p>
|
||||
</div>
|
||||
<p className="font-mono text-xl text-green-400">{cacheHitRate.toFixed(1)}%</p>
|
||||
</div>
|
||||
{/* Progress bar */}
|
||||
<div className="mt-2 h-2 rounded-full bg-border/30 overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-green-500 transition-all duration-300"
|
||||
style={{ width: `${Math.min(cacheHitRate, 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("evictions")}</p>
|
||||
<p className="font-mono text-text-main">{cache.evictions ?? 0}</p>
|
||||
|
||||
{/* Savings */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Tokens Saved</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.tokensSaved.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Est. Cost Saved</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
${metrics.estimatedCostSaved.toFixed(4)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* By Provider */}
|
||||
{Object.keys(metrics.byProvider).length > 0 && (
|
||||
<div className="pt-3 border-t border-border/30">
|
||||
<p className="text-xs font-medium text-text-muted mb-2">By Provider</p>
|
||||
<div className="space-y-2">
|
||||
{Object.entries(metrics.byProvider).map(([provider, stats]) => {
|
||||
const providerCacheRate =
|
||||
stats.inputTokens > 0 ? (stats.cachedTokens / stats.inputTokens) * 100 : 0;
|
||||
return (
|
||||
<div
|
||||
key={provider}
|
||||
className="flex items-center justify-between px-3 py-2 rounded bg-surface/30 text-xs"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-text-main capitalize w-24">{provider}</span>
|
||||
<span className="text-text-muted">{stats.requests} reqs</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4 font-mono">
|
||||
<span className="text-text-muted" title="Input tokens">
|
||||
In: {stats.inputTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400" title="Cached tokens (reads)">
|
||||
Cached: {stats.cachedTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-blue-400" title="Cache creation tokens (writes)">
|
||||
Write: {stats.cacheCreationTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400 w-12 text-right">
|
||||
{providerCacheRate.toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-sm text-text-muted">{t("loadingCacheStats")}</p>
|
||||
<p className="text-sm text-text-muted">Loading cache metrics...</p>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -19,7 +19,10 @@ const STRATEGIES = ROUTING_STRATEGIES.filter((strategy) =>
|
||||
}));
|
||||
|
||||
export default function RoutingTab() {
|
||||
const [settings, setSettings] = useState<any>({ fallbackStrategy: "fill-first" });
|
||||
const [settings, setSettings] = useState<any>({
|
||||
fallbackStrategy: "fill-first",
|
||||
alwaysPreserveClientCache: "auto",
|
||||
});
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [aliases, setAliases] = useState([]);
|
||||
const [newPattern, setNewPattern] = useState("");
|
||||
@@ -218,6 +221,74 @@ export default function RoutingTab() {
|
||||
|
||||
{/* Fallback Chains */}
|
||||
<FallbackChainsEditor />
|
||||
|
||||
{/* Client Cache Control */}
|
||||
<Card>
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div className="p-2 rounded-lg bg-green-500/10 text-green-500">
|
||||
<span className="material-symbols-outlined text-[20px]" aria-hidden="true">
|
||||
cached
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-lg font-semibold">Client Cache Control</h3>
|
||||
<p className="text-sm text-text-muted">
|
||||
Configure how client-side cache_control headers are handled
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
{[
|
||||
{
|
||||
value: "auto",
|
||||
label: "Auto (Recommended)",
|
||||
desc: "Preserve cache_control only for caching-aware clients (Claude Code) with deterministic routing",
|
||||
},
|
||||
{
|
||||
value: "always",
|
||||
label: "Always Preserve",
|
||||
desc: "Always forward client cache_control headers to upstream providers",
|
||||
},
|
||||
{
|
||||
value: "never",
|
||||
label: "Never Preserve",
|
||||
desc: "Always remove client cache_control headers, let OmniRoute manage caching",
|
||||
},
|
||||
].map((option) => (
|
||||
<button
|
||||
key={option.value}
|
||||
onClick={() => updateSetting({ alwaysPreserveClientCache: option.value })}
|
||||
disabled={loading}
|
||||
className={`w-full flex flex-col items-start gap-1 p-3 rounded-lg border text-left transition-all ${
|
||||
settings.alwaysPreserveClientCache === option.value
|
||||
? "border-green-500/50 bg-green-500/5 ring-1 ring-green-500/20"
|
||||
: "border-border/50 hover:border-border hover:bg-surface/30"
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className={`material-symbols-outlined text-[16px] ${
|
||||
settings.alwaysPreserveClientCache === option.value
|
||||
? "text-green-400"
|
||||
: "text-text-muted"
|
||||
}`}
|
||||
>
|
||||
{settings.alwaysPreserveClientCache === option.value
|
||||
? "check_circle"
|
||||
: "radio_button_unchecked"}
|
||||
</span>
|
||||
<span
|
||||
className={`text-sm font-medium ${settings.alwaysPreserveClientCache === option.value ? "text-green-400" : ""}`}
|
||||
>
|
||||
{option.label}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-text-muted ml-7">{option.desc}</p>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
22
src/app/api/settings/cache-metrics/route.ts
Normal file
22
src/app/api/settings/cache-metrics/route.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getCacheMetrics, resetCacheMetrics } from "@/lib/db/settings";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const metrics = await getCacheMetrics();
|
||||
return NextResponse.json(metrics);
|
||||
} catch (error) {
|
||||
console.error("Error getting cache metrics:", error);
|
||||
return NextResponse.json({ error: "Failed to load cache metrics" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE() {
|
||||
try {
|
||||
const metrics = await resetCacheMetrics();
|
||||
return NextResponse.json(metrics);
|
||||
} catch (error) {
|
||||
console.error("Error resetting cache metrics:", error);
|
||||
return NextResponse.json({ error: "Failed to reset cache metrics" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,12 @@ export async function PATCH(request) {
|
||||
invalidateCallLogsMaxCache();
|
||||
}
|
||||
|
||||
// Sync cache control settings to runtime cache
|
||||
if ("alwaysPreserveClientCache" in body) {
|
||||
const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings");
|
||||
invalidateCacheControlSettingsCache();
|
||||
}
|
||||
|
||||
const { password, ...safeSettings } = settings;
|
||||
return NextResponse.json(safeSettings);
|
||||
} catch (error) {
|
||||
|
||||
25
src/lib/cacheControlSettings.ts
Normal file
25
src/lib/cacheControlSettings.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Cache Control Settings
|
||||
*
|
||||
* Provides cached access to cache control settings for performance.
|
||||
* Settings are fetched once and cached to avoid repeated DB hits.
|
||||
*/
|
||||
|
||||
import { getSettings } from "./db/settings";
|
||||
import type { CacheControlMode } from "@omniroute/open-sse/utils/cacheControlPolicy";
|
||||
|
||||
let cachedSettings: CacheControlMode | null = null;
|
||||
|
||||
export async function getCacheControlSettings(): Promise<CacheControlMode> {
|
||||
if (cachedSettings !== null) {
|
||||
return cachedSettings;
|
||||
}
|
||||
|
||||
const settings = await getSettings();
|
||||
cachedSettings = (settings.alwaysPreserveClientCache as CacheControlMode) || "auto";
|
||||
return cachedSettings;
|
||||
}
|
||||
|
||||
export function invalidateCacheControlSettingsCache() {
|
||||
cachedSettings = null;
|
||||
}
|
||||
@@ -46,6 +46,7 @@ export async function getSettings() {
|
||||
stickyRoundRobinLimit: 3,
|
||||
requireLogin: true,
|
||||
hiddenSidebarItems: [],
|
||||
alwaysPreserveClientCache: "auto",
|
||||
};
|
||||
for (const row of rows) {
|
||||
const record = toRecord(row);
|
||||
@@ -486,3 +487,56 @@ export async function setProxyConfig(config: Record<string, unknown>) {
|
||||
backupDbFile("pre-write");
|
||||
return current;
|
||||
}
|
||||
|
||||
// ──────────────── Cache Control Metrics ────────────────
|
||||
|
||||
export async function getCacheMetrics() {
|
||||
const db = getDbInstance();
|
||||
const row = db
|
||||
.prepare("SELECT value FROM key_value WHERE namespace = 'settings' AND key = 'cacheMetrics'")
|
||||
.get() as { value?: string } | undefined;
|
||||
|
||||
if (!row || !row.value) {
|
||||
return {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
return JSON.parse(row.value);
|
||||
}
|
||||
|
||||
export async function updateCacheMetrics(metrics: Record<string, unknown>) {
|
||||
const db = getDbInstance();
|
||||
db.prepare(
|
||||
"INSERT OR REPLACE INTO key_value (namespace, key, value) VALUES ('settings', 'cacheMetrics', ?)"
|
||||
).run(JSON.stringify(metrics));
|
||||
backupDbFile("pre-write");
|
||||
return metrics;
|
||||
}
|
||||
|
||||
export async function resetCacheMetrics() {
|
||||
const db = getDbInstance();
|
||||
db.prepare("DELETE FROM key_value WHERE namespace = 'settings' AND key = 'cacheMetrics'").run();
|
||||
backupDbFile("pre-write");
|
||||
return {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ export const updateSettingsSchema = z.object({
|
||||
cliCompatProviders: z.array(z.string().max(100)).optional(),
|
||||
// Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4")
|
||||
stripModelPrefix: z.boolean().optional(),
|
||||
// Cache control preservation mode
|
||||
alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(),
|
||||
// Custom CLI agent definitions for ACP
|
||||
customAgents: z
|
||||
.array(
|
||||
|
||||
@@ -5,6 +5,8 @@ import {
|
||||
providerSupportsCaching,
|
||||
isDeterministicStrategy,
|
||||
shouldPreserveCacheControl,
|
||||
trackCacheMetrics,
|
||||
updateCacheTokenMetrics,
|
||||
} from "../../open-sse/utils/cacheControlPolicy.ts";
|
||||
|
||||
describe("Cache Control Policy", () => {
|
||||
@@ -248,5 +250,349 @@ describe("Cache Control Policy", () => {
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
describe("settings override", () => {
|
||||
test("alwaysPreserveClientCache=always overrides auto detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "curl/7.68.0", // non-Claude client
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "always" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("alwaysPreserveClientCache=never overrides auto detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0", // Claude client
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "never" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("alwaysPreserveClientCache=auto uses automatic detection", () => {
|
||||
// Should preserve for Claude client + caching provider
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
|
||||
// Should NOT preserve for non-Claude client
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "curl/7.68.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("undefined settings uses automatic detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: undefined,
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("trackCacheMetrics", () => {
|
||||
test("initializes empty metrics", () => {
|
||||
const result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics: undefined,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 500,
|
||||
cacheCreationTokens: 200,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 1);
|
||||
assert.equal(result.requestsWithCacheControl, 1);
|
||||
assert.equal(result.totalInputTokens, 1000);
|
||||
assert.equal(result.totalCachedTokens, 500);
|
||||
assert.equal(result.totalCacheCreationTokens, 200);
|
||||
assert.equal(result.tokensSaved, 500);
|
||||
});
|
||||
|
||||
test("increments total requests without cache control", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = trackCacheMetrics({
|
||||
preserved: false,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
metrics,
|
||||
inputTokens: 500,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 11);
|
||||
assert.equal(result.requestsWithCacheControl, 5); // unchanged
|
||||
assert.equal(result.totalInputTokens, 5500);
|
||||
});
|
||||
|
||||
test("tracks requests with cache control preserved", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 400,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 1);
|
||||
assert.equal(result.requestsWithCacheControl, 1);
|
||||
assert.equal(result.byProvider.claude.requests, 1);
|
||||
assert.equal(result.byProvider.claude.inputTokens, 1000);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 400);
|
||||
assert.equal(result.byProvider.claude.cacheCreationTokens, 100);
|
||||
assert.equal(result.byStrategy.priority.requests, 1);
|
||||
});
|
||||
|
||||
test("tracks by provider", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
let result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 300,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "zai",
|
||||
strategy: null,
|
||||
metrics: result,
|
||||
inputTokens: 800,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 50,
|
||||
});
|
||||
|
||||
assert.equal(result.byProvider.claude.requests, 1);
|
||||
assert.equal(result.byProvider.claude.inputTokens, 1000);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 300);
|
||||
assert.equal(result.byProvider.zai.requests, 1);
|
||||
assert.equal(result.byProvider.zai.inputTokens, 800);
|
||||
assert.equal(result.byProvider.zai.cachedTokens, 200);
|
||||
});
|
||||
|
||||
test("tracks by strategy", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
let result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 300,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "cost-optimized",
|
||||
metrics: result,
|
||||
inputTokens: 800,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 50,
|
||||
});
|
||||
|
||||
assert.equal(result.byStrategy.priority.requests, 1);
|
||||
assert.equal(result.byStrategy.priority.cachedTokens, 300);
|
||||
assert.equal(result.byStrategy["cost-optimized"].requests, 1);
|
||||
assert.equal(result.byStrategy["cost-optimized"].cachedTokens, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateCacheTokenMetrics", () => {
|
||||
test("updates token counts", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 3,
|
||||
inputTokens: 3000,
|
||||
cachedTokens: 1200,
|
||||
cacheCreationTokens: 600,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 4,
|
||||
inputTokens: 4000,
|
||||
cachedTokens: 1600,
|
||||
cacheCreationTokens: 800,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 400,
|
||||
cacheCreationTokens: 200,
|
||||
costSaved: 0.02,
|
||||
});
|
||||
|
||||
assert.equal(result.totalInputTokens, 6000);
|
||||
assert.equal(result.totalCachedTokens, 2400);
|
||||
assert.equal(result.totalCacheCreationTokens, 1200);
|
||||
assert.equal(result.tokensSaved, 2400);
|
||||
assert.equal(result.estimatedCostSaved, 0.52);
|
||||
});
|
||||
|
||||
test("updates provider breakdown", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 3,
|
||||
inputTokens: 3000,
|
||||
cachedTokens: 1200,
|
||||
cacheCreationTokens: 600,
|
||||
},
|
||||
},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
inputTokens: 500,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.byProvider.claude.inputTokens, 3500);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 1400);
|
||||
assert.equal(result.byProvider.claude.cacheCreationTokens, 700);
|
||||
});
|
||||
|
||||
test("updates strategy breakdown", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 4,
|
||||
inputTokens: 4000,
|
||||
cachedTokens: 1600,
|
||||
cacheCreationTokens: 800,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
inputTokens: 500,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.byStrategy.priority.inputTokens, 4500);
|
||||
assert.equal(result.byStrategy.priority.cachedTokens, 1800);
|
||||
assert.equal(result.byStrategy.priority.cacheCreationTokens, 900);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
241
tests/unit/cache-metrics.test.mjs
Normal file
241
tests/unit/cache-metrics.test.mjs
Normal file
@@ -0,0 +1,241 @@
|
||||
import { describe, test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
getCacheMetrics,
|
||||
updateCacheMetrics,
|
||||
resetCacheMetrics,
|
||||
} from "../../src/lib/db/settings.ts";
|
||||
import { getDbInstance } from "../../src/lib/db/core.ts";
|
||||
|
||||
describe("Cache Metrics Database", () => {
|
||||
let db;
|
||||
|
||||
before(() => {
|
||||
db = getDbInstance();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
// Clean up test data
|
||||
await resetCacheMetrics();
|
||||
});
|
||||
|
||||
describe("getCacheMetrics", () => {
|
||||
test("returns default metrics when none exist", async () => {
|
||||
// First reset to ensure clean state
|
||||
await resetCacheMetrics();
|
||||
|
||||
const metrics = await getCacheMetrics();
|
||||
|
||||
assert.equal(metrics.totalRequests, 0);
|
||||
assert.equal(metrics.requestsWithCacheControl, 0);
|
||||
assert.equal(metrics.totalInputTokens, 0);
|
||||
assert.equal(metrics.totalCachedTokens, 0);
|
||||
assert.equal(metrics.totalCacheCreationTokens, 0);
|
||||
assert.equal(metrics.tokensSaved, 0);
|
||||
assert.equal(metrics.estimatedCostSaved, 0);
|
||||
assert.deepStrictEqual(metrics.byProvider, {});
|
||||
assert.deepStrictEqual(metrics.byStrategy, {});
|
||||
assert.ok(metrics.lastUpdated);
|
||||
});
|
||||
|
||||
test("returns persisted metrics", async () => {
|
||||
const testMetrics = {
|
||||
totalRequests: 100,
|
||||
requestsWithCacheControl: 50,
|
||||
totalInputTokens: 50000,
|
||||
totalCachedTokens: 20000,
|
||||
totalCacheCreationTokens: 10000,
|
||||
tokensSaved: 20000,
|
||||
estimatedCostSaved: 1.25,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 30,
|
||||
inputTokens: 30000,
|
||||
cachedTokens: 12000,
|
||||
cacheCreationTokens: 6000,
|
||||
},
|
||||
zai: {
|
||||
requests: 20,
|
||||
inputTokens: 20000,
|
||||
cachedTokens: 8000,
|
||||
cacheCreationTokens: 4000,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 40,
|
||||
inputTokens: 40000,
|
||||
cachedTokens: 16000,
|
||||
cacheCreationTokens: 8000,
|
||||
},
|
||||
"cost-optimized": {
|
||||
requests: 10,
|
||||
inputTokens: 10000,
|
||||
cachedTokens: 4000,
|
||||
cacheCreationTokens: 2000,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
await updateCacheMetrics(testMetrics);
|
||||
const retrieved = await getCacheMetrics();
|
||||
|
||||
assert.equal(retrieved.totalRequests, 100);
|
||||
assert.equal(retrieved.requestsWithCacheControl, 50);
|
||||
assert.equal(retrieved.totalInputTokens, 50000);
|
||||
assert.equal(retrieved.totalCachedTokens, 20000);
|
||||
assert.equal(retrieved.totalCacheCreationTokens, 10000);
|
||||
assert.deepStrictEqual(retrieved.byProvider, testMetrics.byProvider);
|
||||
assert.deepStrictEqual(retrieved.byStrategy, testMetrics.byStrategy);
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateCacheMetrics", () => {
|
||||
test("persists metrics to database", async () => {
|
||||
const testMetrics = {
|
||||
totalRequests: 42,
|
||||
requestsWithCacheControl: 20,
|
||||
totalInputTokens: 21000,
|
||||
totalCachedTokens: 8400,
|
||||
totalCacheCreationTokens: 4200,
|
||||
tokensSaved: 8400,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 15,
|
||||
inputTokens: 15000,
|
||||
cachedTokens: 6000,
|
||||
cacheCreationTokens: 3000,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 18,
|
||||
inputTokens: 18000,
|
||||
cachedTokens: 7200,
|
||||
cacheCreationTokens: 3600,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = await updateCacheMetrics(testMetrics);
|
||||
|
||||
assert.equal(result.totalRequests, 42);
|
||||
assert.equal(result.requestsWithCacheControl, 20);
|
||||
|
||||
// Verify persistence by retrieving
|
||||
const retrieved = await getCacheMetrics();
|
||||
assert.equal(retrieved.totalRequests, 42);
|
||||
});
|
||||
|
||||
test("updates existing metrics", async () => {
|
||||
// Set initial metrics
|
||||
await updateCacheMetrics({
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.1,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Update with new values
|
||||
await updateCacheMetrics({
|
||||
totalRequests: 20,
|
||||
requestsWithCacheControl: 10,
|
||||
totalInputTokens: 10000,
|
||||
totalCachedTokens: 4000,
|
||||
totalCacheCreationTokens: 2000,
|
||||
tokensSaved: 4000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 8,
|
||||
inputTokens: 8000,
|
||||
cachedTokens: 3200,
|
||||
cacheCreationTokens: 1600,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 9,
|
||||
inputTokens: 9000,
|
||||
cachedTokens: 3600,
|
||||
cacheCreationTokens: 1800,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const retrieved = await getCacheMetrics();
|
||||
|
||||
assert.equal(retrieved.totalRequests, 20);
|
||||
assert.equal(retrieved.requestsWithCacheControl, 10);
|
||||
assert.equal(retrieved.totalInputTokens, 10000);
|
||||
assert.equal(retrieved.totalCachedTokens, 4000);
|
||||
assert.deepStrictEqual(retrieved.byProvider, {
|
||||
claude: {
|
||||
requests: 8,
|
||||
inputTokens: 8000,
|
||||
cachedTokens: 3200,
|
||||
cacheCreationTokens: 1600,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resetCacheMetrics", () => {
|
||||
test("clears all metrics", async () => {
|
||||
// Set some metrics first
|
||||
await updateCacheMetrics({
|
||||
totalRequests: 100,
|
||||
requestsWithCacheControl: 50,
|
||||
totalInputTokens: 50000,
|
||||
totalCachedTokens: 20000,
|
||||
totalCacheCreationTokens: 10000,
|
||||
tokensSaved: 20000,
|
||||
estimatedCostSaved: 2.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 40,
|
||||
inputTokens: 40000,
|
||||
cachedTokens: 16000,
|
||||
cacheCreationTokens: 8000,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 45,
|
||||
inputTokens: 45000,
|
||||
cachedTokens: 18000,
|
||||
cacheCreationTokens: 9000,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Reset
|
||||
const result = await resetCacheMetrics();
|
||||
|
||||
assert.equal(result.totalRequests, 0);
|
||||
assert.equal(result.requestsWithCacheControl, 0);
|
||||
assert.equal(result.totalInputTokens, 0);
|
||||
assert.equal(result.totalCachedTokens, 0);
|
||||
assert.equal(result.totalCacheCreationTokens, 0);
|
||||
assert.equal(result.tokensSaved, 0);
|
||||
assert.equal(result.estimatedCostSaved, 0);
|
||||
assert.deepStrictEqual(result.byProvider, {});
|
||||
assert.deepStrictEqual(result.byStrategy, {});
|
||||
|
||||
// Verify database is cleared
|
||||
const retrieved = await getCacheMetrics();
|
||||
assert.equal(retrieved.totalRequests, 0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user