diff --git a/.github/workflows/sync-upstream.yml b/.github/workflows/sync-upstream.yml new file mode 100644 index 0000000000..dc9129a125 --- /dev/null +++ b/.github/workflows/sync-upstream.yml @@ -0,0 +1,43 @@ +name: Sync Upstream + +on: + schedule: + # Run every 6 hours + - cron: '0 */6 * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + sync: + name: Sync with upstream + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Fetch upstream + run: | + git remote add upstream https://github.com/diegosouzapw/OmniRoute.git || true + git fetch upstream + git fetch origin + + - name: Sync main branch + run: | + git checkout main + git merge upstream/main --no-edit || { + echo "Merge conflict detected. Manual intervention required." + exit 1 + } + + - name: Push changes + run: git push https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/tombii/OmniRoute.git main diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 4cdbf8d139..f8527678d5 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -42,6 +42,14 @@ import { getModelUpstreamExtraHeaders, } from "@/lib/localDb"; import { getExecutor } from "../executors/index.ts"; +import { getCacheControlSettings } from "@/lib/cacheControlSettings"; +import { + shouldPreserveCacheControl, + trackCacheMetrics, + recordCacheHit, + type CacheControlMetrics, +} from "../utils/cacheControlPolicy.ts"; +import { getCacheMetrics, updateCacheMetrics } from "@/lib/db/settings.ts"; import { parseCodexQuotaHeaders, @@ -306,6 +314,11 @@ function attachLogMeta( * @param {function} options.onDisconnect - Callback when client disconnects * @param {string} options.connectionId - Connection ID for usage tracking * @param {object} options.apiKeyInfo - API key metadata for usage attribution + * @param {string} options.userAgent - Client user agent for caching decisions + * @param {string} options.comboName - Combo name if this is a combo request + * @param {string} options.comboStrategy - Combo routing strategy (e.g., 'priority', 'cost-optimized') + * @param {boolean} options.isCombo - Whether this request is from a combo + * @param {string} options.connectionId - Connection ID for settings lookup */ export async function handleChatCore({ body, @@ -320,6 +333,8 @@ export async function handleChatCore({ apiKeyInfo = null, userAgent, comboName, + comboStrategy = null, + isCombo = false, }) { let { provider, model, extendedContext } = modelInfo; const requestedModel = @@ -674,6 +689,46 @@ export async function handleChatCore({ // Translate request (pass reqLogger for intermediate logging) let translatedBody = body; const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE; + + // Determine if we should preserve client-side cache_control headers + // Fetch settings from DB to get user preference + const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const); + const preserveCacheControl = shouldPreserveCacheControl({ + userAgent, + isCombo, + comboStrategy, + targetProvider: provider, + settings: { alwaysPreserveClientCache: cacheControlMode }, + }); + + // Track cache metrics for this request + let currentMetrics = await getCacheMetrics().catch(() => ({ + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + })); + + currentMetrics = trackCacheMetrics({ + preserved: preserveCacheControl, + provider, + strategy: comboStrategy, + metrics: currentMetrics, + }); + + if (preserveCacheControl) { + log?.debug?.( + "CACHE", + `Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})` + ); + } + try { if (nativeCodexPassthrough) { translatedBody = { ...body, _nativeCodexPassthrough: true }; @@ -701,7 +756,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); translatedBody = translateRequest( FORMATS.OPENAI, @@ -712,7 +767,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); log?.debug?.("FORMAT", "claude->openai->claude normalized passthrough"); } else { @@ -816,7 +871,7 @@ export async function handleChatCore({ credentials, provider, reqLogger, - { normalizeToolCallId, preserveDeveloperRole } + { normalizeToolCallId, preserveDeveloperRole, preserveCacheControl } ); } } catch (error) { @@ -1406,6 +1461,30 @@ export async function handleChatCore({ const msg = `[${new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit" })}] 📊 [USAGE] ${provider.toUpperCase()} | in=${getLoggedInputTokens(usage)} | out=${getLoggedOutputTokens(usage)}${connectionId ? ` | account=${connectionId.slice(0, 8)}...` : ""}`; console.log(`${COLORS.green}${msg}${COLORS.reset}`); + // Track cache token metrics + const inputTokens = usage.prompt_tokens || 0; + const cachedTokens = toPositiveNumber( + usage.cache_read_input_tokens ?? + usage.cached_tokens ?? + (usage as any).prompt_tokens_details?.cached_tokens + ); + const cacheCreationTokens = toPositiveNumber( + usage.cache_creation_input_tokens ?? + (usage as any).prompt_tokens_details?.cache_creation_tokens + ); + + if (cachedTokens > 0 || cacheCreationTokens > 0) { + currentMetrics = updateCacheTokenMetrics({ + metrics: currentMetrics, + provider, + strategy: comboStrategy, + inputTokens, + cachedTokens, + cacheCreationTokens, + costSaved: 0, // Will be calculated based on pricing + }); + } + saveRequestUsage({ provider: provider || "unknown", model: model || "unknown", @@ -1513,6 +1592,11 @@ export async function handleChatCore({ claudeCacheUsageMeta: cacheUsageLogMeta, }); + // Persist cache metrics to database + updateCacheMetrics(currentMetrics).catch((err) => { + log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`); + }); + return { success: true, response: new Response(JSON.stringify(translatedResponse), { @@ -1551,6 +1635,33 @@ export async function handleChatCore({ clientPayload, }) => { const cacheUsageLogMeta = buildCacheUsageLogMeta(streamUsage); + + // Track cache token metrics for streaming responses + if (streamUsage && typeof streamUsage === "object") { + const inputTokens = streamUsage.prompt_tokens || 0; + const cachedTokens = toPositiveNumber( + streamUsage.cache_read_input_tokens ?? + streamUsage.cached_tokens ?? + (streamUsage as any).prompt_tokens_details?.cached_tokens + ); + const cacheCreationTokens = toPositiveNumber( + streamUsage.cache_creation_input_tokens ?? + (streamUsage as any).prompt_tokens_details?.cache_creation_tokens + ); + + if (cachedTokens > 0 || cacheCreationTokens > 0) { + currentMetrics = updateCacheTokenMetrics({ + metrics: currentMetrics, + provider, + strategy: comboStrategy, + inputTokens, + cachedTokens, + cacheCreationTokens, + costSaved: 0, + }); + } + } + persistAttemptLogs({ status: streamStatus || 200, tokens: streamUsage || {}, @@ -1562,6 +1673,11 @@ export async function handleChatCore({ claudeCacheUsageMeta: cacheUsageLogMeta, }); + // Persist cache metrics to database + updateCacheMetrics(currentMetrics).catch((err) => { + log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`); + }); + if (apiKeyInfo?.id && streamUsage) { calculateCost(provider, model, streamUsage) .then((estimatedCost) => { diff --git a/open-sse/translator/index.ts b/open-sse/translator/index.ts index 10f3675272..79be284b99 100644 --- a/open-sse/translator/index.ts +++ b/open-sse/translator/index.ts @@ -73,6 +73,7 @@ function normalizeOpenAIResponsesRequest(body) { /** @param options.normalizeToolCallId - When true, use 9-char tool call ids (e.g. Mistral); when false, leave ids as-is */ /** @param options.preserveDeveloperRole - undefined/true: keep developer for OpenAI format (default); false: map to system */ +/** @param options.preserveCacheControl - When true, preserve client-side cache_control markers (for Claude Code, etc.) */ // Translate request: source -> openai -> target export function translateRequest( sourceFormat, @@ -83,7 +84,7 @@ export function translateRequest( credentials = null, provider = null, reqLogger = null, - options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean } + options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean; preserveCacheControl?: boolean } ) { let result = body; const use9CharId = options?.normalizeToolCallId === true; @@ -149,10 +150,13 @@ export function translateRequest( } // Final step: prepare request for Claude format endpoints - // In Claude passthrough mode (Claude → Claude), preserve cache_control markers + // Preserve cache_control when: + // 1. Claude passthrough mode (Claude → Claude), OR + // 2. Explicitly requested via options (for caching-aware clients like Claude Code) if (targetFormat === FORMATS.CLAUDE) { const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE; - result = prepareClaudeRequest(result, provider, isClaudePassthrough); + const preserveCache = isClaudePassthrough || options?.preserveCacheControl === true; + result = prepareClaudeRequest(result, provider, preserveCache); } // Normalize openai-responses input shape for providers that require list input. diff --git a/open-sse/utils/cacheControlPolicy.ts b/open-sse/utils/cacheControlPolicy.ts new file mode 100644 index 0000000000..af501d7495 --- /dev/null +++ b/open-sse/utils/cacheControlPolicy.ts @@ -0,0 +1,305 @@ +/** + * Cache Control Policy + * + * Determines when to preserve client-side prompt caching headers (cache_control) + * vs. applying OmniRoute's own caching strategy. + * + * Client-side caching (e.g., Claude Code) should be preserved when: + * 1. Client is Claude Code or similar caching-aware client + * 2. Request will hit a deterministic target (single model or deterministic combo strategy) + * 3. Provider supports prompt caching (Anthropic, Alibaba Qwen, etc.) + */ + +import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies"; + +/** + * Cache control preservation modes + */ +export type CacheControlMode = "auto" | "always" | "never"; + +/** + * Cache control settings from the database + */ +export interface CacheControlSettings { + alwaysPreserveClientCache?: CacheControlMode; +} + +/** + * Cache metrics for tracking effectiveness + */ +export interface CacheControlMetrics { + // Totals + totalRequests: number; + requestsWithCacheControl: number; + + // Token counts + totalInputTokens: number; + totalCachedTokens: number; + totalCacheCreationTokens: number; + + // Savings + tokensSaved: number; + estimatedCostSaved: number; + + // Breakdowns + byProvider: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + >; + byStrategy: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + >; + + lastUpdated: string; +} + +/** + * Routing strategies that are deterministic (same request → same provider) + */ +const DETERMINISTIC_STRATEGIES: Set = new Set(["priority", "cost-optimized"]); + +/** + * Providers that support prompt caching + */ +const CACHING_PROVIDERS = new Set([ + "claude", + "anthropic", + "zai", + "qwen", // Alibaba Qwen Coding Plan International +]); + +/** + * Detect if the client is Claude Code or another caching-aware client + */ +export function isClaudeCodeClient(userAgent: string | null | undefined): boolean { + if (!userAgent) return false; + const ua = userAgent.toLowerCase(); + + // Claude Code user agents + if (ua.includes("claude-code") || ua.includes("claude_code")) return true; + if (ua.includes("anthropic") && ua.includes("cli")) return true; + + return false; +} + +/** + * Check if a provider supports prompt caching + */ +export function providerSupportsCaching(provider: string | null | undefined): boolean { + if (!provider) return false; + return CACHING_PROVIDERS.has(provider.toLowerCase()); +} + +/** + * Check if a routing strategy is deterministic + */ +export function isDeterministicStrategy( + strategy: RoutingStrategyValue | null | undefined +): boolean { + if (!strategy) return false; + return DETERMINISTIC_STRATEGIES.has(strategy); +} + +/** + * Determine if client-side cache_control headers should be preserved + * + * @param userAgent - User-Agent header from the request + * @param isCombo - Whether this is a combo model + * @param comboStrategy - The combo's routing strategy (if applicable) + * @param targetProvider - The target provider for the request + * @param settings - Cache control settings from database (optional) + * @returns true if cache_control should be preserved, false if OmniRoute should manage it + */ +export function shouldPreserveCacheControl({ + userAgent, + isCombo, + comboStrategy, + targetProvider, + settings, +}: { + userAgent: string | null | undefined; + isCombo: boolean; + comboStrategy?: RoutingStrategyValue | null; + targetProvider: string | null | undefined; + settings?: CacheControlSettings; +}): boolean { + // User override takes precedence + if (settings?.alwaysPreserveClientCache === "always") { + return true; + } + if (settings?.alwaysPreserveClientCache === "never") { + return false; + } + + // Auto mode: use automatic detection (existing logic) + // Must be a caching-aware client + if (!isClaudeCodeClient(userAgent)) { + return false; + } + + // Target provider must support caching + if (!providerSupportsCaching(targetProvider)) { + return false; + } + + // Single model: always preserve (deterministic) + if (!isCombo) { + return true; + } + + // Combo: only preserve if strategy is deterministic + return isDeterministicStrategy(comboStrategy); +} + +/** + * Track cache control metrics for a request + */ +export function trackCacheMetrics({ + preserved, + provider, + strategy, + metrics, + inputTokens, + cachedTokens, + cacheCreationTokens, +}: { + preserved: boolean; + provider: string; + strategy: string | null | undefined; + metrics: CacheControlMetrics; + inputTokens?: number; + cachedTokens?: number; + cacheCreationTokens?: number; +}): CacheControlMetrics { + const now = new Date().toISOString(); + + // Initialize metrics if empty + if (!metrics) { + metrics = { + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: now, + }; + } + + // Increment total requests + metrics.totalRequests++; + + // Track token counts + const input = inputTokens || 0; + const cached = cachedTokens || 0; + const creation = cacheCreationTokens || 0; + + metrics.totalInputTokens += input; + metrics.totalCachedTokens += cached; + metrics.totalCacheCreationTokens += creation; + + // Calculate tokens saved (cached tokens are reused, not charged) + if (cached > 0) { + metrics.tokensSaved += cached; + } + + // Only track requests where cache_control was preserved + if (preserved) { + metrics.requestsWithCacheControl++; + + // Initialize provider tracking + if (!metrics.byProvider[provider]) { + metrics.byProvider[provider] = { + requests: 0, + inputTokens: 0, + cachedTokens: 0, + cacheCreationTokens: 0, + }; + } + metrics.byProvider[provider].requests++; + metrics.byProvider[provider].inputTokens += input; + metrics.byProvider[provider].cachedTokens += cached; + metrics.byProvider[provider].cacheCreationTokens += creation; + + // Initialize strategy tracking + if (strategy && !metrics.byStrategy[strategy]) { + metrics.byStrategy[strategy] = { + requests: 0, + inputTokens: 0, + cachedTokens: 0, + cacheCreationTokens: 0, + }; + } + if (strategy) { + metrics.byStrategy[strategy].requests++; + metrics.byStrategy[strategy].inputTokens += input; + metrics.byStrategy[strategy].cachedTokens += cached; + metrics.byStrategy[strategy].cacheCreationTokens += creation; + } + } + + metrics.lastUpdated = now; + return metrics; +} + +/** + * Record cache token usage and update metrics + */ +export function updateCacheTokenMetrics({ + metrics, + provider, + strategy, + inputTokens, + cachedTokens, + cacheCreationTokens, + costSaved, +}: { + metrics: CacheControlMetrics; + provider: string; + strategy: string | null | undefined; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + costSaved?: number; +}): CacheControlMetrics { + metrics.totalCachedTokens += cachedTokens; + metrics.totalCacheCreationTokens += cacheCreationTokens; + metrics.totalInputTokens += inputTokens; + + // Cached tokens are reused (saved), creation tokens are new cache writes + metrics.tokensSaved += cachedTokens; + if (costSaved !== undefined) { + metrics.estimatedCostSaved += costSaved; + } + + // Update provider tracking + if (metrics.byProvider[provider]) { + metrics.byProvider[provider].cachedTokens += cachedTokens; + metrics.byProvider[provider].cacheCreationTokens += cacheCreationTokens; + metrics.byProvider[provider].inputTokens += inputTokens; + } + + // Update strategy tracking + if (strategy && metrics.byStrategy[strategy]) { + metrics.byStrategy[strategy].cachedTokens += cachedTokens; + metrics.byStrategy[strategy].cacheCreationTokens += cacheCreationTokens; + metrics.byStrategy[strategy].inputTokens += inputTokens; + } + + metrics.lastUpdated = new Date().toISOString(); + return metrics; +} diff --git a/src/app/(dashboard)/dashboard/settings/components/CacheStatsCard.tsx b/src/app/(dashboard)/dashboard/settings/components/CacheStatsCard.tsx index a9f43a5783..d269907683 100644 --- a/src/app/(dashboard)/dashboard/settings/components/CacheStatsCard.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/CacheStatsCard.tsx @@ -4,69 +4,190 @@ import { useState, useEffect } from "react"; import { Card } from "@/shared/components"; import { useTranslations } from "next-intl"; +interface CacheMetrics { + totalRequests: number; + requestsWithCacheControl: number; + totalInputTokens: number; + totalCachedTokens: number; + totalCacheCreationTokens: number; + tokensSaved: number; + estimatedCostSaved: number; + byProvider: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + >; + byStrategy: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + >; + lastUpdated: string; +} + export default function CacheStatsCard() { - const [cache, setCache] = useState(null); - const [flushing, setFlushing] = useState(false); + const [metrics, setMetrics] = useState(null); + const [resetting, setResetting] = useState(false); const t = useTranslations("settings"); - const fetchStats = () => { - fetch("/api/cache/stats") + const fetchMetrics = () => { + fetch("/api/settings/cache-metrics") .then((r) => r.json()) - .then(setCache) + .then(setMetrics) .catch(() => {}); }; - useEffect(fetchStats, []); + useEffect(fetchMetrics, []); - const handleFlush = async () => { - setFlushing(true); + const handleReset = async () => { + setResetting(true); try { - await fetch("/api/cache/stats", { method: "DELETE" }); - fetchStats(); + await fetch("/api/settings/cache-metrics", { method: "DELETE" }); + fetchMetrics(); } finally { - setFlushing(false); + setResetting(false); } }; + const cacheHitRate = + metrics && metrics.totalInputTokens > 0 + ? (metrics.totalCachedTokens / metrics.totalInputTokens) * 100 + : 0; + return (

- cached - {t("promptCache")} + insights + Prompt Cache Metrics

- {cache ? ( -
-
-

{t("size")}

-

- {cache.size}/{cache.maxSize} -

+ {metrics ? ( +
+ {/* Overview Stats */} +
+
+

Total Requests

+

{metrics.totalRequests}

+
+
+

With Cache Control

+

{metrics.requestsWithCacheControl}

+
-
-

{t("hitRate")}

-

{cache.hitRate?.toFixed(1) ?? 0}%

+ + {/* Token Stats */} +
+
+

Input Tokens

+

+ {metrics.totalInputTokens.toLocaleString()} +

+
+
+

Cached Tokens (Read)

+

+ {metrics.totalCachedTokens.toLocaleString()} +

+
+
+

Cache Creation (Write)

+

+ {metrics.totalCacheCreationTokens.toLocaleString()} +

+
-
-

{t("hits")}

-

{cache.hits ?? 0}

+ + {/* Cache Ratio */} +
+
+
+

Cache Reuse Ratio

+

Cached tokens / Total input tokens

+
+

{cacheHitRate.toFixed(1)}%

+
+ {/* Progress bar */} +
+
+
-
-

{t("evictions")}

-

{cache.evictions ?? 0}

+ + {/* Savings */} +
+
+

Tokens Saved

+

+ {metrics.tokensSaved.toLocaleString()} +

+
+
+

Est. Cost Saved

+

+ ${metrics.estimatedCostSaved.toFixed(4)} +

+
+ + {/* By Provider */} + {Object.keys(metrics.byProvider).length > 0 && ( +
+

By Provider

+
+ {Object.entries(metrics.byProvider).map(([provider, stats]) => { + const providerCacheRate = + stats.inputTokens > 0 ? (stats.cachedTokens / stats.inputTokens) * 100 : 0; + return ( +
+
+ {provider} + {stats.requests} reqs +
+
+ + In: {stats.inputTokens.toLocaleString()} + + + Cached: {stats.cachedTokens.toLocaleString()} + + + Write: {stats.cacheCreationTokens.toLocaleString()} + + + {providerCacheRate.toFixed(0)}% + +
+
+ ); + })} +
+
+ )}
) : ( -

{t("loadingCacheStats")}

+

Loading cache metrics...

)} ); diff --git a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx index 87054cb6f5..697a69eb06 100644 --- a/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx +++ b/src/app/(dashboard)/dashboard/settings/components/RoutingTab.tsx @@ -19,7 +19,10 @@ const STRATEGIES = ROUTING_STRATEGIES.filter((strategy) => })); export default function RoutingTab() { - const [settings, setSettings] = useState({ fallbackStrategy: "fill-first" }); + const [settings, setSettings] = useState({ + fallbackStrategy: "fill-first", + alwaysPreserveClientCache: "auto", + }); const [loading, setLoading] = useState(true); const [aliases, setAliases] = useState([]); const [newPattern, setNewPattern] = useState(""); @@ -218,6 +221,74 @@ export default function RoutingTab() { {/* Fallback Chains */} + + {/* Client Cache Control */} + +
+
+ +
+
+

Client Cache Control

+

+ Configure how client-side cache_control headers are handled +

+
+
+ +
+ {[ + { + value: "auto", + label: "Auto (Recommended)", + desc: "Preserve cache_control only for caching-aware clients (Claude Code) with deterministic routing", + }, + { + value: "always", + label: "Always Preserve", + desc: "Always forward client cache_control headers to upstream providers", + }, + { + value: "never", + label: "Never Preserve", + desc: "Always remove client cache_control headers, let OmniRoute manage caching", + }, + ].map((option) => ( + + ))} +
+
); } diff --git a/src/app/api/settings/cache-metrics/route.ts b/src/app/api/settings/cache-metrics/route.ts new file mode 100644 index 0000000000..5154802455 --- /dev/null +++ b/src/app/api/settings/cache-metrics/route.ts @@ -0,0 +1,22 @@ +import { NextResponse } from "next/server"; +import { getCacheMetrics, resetCacheMetrics } from "@/lib/db/settings"; + +export async function GET() { + try { + const metrics = await getCacheMetrics(); + return NextResponse.json(metrics); + } catch (error) { + console.error("Error getting cache metrics:", error); + return NextResponse.json({ error: "Failed to load cache metrics" }, { status: 500 }); + } +} + +export async function DELETE() { + try { + const metrics = await resetCacheMetrics(); + return NextResponse.json(metrics); + } catch (error) { + console.error("Error resetting cache metrics:", error); + return NextResponse.json({ error: "Failed to reset cache metrics" }, { status: 500 }); + } +} diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 2c73f0d0d4..7bf98a878e 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -119,6 +119,12 @@ export async function PATCH(request) { invalidateCallLogsMaxCache(); } + // Sync cache control settings to runtime cache + if ("alwaysPreserveClientCache" in body) { + const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings"); + invalidateCacheControlSettingsCache(); + } + const { password, ...safeSettings } = settings; return NextResponse.json(safeSettings); } catch (error) { diff --git a/src/lib/cacheControlSettings.ts b/src/lib/cacheControlSettings.ts new file mode 100644 index 0000000000..35130837a1 --- /dev/null +++ b/src/lib/cacheControlSettings.ts @@ -0,0 +1,25 @@ +/** + * Cache Control Settings + * + * Provides cached access to cache control settings for performance. + * Settings are fetched once and cached to avoid repeated DB hits. + */ + +import { getSettings } from "./db/settings"; +import type { CacheControlMode } from "@omniroute/open-sse/utils/cacheControlPolicy"; + +let cachedSettings: CacheControlMode | null = null; + +export async function getCacheControlSettings(): Promise { + if (cachedSettings !== null) { + return cachedSettings; + } + + const settings = await getSettings(); + cachedSettings = (settings.alwaysPreserveClientCache as CacheControlMode) || "auto"; + return cachedSettings; +} + +export function invalidateCacheControlSettingsCache() { + cachedSettings = null; +} diff --git a/src/lib/db/settings.ts b/src/lib/db/settings.ts index 4822d08dc6..00224a353a 100644 --- a/src/lib/db/settings.ts +++ b/src/lib/db/settings.ts @@ -46,6 +46,7 @@ export async function getSettings() { stickyRoundRobinLimit: 3, requireLogin: true, hiddenSidebarItems: [], + alwaysPreserveClientCache: "auto", }; for (const row of rows) { const record = toRecord(row); @@ -486,3 +487,177 @@ export async function setProxyConfig(config: Record) { backupDbFile("pre-write"); return current; } + +// ──────────────── Cache Control Metrics ──────────────── +// Cache metrics are now computed from usage_history table on-the-fly +// This avoids race conditions and keeps a single source of truth for token data + +export async function getCacheMetrics() { + const db = getDbInstance(); + + try { + // Aggregate totals from usage_history + const totalsRow = db + .prepare( + ` + SELECT + COUNT(*) as totalRequests, + SUM(tokens_input) as totalInputTokens, + SUM(tokens_cache_read) as totalCachedTokens, + SUM(tokens_cache_creation) as totalCacheCreationTokens + FROM usage_history + WHERE tokens_cache_read > 0 OR tokens_cache_creation > 0 + ` + ) + .get() as + | { + totalRequests: number; + totalInputTokens: number | null; + totalCachedTokens: number | null; + totalCacheCreationTokens: number | null; + } + | undefined; + + // Get all requests count (including those without cache activity) + const allRequestsRow = db + .prepare( + ` + SELECT COUNT(*) as totalRequests + FROM usage_history + ` + ) + .get() as { totalRequests: number } | undefined; + + // Aggregate by provider + const byProviderRows = db + .prepare( + ` + SELECT + provider, + COUNT(*) as requests, + SUM(tokens_input) as inputTokens, + SUM(tokens_cache_read) as cachedTokens, + SUM(tokens_cache_creation) as cacheCreationTokens + FROM usage_history + WHERE (tokens_cache_read > 0 OR tokens_cache_creation > 0) + AND provider IS NOT NULL + GROUP BY provider + ` + ) + .all() as Array<{ + provider: string; + requests: number; + inputTokens: number | null; + cachedTokens: number | null; + cacheCreationTokens: number | null; + }>; + + // Aggregate by strategy + // Since combo_strategy isn't tracked in usage_history yet, we use 'direct' for all requests + // TODO: Add combo_strategy column to usage_history for proper strategy tracking + const byStrategyRows = db + .prepare( + ` + SELECT + 'direct' as strategy, + COUNT(*) as requests, + SUM(tokens_input) as inputTokens, + SUM(tokens_cache_read) as cachedTokens, + SUM(tokens_cache_creation) as cacheCreationTokens + FROM usage_history + WHERE (tokens_cache_read > 0 OR tokens_cache_creation > 0) + GROUP BY 'direct' + ` + ) + .all() as Array<{ + strategy: string; + requests: number; + inputTokens: number | null; + cachedTokens: number | null; + cacheCreationTokens: number | null; + }>; + + // Calculate tokens saved (cached tokens are reused, not charged at full price) + const tokensSaved = totalsRow?.totalCachedTokens || 0; + + // Build byProvider object + const byProvider: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + > = {}; + for (const row of byProviderRows) { + byProvider[row.provider] = { + requests: row.requests, + inputTokens: row.inputTokens || 0, + cachedTokens: row.cachedTokens || 0, + cacheCreationTokens: row.cacheCreationTokens || 0, + }; + } + + // Build byStrategy object + const byStrategy: Record< + string, + { + requests: number; + inputTokens: number; + cachedTokens: number; + cacheCreationTokens: number; + } + > = {}; + for (const row of byStrategyRows) { + byStrategy[row.strategy] = { + requests: row.requests, + inputTokens: row.inputTokens || 0, + cachedTokens: row.cachedTokens || 0, + cacheCreationTokens: row.cacheCreationTokens || 0, + }; + } + + return { + totalRequests: allRequestsRow?.totalRequests || totalsRow?.totalRequests || 0, + requestsWithCacheControl: totalsRow?.totalRequests || 0, + totalInputTokens: totalsRow?.totalInputTokens || 0, + totalCachedTokens: totalsRow?.totalCachedTokens || 0, + totalCacheCreationTokens: totalsRow?.totalCacheCreationTokens || 0, + tokensSaved, + estimatedCostSaved: 0, // Would need pricing data to calculate + byProvider, + byStrategy, + lastUpdated: new Date().toISOString(), + }; + } catch (error) { + console.error("Failed to fetch cache metrics from usage_history:", error); + return { + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + } +} + +export async function updateCacheMetrics(_metrics: Record) { + // No-op: metrics are now computed from usage_history on-the-fly + // The usage_history table is the single source of truth + return getCacheMetrics(); +} + +export async function resetCacheMetrics() { + // No-op: cannot delete historical usage data + // Cache metrics are computed from usage_history, so they reflect actual request history + console.warn( + "resetCacheMetrics is deprecated - cache metrics are now computed from usage_history" + ); + return getCacheMetrics(); +} diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts index ec3392f792..9fea61ae3d 100644 --- a/src/shared/validation/settingsSchemas.ts +++ b/src/shared/validation/settingsSchemas.ts @@ -47,6 +47,8 @@ export const updateSettingsSchema = z.object({ cliCompatProviders: z.array(z.string().max(100)).optional(), // Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4") stripModelPrefix: z.boolean().optional(), + // Cache control preservation mode + alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(), // Custom CLI agent definitions for ACP customAgents: z .array( diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts index 0fbf689550..7013b1d86f 100644 --- a/src/sse/handlers/chat.ts +++ b/src/sse/handlers/chat.ts @@ -275,7 +275,7 @@ export async function handleChat(request: any, clientRawRequest: any = null) { handleSingleModel: (b: any, m: string) => handleSingleModelChat(b, m, clientRawRequest, request, combo.name, apiKeyInfo, telemetry, { sessionId, - }), + }, combo.strategy, true), isModelAvailable: checkModelAvailable, log, settings, @@ -304,7 +304,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) { combo.name, apiKeyInfo, telemetry, - { sessionId, emergencyFallbackTried: true } + { sessionId, emergencyFallbackTried: true }, + combo.strategy, + true ); if (fallbackResponse.ok) { log.info("GLOBAL_FALLBACK", `Global fallback ${fallbackModel} succeeded`); @@ -336,7 +338,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) { null, apiKeyInfo, telemetry, - { sessionId } + { sessionId }, + null, + false ); recordTelemetry(telemetry); return withSessionHeader(response, sessionId); @@ -366,7 +370,9 @@ async function handleSingleModelChat( comboName: string | null = null, apiKeyInfo: any = null, telemetry: any = null, - runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {} + runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {}, + comboStrategy: string | null = null, + isCombo: boolean = false ) { // 1. Resolve model → provider/model const resolved = await resolveModelOrError(modelStr, body, clientRawRequest?.endpoint); @@ -443,6 +449,8 @@ async function handleSingleModelChat( apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, extendedContext, }); if (telemetry) telemetry.endPhase(); @@ -512,7 +520,9 @@ async function handleSingleModelChat( comboName, apiKeyInfo, telemetry, - { ...runtimeOptions, emergencyFallbackTried: true } + { ...runtimeOptions, emergencyFallbackTried: true }, + null, // no strategy for emergency fallback + Boolean(comboName) // isCombo if comboName exists ); if (fallbackResponse.ok) { @@ -648,6 +658,8 @@ async function executeChatWithBreaker({ apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, extendedContext, }: any): Promise<{ result: any; tlsFingerprintUsed: boolean }> { let tlsFingerprintUsed = false; @@ -665,6 +677,8 @@ async function executeChatWithBreaker({ apiKeyInfo, userAgent, comboName, + comboStrategy, + isCombo, onCredentialsRefreshed: async (newCreds: any) => { await updateProviderCredentials(credentials.connectionId, { accessToken: newCreds.accessToken, diff --git a/tests/unit/cache-control-policy.test.mjs b/tests/unit/cache-control-policy.test.mjs new file mode 100644 index 0000000000..abf6456f37 --- /dev/null +++ b/tests/unit/cache-control-policy.test.mjs @@ -0,0 +1,598 @@ +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { + isClaudeCodeClient, + providerSupportsCaching, + isDeterministicStrategy, + shouldPreserveCacheControl, + trackCacheMetrics, + updateCacheTokenMetrics, +} from "../../open-sse/utils/cacheControlPolicy.ts"; + +describe("Cache Control Policy", () => { + describe("isClaudeCodeClient", () => { + test("detects claude-code user agent", () => { + assert.equal(isClaudeCodeClient("claude-code/0.1.0"), true); + assert.equal(isClaudeCodeClient("claude_code/0.1.0"), true); + assert.equal(isClaudeCodeClient("Anthropic CLI/1.0"), true); + }); + + test("rejects non-Claude clients", () => { + assert.equal(isClaudeCodeClient("curl/7.68.0"), false); + assert.equal(isClaudeCodeClient("OpenAI/1.0"), false); + assert.equal(isClaudeCodeClient(null), false); + assert.equal(isClaudeCodeClient(undefined), false); + assert.equal(isClaudeCodeClient(""), false); + }); + + test("is case-insensitive", () => { + assert.equal(isClaudeCodeClient("Claude-Code/0.1.0"), true); + assert.equal(isClaudeCodeClient("CLAUDE-CODE/0.1.0"), true); + }); + }); + + describe("providerSupportsCaching", () => { + test("detects caching providers", () => { + assert.equal(providerSupportsCaching("claude"), true); + assert.equal(providerSupportsCaching("anthropic"), true); + assert.equal(providerSupportsCaching("zai"), true); + assert.equal(providerSupportsCaching("qwen"), true); + }); + + test("rejects non-caching providers", () => { + assert.equal(providerSupportsCaching("openai"), false); + assert.equal(providerSupportsCaching("gemini"), false); + assert.equal(providerSupportsCaching("unknown"), false); + assert.equal(providerSupportsCaching(null), false); + assert.equal(providerSupportsCaching(undefined), false); + }); + + test("is case-insensitive", () => { + assert.equal(providerSupportsCaching("Claude"), true); + assert.equal(providerSupportsCaching("ANTHROPIC"), true); + }); + }); + + describe("isDeterministicStrategy", () => { + test("identifies deterministic strategies", () => { + assert.equal(isDeterministicStrategy("priority"), true); + assert.equal(isDeterministicStrategy("cost-optimized"), true); + }); + + test("identifies non-deterministic strategies", () => { + assert.equal(isDeterministicStrategy("weighted"), false); + assert.equal(isDeterministicStrategy("round-robin"), false); + assert.equal(isDeterministicStrategy("random"), false); + assert.equal(isDeterministicStrategy("fill-first"), false); + assert.equal(isDeterministicStrategy("p2c"), false); + assert.equal(isDeterministicStrategy("least-used"), false); + assert.equal(isDeterministicStrategy("strict-random"), false); + }); + + test("handles null/undefined", () => { + assert.equal(isDeterministicStrategy(null), false); + assert.equal(isDeterministicStrategy(undefined), false); + }); + }); + + describe("shouldPreserveCacheControl", () => { + test("preserves for single model + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "claude", + }), + true + ); + }); + + test("preserves for combo with priority strategy + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "priority", + targetProvider: "claude", + }), + true + ); + }); + + test("preserves for combo with cost-optimized strategy + Claude client + caching provider", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "cost-optimized", + targetProvider: "anthropic", + }), + true + ); + }); + + test("rejects non-Claude clients", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "curl/7.68.0", + isCombo: false, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects non-caching providers", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "openai", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (weighted)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "weighted", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (round-robin)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "round-robin", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with non-deterministic strategy (random)", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "random", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with fill-first strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "fill-first", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with p2c strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "p2c", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with least-used strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "least-used", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with strict-random strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: "strict-random", + targetProvider: "claude", + }), + false + ); + }); + + test("rejects combo with null strategy", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: true, + comboStrategy: null, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects when userAgent is null", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: null, + isCombo: false, + targetProvider: "claude", + }), + false + ); + }); + + test("rejects when targetProvider is null", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: null, + }), + false + ); + }); + + describe("settings override", () => { + test("alwaysPreserveClientCache=always overrides auto detection", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "curl/7.68.0", // non-Claude client + isCombo: false, + targetProvider: "claude", + settings: { alwaysPreserveClientCache: "always" }, + }), + true + ); + }); + + test("alwaysPreserveClientCache=never overrides auto detection", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", // Claude client + isCombo: false, + targetProvider: "claude", + settings: { alwaysPreserveClientCache: "never" }, + }), + false + ); + }); + + test("alwaysPreserveClientCache=auto uses automatic detection", () => { + // Should preserve for Claude client + caching provider + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + true + ); + + // Should NOT preserve for non-Claude client + assert.equal( + shouldPreserveCacheControl({ + userAgent: "curl/7.68.0", + isCombo: false, + targetProvider: "claude", + settings: { alwaysPreserveClientCache: "auto" }, + }), + false + ); + }); + + test("undefined settings uses automatic detection", () => { + assert.equal( + shouldPreserveCacheControl({ + userAgent: "claude-code/0.1.0", + isCombo: false, + targetProvider: "claude", + settings: undefined, + }), + true + ); + }); + }); + }); + + describe("trackCacheMetrics", () => { + test("initializes empty metrics", () => { + const result = trackCacheMetrics({ + preserved: true, + provider: "claude", + strategy: "priority", + metrics: undefined, + inputTokens: 1000, + cachedTokens: 500, + cacheCreationTokens: 200, + }); + + assert.equal(result.totalRequests, 1); + assert.equal(result.requestsWithCacheControl, 1); + assert.equal(result.totalInputTokens, 1000); + assert.equal(result.totalCachedTokens, 500); + assert.equal(result.totalCacheCreationTokens, 200); + assert.equal(result.tokensSaved, 500); + }); + + test("increments total requests without cache control", () => { + const metrics = { + totalRequests: 10, + requestsWithCacheControl: 5, + totalInputTokens: 5000, + totalCachedTokens: 2000, + totalCacheCreationTokens: 1000, + tokensSaved: 2000, + estimatedCostSaved: 0.5, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + + const result = trackCacheMetrics({ + preserved: false, + provider: "claude", + strategy: null, + metrics, + inputTokens: 500, + cachedTokens: 0, + cacheCreationTokens: 0, + }); + + assert.equal(result.totalRequests, 11); + assert.equal(result.requestsWithCacheControl, 5); // unchanged + assert.equal(result.totalInputTokens, 5500); + }); + + test("tracks requests with cache control preserved", () => { + const metrics = { + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + + const result = trackCacheMetrics({ + preserved: true, + provider: "claude", + strategy: "priority", + metrics, + inputTokens: 1000, + cachedTokens: 400, + cacheCreationTokens: 100, + }); + + assert.equal(result.totalRequests, 1); + assert.equal(result.requestsWithCacheControl, 1); + assert.equal(result.byProvider.claude.requests, 1); + assert.equal(result.byProvider.claude.inputTokens, 1000); + assert.equal(result.byProvider.claude.cachedTokens, 400); + assert.equal(result.byProvider.claude.cacheCreationTokens, 100); + assert.equal(result.byStrategy.priority.requests, 1); + }); + + test("tracks by provider", () => { + const metrics = { + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + + let result = trackCacheMetrics({ + preserved: true, + provider: "claude", + strategy: null, + metrics, + inputTokens: 1000, + cachedTokens: 300, + cacheCreationTokens: 100, + }); + + result = trackCacheMetrics({ + preserved: true, + provider: "zai", + strategy: null, + metrics: result, + inputTokens: 800, + cachedTokens: 200, + cacheCreationTokens: 50, + }); + + assert.equal(result.byProvider.claude.requests, 1); + assert.equal(result.byProvider.claude.inputTokens, 1000); + assert.equal(result.byProvider.claude.cachedTokens, 300); + assert.equal(result.byProvider.zai.requests, 1); + assert.equal(result.byProvider.zai.inputTokens, 800); + assert.equal(result.byProvider.zai.cachedTokens, 200); + }); + + test("tracks by strategy", () => { + const metrics = { + totalRequests: 0, + requestsWithCacheControl: 0, + totalInputTokens: 0, + totalCachedTokens: 0, + totalCacheCreationTokens: 0, + tokensSaved: 0, + estimatedCostSaved: 0, + byProvider: {}, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + + let result = trackCacheMetrics({ + preserved: true, + provider: "claude", + strategy: "priority", + metrics, + inputTokens: 1000, + cachedTokens: 300, + cacheCreationTokens: 100, + }); + + result = trackCacheMetrics({ + preserved: true, + provider: "claude", + strategy: "cost-optimized", + metrics: result, + inputTokens: 800, + cachedTokens: 200, + cacheCreationTokens: 50, + }); + + assert.equal(result.byStrategy.priority.requests, 1); + assert.equal(result.byStrategy.priority.cachedTokens, 300); + assert.equal(result.byStrategy["cost-optimized"].requests, 1); + assert.equal(result.byStrategy["cost-optimized"].cachedTokens, 200); + }); + }); + + describe("updateCacheTokenMetrics", () => { + test("updates token counts", () => { + const metrics = { + totalRequests: 10, + requestsWithCacheControl: 5, + totalInputTokens: 5000, + totalCachedTokens: 2000, + totalCacheCreationTokens: 1000, + tokensSaved: 2000, + estimatedCostSaved: 0.5, + byProvider: { + claude: { + requests: 3, + inputTokens: 3000, + cachedTokens: 1200, + cacheCreationTokens: 600, + }, + }, + byStrategy: { + priority: { + requests: 4, + inputTokens: 4000, + cachedTokens: 1600, + cacheCreationTokens: 800, + }, + }, + lastUpdated: new Date().toISOString(), + }; + + const result = updateCacheTokenMetrics({ + metrics, + provider: "claude", + strategy: "priority", + inputTokens: 1000, + cachedTokens: 400, + cacheCreationTokens: 200, + costSaved: 0.02, + }); + + assert.equal(result.totalInputTokens, 6000); + assert.equal(result.totalCachedTokens, 2400); + assert.equal(result.totalCacheCreationTokens, 1200); + assert.equal(result.tokensSaved, 2400); + assert.equal(result.estimatedCostSaved, 0.52); + }); + + test("updates provider breakdown", () => { + const metrics = { + totalRequests: 10, + requestsWithCacheControl: 5, + totalInputTokens: 5000, + totalCachedTokens: 2000, + totalCacheCreationTokens: 1000, + tokensSaved: 2000, + estimatedCostSaved: 0.5, + byProvider: { + claude: { + requests: 3, + inputTokens: 3000, + cachedTokens: 1200, + cacheCreationTokens: 600, + }, + }, + byStrategy: {}, + lastUpdated: new Date().toISOString(), + }; + + const result = updateCacheTokenMetrics({ + metrics, + provider: "claude", + strategy: null, + inputTokens: 500, + cachedTokens: 200, + cacheCreationTokens: 100, + }); + + assert.equal(result.byProvider.claude.inputTokens, 3500); + assert.equal(result.byProvider.claude.cachedTokens, 1400); + assert.equal(result.byProvider.claude.cacheCreationTokens, 700); + }); + + test("updates strategy breakdown", () => { + const metrics = { + totalRequests: 10, + requestsWithCacheControl: 5, + totalInputTokens: 5000, + totalCachedTokens: 2000, + totalCacheCreationTokens: 1000, + tokensSaved: 2000, + estimatedCostSaved: 0.5, + byProvider: {}, + byStrategy: { + priority: { + requests: 4, + inputTokens: 4000, + cachedTokens: 1600, + cacheCreationTokens: 800, + }, + }, + lastUpdated: new Date().toISOString(), + }; + + const result = updateCacheTokenMetrics({ + metrics, + provider: "claude", + strategy: "priority", + inputTokens: 500, + cachedTokens: 200, + cacheCreationTokens: 100, + }); + + assert.equal(result.byStrategy.priority.inputTokens, 4500); + assert.equal(result.byStrategy.priority.cachedTokens, 1800); + assert.equal(result.byStrategy.priority.cacheCreationTokens, 900); + }); + }); +}); diff --git a/tests/unit/cache-metrics.test.mjs b/tests/unit/cache-metrics.test.mjs new file mode 100644 index 0000000000..736b453d5f --- /dev/null +++ b/tests/unit/cache-metrics.test.mjs @@ -0,0 +1,134 @@ +import { describe, test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { getCacheMetrics } from "../../src/lib/db/settings.ts"; +import { getDbInstance } from "../../src/lib/db/core.ts"; + +describe("Cache Metrics Database", () => { + let db; + + before(() => { + db = getDbInstance(); + // Create usage_history table if it doesn't exist (mimicking production schema) + db.prepare( + ` + CREATE TABLE IF NOT EXISTS usage_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT, + model TEXT, + connection_id TEXT, + api_key_id TEXT, + api_key_name TEXT, + tokens_input INTEGER DEFAULT 0, + tokens_output INTEGER DEFAULT 0, + tokens_cache_read INTEGER DEFAULT 0, + tokens_cache_creation INTEGER DEFAULT 0, + tokens_reasoning INTEGER DEFAULT 0, + status TEXT, + timestamp TEXT, + success INTEGER, + latency_ms INTEGER DEFAULT 0, + ttft_ms INTEGER DEFAULT 0, + error_code TEXT + ) + ` + ).run(); + }); + + after(async () => { + // Clean up test data + db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run(); + }); + + describe("getCacheMetrics", () => { + test("returns metrics even with no cache activity", async () => { + // Verify the function works even if usage_history has data but no cache activity + const metrics = await getCacheMetrics(); + + assert.ok(metrics.totalRequests >= 0); + assert.ok(metrics.totalInputTokens >= 0); + assert.ok(metrics.totalCachedTokens >= 0); + assert.ok(metrics.totalCacheCreationTokens >= 0); + assert.ok(metrics.tokensSaved >= 0); + assert.ok(metrics.lastUpdated); + }); + + test("returns aggregated metrics from usage_history", async () => { + // Clean up any existing test data first + db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run(); + + const now = new Date().toISOString(); + + db.prepare( + ` + INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name, + tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning, + status, success, latency_ms, ttft_ms, error_code, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ).run( + "test-provider", + "test-model", + "test-connection", + "test-key-id", + "test-key", + 1000, // tokens_input + 500, // tokens_output + 400, // tokens_cache_read + 200, // tokens_cache_creation + 0, // tokens_reasoning + "200", // status + 1, // success + 100, // latency_ms + 50, // ttft_ms + null, // error_code + now // timestamp + ); + + // Insert another row + db.prepare( + ` + INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name, + tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning, + status, success, latency_ms, ttft_ms, error_code, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ).run( + "test-provider", + "test-model", + "test-connection", + "test-key-id", + "test-key", + 500, // tokens_input + 300, // tokens_output + 200, // tokens_cache_read + 100, // tokens_cache_creation + 0, // tokens_reasoning + "200", // status + 1, // success + 80, // latency_ms + 40, // ttft_ms + null, // error_code + now // timestamp + ); + + const metrics = await getCacheMetrics(); + + // Should have at least the 2 test requests with cache activity + assert.ok(metrics.requestsWithCacheControl >= 2); + assert.ok(metrics.totalInputTokens >= 1500); + assert.ok(metrics.totalCachedTokens >= 600); + assert.ok(metrics.totalCacheCreationTokens >= 300); + assert.ok(metrics.tokensSaved >= 600); + + // Check provider breakdown + assert.ok(metrics.byProvider["test-provider"]); + assert.ok(metrics.byProvider["test-provider"].requests >= 2); + assert.ok(metrics.byProvider["test-provider"].inputTokens >= 1500); + assert.ok(metrics.byProvider["test-provider"].cachedTokens >= 600); + assert.ok(metrics.byProvider["test-provider"].cacheCreationTokens >= 300); + + // Clean up + db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run(); + }); + }); +});