mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-05 23:02:10 +03:00
Merge pull request #752 from tombii/feat/preserve-client-cache-control
feat: preserve client cache_control with deterministic routing + metrics dashboard
This commit is contained in:
Diego Rodrigues de Sa e Souza
43
.github/workflows/sync-upstream.yml
vendored
Normal file
43
.github/workflows/sync-upstream.yml
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
name: Sync Upstream
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run every 6 hours
|
||||
- cron: '0 */6 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
name: Sync with upstream
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Fetch upstream
|
||||
run: |
|
||||
git remote add upstream https://github.com/diegosouzapw/OmniRoute.git || true
|
||||
git fetch upstream
|
||||
git fetch origin
|
||||
|
||||
- name: Sync main branch
|
||||
run: |
|
||||
git checkout main
|
||||
git merge upstream/main --no-edit || {
|
||||
echo "Merge conflict detected. Manual intervention required."
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Push changes
|
||||
run: git push https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/tombii/OmniRoute.git main
|
||||
@@ -42,6 +42,14 @@ import {
|
||||
getModelUpstreamExtraHeaders,
|
||||
} from "@/lib/localDb";
|
||||
import { getExecutor } from "../executors/index.ts";
|
||||
import { getCacheControlSettings } from "@/lib/cacheControlSettings";
|
||||
import {
|
||||
shouldPreserveCacheControl,
|
||||
trackCacheMetrics,
|
||||
recordCacheHit,
|
||||
type CacheControlMetrics,
|
||||
} from "../utils/cacheControlPolicy.ts";
|
||||
import { getCacheMetrics, updateCacheMetrics } from "@/lib/db/settings.ts";
|
||||
|
||||
import {
|
||||
parseCodexQuotaHeaders,
|
||||
@@ -306,6 +314,11 @@ function attachLogMeta(
|
||||
* @param {function} options.onDisconnect - Callback when client disconnects
|
||||
* @param {string} options.connectionId - Connection ID for usage tracking
|
||||
* @param {object} options.apiKeyInfo - API key metadata for usage attribution
|
||||
* @param {string} options.userAgent - Client user agent for caching decisions
|
||||
* @param {string} options.comboName - Combo name if this is a combo request
|
||||
* @param {string} options.comboStrategy - Combo routing strategy (e.g., 'priority', 'cost-optimized')
|
||||
* @param {boolean} options.isCombo - Whether this request is from a combo
|
||||
* @param {string} options.connectionId - Connection ID for settings lookup
|
||||
*/
|
||||
export async function handleChatCore({
|
||||
body,
|
||||
@@ -320,6 +333,8 @@ export async function handleChatCore({
|
||||
apiKeyInfo = null,
|
||||
userAgent,
|
||||
comboName,
|
||||
comboStrategy = null,
|
||||
isCombo = false,
|
||||
}) {
|
||||
let { provider, model, extendedContext } = modelInfo;
|
||||
const requestedModel =
|
||||
@@ -674,6 +689,46 @@ export async function handleChatCore({
|
||||
// Translate request (pass reqLogger for intermediate logging)
|
||||
let translatedBody = body;
|
||||
const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE && targetFormat === FORMATS.CLAUDE;
|
||||
|
||||
// Determine if we should preserve client-side cache_control headers
|
||||
// Fetch settings from DB to get user preference
|
||||
const cacheControlMode = await getCacheControlSettings().catch(() => "auto" as const);
|
||||
const preserveCacheControl = shouldPreserveCacheControl({
|
||||
userAgent,
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider: provider,
|
||||
settings: { alwaysPreserveClientCache: cacheControlMode },
|
||||
});
|
||||
|
||||
// Track cache metrics for this request
|
||||
let currentMetrics = await getCacheMetrics().catch(() => ({
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
}));
|
||||
|
||||
currentMetrics = trackCacheMetrics({
|
||||
preserved: preserveCacheControl,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
metrics: currentMetrics,
|
||||
});
|
||||
|
||||
if (preserveCacheControl) {
|
||||
log?.debug?.(
|
||||
"CACHE",
|
||||
`Preserving client cache_control (client=${userAgent?.substring(0, 20)}, combo=${isCombo}, strategy=${comboStrategy}, provider=${provider})`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
if (nativeCodexPassthrough) {
|
||||
translatedBody = { ...body, _nativeCodexPassthrough: true };
|
||||
@@ -701,7 +756,7 @@ export async function handleChatCore({
|
||||
credentials,
|
||||
provider,
|
||||
reqLogger,
|
||||
{ normalizeToolCallId, preserveDeveloperRole }
|
||||
{ normalizeToolCallId, preserveDeveloperRole, preserveCacheControl }
|
||||
);
|
||||
translatedBody = translateRequest(
|
||||
FORMATS.OPENAI,
|
||||
@@ -712,7 +767,7 @@ export async function handleChatCore({
|
||||
credentials,
|
||||
provider,
|
||||
reqLogger,
|
||||
{ normalizeToolCallId, preserveDeveloperRole }
|
||||
{ normalizeToolCallId, preserveDeveloperRole, preserveCacheControl }
|
||||
);
|
||||
log?.debug?.("FORMAT", "claude->openai->claude normalized passthrough");
|
||||
} else {
|
||||
@@ -816,7 +871,7 @@ export async function handleChatCore({
|
||||
credentials,
|
||||
provider,
|
||||
reqLogger,
|
||||
{ normalizeToolCallId, preserveDeveloperRole }
|
||||
{ normalizeToolCallId, preserveDeveloperRole, preserveCacheControl }
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -1406,6 +1461,30 @@ export async function handleChatCore({
|
||||
const msg = `[${new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit" })}] 📊 [USAGE] ${provider.toUpperCase()} | in=${getLoggedInputTokens(usage)} | out=${getLoggedOutputTokens(usage)}${connectionId ? ` | account=${connectionId.slice(0, 8)}...` : ""}`;
|
||||
console.log(`${COLORS.green}${msg}${COLORS.reset}`);
|
||||
|
||||
// Track cache token metrics
|
||||
const inputTokens = usage.prompt_tokens || 0;
|
||||
const cachedTokens = toPositiveNumber(
|
||||
usage.cache_read_input_tokens ??
|
||||
usage.cached_tokens ??
|
||||
(usage as any).prompt_tokens_details?.cached_tokens
|
||||
);
|
||||
const cacheCreationTokens = toPositiveNumber(
|
||||
usage.cache_creation_input_tokens ??
|
||||
(usage as any).prompt_tokens_details?.cache_creation_tokens
|
||||
);
|
||||
|
||||
if (cachedTokens > 0 || cacheCreationTokens > 0) {
|
||||
currentMetrics = updateCacheTokenMetrics({
|
||||
metrics: currentMetrics,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved: 0, // Will be calculated based on pricing
|
||||
});
|
||||
}
|
||||
|
||||
saveRequestUsage({
|
||||
provider: provider || "unknown",
|
||||
model: model || "unknown",
|
||||
@@ -1513,6 +1592,11 @@ export async function handleChatCore({
|
||||
claudeCacheUsageMeta: cacheUsageLogMeta,
|
||||
});
|
||||
|
||||
// Persist cache metrics to database
|
||||
updateCacheMetrics(currentMetrics).catch((err) => {
|
||||
log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`);
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
response: new Response(JSON.stringify(translatedResponse), {
|
||||
@@ -1551,6 +1635,33 @@ export async function handleChatCore({
|
||||
clientPayload,
|
||||
}) => {
|
||||
const cacheUsageLogMeta = buildCacheUsageLogMeta(streamUsage);
|
||||
|
||||
// Track cache token metrics for streaming responses
|
||||
if (streamUsage && typeof streamUsage === "object") {
|
||||
const inputTokens = streamUsage.prompt_tokens || 0;
|
||||
const cachedTokens = toPositiveNumber(
|
||||
streamUsage.cache_read_input_tokens ??
|
||||
streamUsage.cached_tokens ??
|
||||
(streamUsage as any).prompt_tokens_details?.cached_tokens
|
||||
);
|
||||
const cacheCreationTokens = toPositiveNumber(
|
||||
streamUsage.cache_creation_input_tokens ??
|
||||
(streamUsage as any).prompt_tokens_details?.cache_creation_tokens
|
||||
);
|
||||
|
||||
if (cachedTokens > 0 || cacheCreationTokens > 0) {
|
||||
currentMetrics = updateCacheTokenMetrics({
|
||||
metrics: currentMetrics,
|
||||
provider,
|
||||
strategy: comboStrategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
persistAttemptLogs({
|
||||
status: streamStatus || 200,
|
||||
tokens: streamUsage || {},
|
||||
@@ -1562,6 +1673,11 @@ export async function handleChatCore({
|
||||
claudeCacheUsageMeta: cacheUsageLogMeta,
|
||||
});
|
||||
|
||||
// Persist cache metrics to database
|
||||
updateCacheMetrics(currentMetrics).catch((err) => {
|
||||
log?.debug?.("CACHE", `Failed to persist cache metrics: ${err?.message || "unknown"}`);
|
||||
});
|
||||
|
||||
if (apiKeyInfo?.id && streamUsage) {
|
||||
calculateCost(provider, model, streamUsage)
|
||||
.then((estimatedCost) => {
|
||||
|
||||
@@ -73,6 +73,7 @@ function normalizeOpenAIResponsesRequest(body) {
|
||||
|
||||
/** @param options.normalizeToolCallId - When true, use 9-char tool call ids (e.g. Mistral); when false, leave ids as-is */
|
||||
/** @param options.preserveDeveloperRole - undefined/true: keep developer for OpenAI format (default); false: map to system */
|
||||
/** @param options.preserveCacheControl - When true, preserve client-side cache_control markers (for Claude Code, etc.) */
|
||||
// Translate request: source -> openai -> target
|
||||
export function translateRequest(
|
||||
sourceFormat,
|
||||
@@ -83,7 +84,7 @@ export function translateRequest(
|
||||
credentials = null,
|
||||
provider = null,
|
||||
reqLogger = null,
|
||||
options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean }
|
||||
options?: { normalizeToolCallId?: boolean; preserveDeveloperRole?: boolean; preserveCacheControl?: boolean }
|
||||
) {
|
||||
let result = body;
|
||||
const use9CharId = options?.normalizeToolCallId === true;
|
||||
@@ -149,10 +150,13 @@ export function translateRequest(
|
||||
}
|
||||
|
||||
// Final step: prepare request for Claude format endpoints
|
||||
// In Claude passthrough mode (Claude → Claude), preserve cache_control markers
|
||||
// Preserve cache_control when:
|
||||
// 1. Claude passthrough mode (Claude → Claude), OR
|
||||
// 2. Explicitly requested via options (for caching-aware clients like Claude Code)
|
||||
if (targetFormat === FORMATS.CLAUDE) {
|
||||
const isClaudePassthrough = sourceFormat === FORMATS.CLAUDE;
|
||||
result = prepareClaudeRequest(result, provider, isClaudePassthrough);
|
||||
const preserveCache = isClaudePassthrough || options?.preserveCacheControl === true;
|
||||
result = prepareClaudeRequest(result, provider, preserveCache);
|
||||
}
|
||||
|
||||
// Normalize openai-responses input shape for providers that require list input.
|
||||
|
||||
305
open-sse/utils/cacheControlPolicy.ts
Normal file
305
open-sse/utils/cacheControlPolicy.ts
Normal file
@@ -0,0 +1,305 @@
|
||||
/**
|
||||
* Cache Control Policy
|
||||
*
|
||||
* Determines when to preserve client-side prompt caching headers (cache_control)
|
||||
* vs. applying OmniRoute's own caching strategy.
|
||||
*
|
||||
* Client-side caching (e.g., Claude Code) should be preserved when:
|
||||
* 1. Client is Claude Code or similar caching-aware client
|
||||
* 2. Request will hit a deterministic target (single model or deterministic combo strategy)
|
||||
* 3. Provider supports prompt caching (Anthropic, Alibaba Qwen, etc.)
|
||||
*/
|
||||
|
||||
import type { RoutingStrategyValue } from "../../src/shared/constants/routingStrategies";
|
||||
|
||||
/**
|
||||
* Cache control preservation modes
|
||||
*/
|
||||
export type CacheControlMode = "auto" | "always" | "never";
|
||||
|
||||
/**
|
||||
* Cache control settings from the database
|
||||
*/
|
||||
export interface CacheControlSettings {
|
||||
alwaysPreserveClientCache?: CacheControlMode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache metrics for tracking effectiveness
|
||||
*/
|
||||
export interface CacheControlMetrics {
|
||||
// Totals
|
||||
totalRequests: number;
|
||||
requestsWithCacheControl: number;
|
||||
|
||||
// Token counts
|
||||
totalInputTokens: number;
|
||||
totalCachedTokens: number;
|
||||
totalCacheCreationTokens: number;
|
||||
|
||||
// Savings
|
||||
tokensSaved: number;
|
||||
estimatedCostSaved: number;
|
||||
|
||||
// Breakdowns
|
||||
byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Routing strategies that are deterministic (same request → same provider)
|
||||
*/
|
||||
const DETERMINISTIC_STRATEGIES: Set<RoutingStrategyValue> = new Set(["priority", "cost-optimized"]);
|
||||
|
||||
/**
|
||||
* Providers that support prompt caching
|
||||
*/
|
||||
const CACHING_PROVIDERS = new Set([
|
||||
"claude",
|
||||
"anthropic",
|
||||
"zai",
|
||||
"qwen", // Alibaba Qwen Coding Plan International
|
||||
]);
|
||||
|
||||
/**
|
||||
* Detect if the client is Claude Code or another caching-aware client
|
||||
*/
|
||||
export function isClaudeCodeClient(userAgent: string | null | undefined): boolean {
|
||||
if (!userAgent) return false;
|
||||
const ua = userAgent.toLowerCase();
|
||||
|
||||
// Claude Code user agents
|
||||
if (ua.includes("claude-code") || ua.includes("claude_code")) return true;
|
||||
if (ua.includes("anthropic") && ua.includes("cli")) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a provider supports prompt caching
|
||||
*/
|
||||
export function providerSupportsCaching(provider: string | null | undefined): boolean {
|
||||
if (!provider) return false;
|
||||
return CACHING_PROVIDERS.has(provider.toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a routing strategy is deterministic
|
||||
*/
|
||||
export function isDeterministicStrategy(
|
||||
strategy: RoutingStrategyValue | null | undefined
|
||||
): boolean {
|
||||
if (!strategy) return false;
|
||||
return DETERMINISTIC_STRATEGIES.has(strategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if client-side cache_control headers should be preserved
|
||||
*
|
||||
* @param userAgent - User-Agent header from the request
|
||||
* @param isCombo - Whether this is a combo model
|
||||
* @param comboStrategy - The combo's routing strategy (if applicable)
|
||||
* @param targetProvider - The target provider for the request
|
||||
* @param settings - Cache control settings from database (optional)
|
||||
* @returns true if cache_control should be preserved, false if OmniRoute should manage it
|
||||
*/
|
||||
export function shouldPreserveCacheControl({
|
||||
userAgent,
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider,
|
||||
settings,
|
||||
}: {
|
||||
userAgent: string | null | undefined;
|
||||
isCombo: boolean;
|
||||
comboStrategy?: RoutingStrategyValue | null;
|
||||
targetProvider: string | null | undefined;
|
||||
settings?: CacheControlSettings;
|
||||
}): boolean {
|
||||
// User override takes precedence
|
||||
if (settings?.alwaysPreserveClientCache === "always") {
|
||||
return true;
|
||||
}
|
||||
if (settings?.alwaysPreserveClientCache === "never") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Auto mode: use automatic detection (existing logic)
|
||||
// Must be a caching-aware client
|
||||
if (!isClaudeCodeClient(userAgent)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Target provider must support caching
|
||||
if (!providerSupportsCaching(targetProvider)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Single model: always preserve (deterministic)
|
||||
if (!isCombo) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Combo: only preserve if strategy is deterministic
|
||||
return isDeterministicStrategy(comboStrategy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Track cache control metrics for a request
|
||||
*/
|
||||
export function trackCacheMetrics({
|
||||
preserved,
|
||||
provider,
|
||||
strategy,
|
||||
metrics,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
}: {
|
||||
preserved: boolean;
|
||||
provider: string;
|
||||
strategy: string | null | undefined;
|
||||
metrics: CacheControlMetrics;
|
||||
inputTokens?: number;
|
||||
cachedTokens?: number;
|
||||
cacheCreationTokens?: number;
|
||||
}): CacheControlMetrics {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// Initialize metrics if empty
|
||||
if (!metrics) {
|
||||
metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: now,
|
||||
};
|
||||
}
|
||||
|
||||
// Increment total requests
|
||||
metrics.totalRequests++;
|
||||
|
||||
// Track token counts
|
||||
const input = inputTokens || 0;
|
||||
const cached = cachedTokens || 0;
|
||||
const creation = cacheCreationTokens || 0;
|
||||
|
||||
metrics.totalInputTokens += input;
|
||||
metrics.totalCachedTokens += cached;
|
||||
metrics.totalCacheCreationTokens += creation;
|
||||
|
||||
// Calculate tokens saved (cached tokens are reused, not charged)
|
||||
if (cached > 0) {
|
||||
metrics.tokensSaved += cached;
|
||||
}
|
||||
|
||||
// Only track requests where cache_control was preserved
|
||||
if (preserved) {
|
||||
metrics.requestsWithCacheControl++;
|
||||
|
||||
// Initialize provider tracking
|
||||
if (!metrics.byProvider[provider]) {
|
||||
metrics.byProvider[provider] = {
|
||||
requests: 0,
|
||||
inputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
};
|
||||
}
|
||||
metrics.byProvider[provider].requests++;
|
||||
metrics.byProvider[provider].inputTokens += input;
|
||||
metrics.byProvider[provider].cachedTokens += cached;
|
||||
metrics.byProvider[provider].cacheCreationTokens += creation;
|
||||
|
||||
// Initialize strategy tracking
|
||||
if (strategy && !metrics.byStrategy[strategy]) {
|
||||
metrics.byStrategy[strategy] = {
|
||||
requests: 0,
|
||||
inputTokens: 0,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
};
|
||||
}
|
||||
if (strategy) {
|
||||
metrics.byStrategy[strategy].requests++;
|
||||
metrics.byStrategy[strategy].inputTokens += input;
|
||||
metrics.byStrategy[strategy].cachedTokens += cached;
|
||||
metrics.byStrategy[strategy].cacheCreationTokens += creation;
|
||||
}
|
||||
}
|
||||
|
||||
metrics.lastUpdated = now;
|
||||
return metrics;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record cache token usage and update metrics
|
||||
*/
|
||||
export function updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider,
|
||||
strategy,
|
||||
inputTokens,
|
||||
cachedTokens,
|
||||
cacheCreationTokens,
|
||||
costSaved,
|
||||
}: {
|
||||
metrics: CacheControlMetrics;
|
||||
provider: string;
|
||||
strategy: string | null | undefined;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
costSaved?: number;
|
||||
}): CacheControlMetrics {
|
||||
metrics.totalCachedTokens += cachedTokens;
|
||||
metrics.totalCacheCreationTokens += cacheCreationTokens;
|
||||
metrics.totalInputTokens += inputTokens;
|
||||
|
||||
// Cached tokens are reused (saved), creation tokens are new cache writes
|
||||
metrics.tokensSaved += cachedTokens;
|
||||
if (costSaved !== undefined) {
|
||||
metrics.estimatedCostSaved += costSaved;
|
||||
}
|
||||
|
||||
// Update provider tracking
|
||||
if (metrics.byProvider[provider]) {
|
||||
metrics.byProvider[provider].cachedTokens += cachedTokens;
|
||||
metrics.byProvider[provider].cacheCreationTokens += cacheCreationTokens;
|
||||
metrics.byProvider[provider].inputTokens += inputTokens;
|
||||
}
|
||||
|
||||
// Update strategy tracking
|
||||
if (strategy && metrics.byStrategy[strategy]) {
|
||||
metrics.byStrategy[strategy].cachedTokens += cachedTokens;
|
||||
metrics.byStrategy[strategy].cacheCreationTokens += cacheCreationTokens;
|
||||
metrics.byStrategy[strategy].inputTokens += inputTokens;
|
||||
}
|
||||
|
||||
metrics.lastUpdated = new Date().toISOString();
|
||||
return metrics;
|
||||
}
|
||||
@@ -4,69 +4,190 @@ import { useState, useEffect } from "react";
|
||||
import { Card } from "@/shared/components";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
interface CacheMetrics {
|
||||
totalRequests: number;
|
||||
requestsWithCacheControl: number;
|
||||
totalInputTokens: number;
|
||||
totalCachedTokens: number;
|
||||
totalCacheCreationTokens: number;
|
||||
tokensSaved: number;
|
||||
estimatedCostSaved: number;
|
||||
byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
export default function CacheStatsCard() {
|
||||
const [cache, setCache] = useState(null);
|
||||
const [flushing, setFlushing] = useState(false);
|
||||
const [metrics, setMetrics] = useState<CacheMetrics | null>(null);
|
||||
const [resetting, setResetting] = useState(false);
|
||||
const t = useTranslations("settings");
|
||||
|
||||
const fetchStats = () => {
|
||||
fetch("/api/cache/stats")
|
||||
const fetchMetrics = () => {
|
||||
fetch("/api/settings/cache-metrics")
|
||||
.then((r) => r.json())
|
||||
.then(setCache)
|
||||
.then(setMetrics)
|
||||
.catch(() => {});
|
||||
};
|
||||
|
||||
useEffect(fetchStats, []);
|
||||
useEffect(fetchMetrics, []);
|
||||
|
||||
const handleFlush = async () => {
|
||||
setFlushing(true);
|
||||
const handleReset = async () => {
|
||||
setResetting(true);
|
||||
try {
|
||||
await fetch("/api/cache/stats", { method: "DELETE" });
|
||||
fetchStats();
|
||||
await fetch("/api/settings/cache-metrics", { method: "DELETE" });
|
||||
fetchMetrics();
|
||||
} finally {
|
||||
setFlushing(false);
|
||||
setResetting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const cacheHitRate =
|
||||
metrics && metrics.totalInputTokens > 0
|
||||
? (metrics.totalCachedTokens / metrics.totalInputTokens) * 100
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<Card className="p-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<h3 className="text-lg font-semibold text-text-main flex items-center gap-2">
|
||||
<span className="material-symbols-outlined text-[20px]">cached</span>
|
||||
{t("promptCache")}
|
||||
<span className="material-symbols-outlined text-[20px]">insights</span>
|
||||
Prompt Cache Metrics
|
||||
</h3>
|
||||
<button
|
||||
onClick={handleFlush}
|
||||
disabled={flushing}
|
||||
onClick={handleReset}
|
||||
disabled={resetting}
|
||||
className="px-3 py-1.5 text-xs rounded-lg bg-red-500/10 text-red-400 hover:bg-red-500/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
{flushing ? t("flushing") : t("flushCache")}
|
||||
{resetting ? "Resetting..." : "Reset Metrics"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{cache ? (
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">{t("size")}</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{cache.size}/{cache.maxSize}
|
||||
</p>
|
||||
{metrics ? (
|
||||
<div className="space-y-4">
|
||||
{/* Overview Stats */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Total Requests</p>
|
||||
<p className="font-mono text-lg text-text-main">{metrics.totalRequests}</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">With Cache Control</p>
|
||||
<p className="font-mono text-lg text-text-main">{metrics.requestsWithCacheControl}</p>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("hitRate")}</p>
|
||||
<p className="font-mono text-lg text-text-main">{cache.hitRate?.toFixed(1) ?? 0}%</p>
|
||||
|
||||
{/* Token Stats */}
|
||||
<div className="grid grid-cols-3 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Input Tokens</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{metrics.totalInputTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Cached Tokens (Read)</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.totalCachedTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Cache Creation (Write)</p>
|
||||
<p className="font-mono text-lg text-blue-400">
|
||||
{metrics.totalCacheCreationTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("hits")}</p>
|
||||
<p className="font-mono text-text-main">{cache.hits ?? 0}</p>
|
||||
|
||||
{/* Cache Ratio */}
|
||||
<div className="rounded-lg bg-surface/50 border border-border/30 p-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-text-main">Cache Reuse Ratio</p>
|
||||
<p className="text-xs text-text-muted">Cached tokens / Total input tokens</p>
|
||||
</div>
|
||||
<p className="font-mono text-xl text-green-400">{cacheHitRate.toFixed(1)}%</p>
|
||||
</div>
|
||||
{/* Progress bar */}
|
||||
<div className="mt-2 h-2 rounded-full bg-border/30 overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-green-500 transition-all duration-300"
|
||||
style={{ width: `${Math.min(cacheHitRate, 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("evictions")}</p>
|
||||
<p className="font-mono text-text-main">{cache.evictions ?? 0}</p>
|
||||
|
||||
{/* Savings */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">Tokens Saved</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.tokensSaved.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">Est. Cost Saved</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
${metrics.estimatedCostSaved.toFixed(4)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* By Provider */}
|
||||
{Object.keys(metrics.byProvider).length > 0 && (
|
||||
<div className="pt-3 border-t border-border/30">
|
||||
<p className="text-xs font-medium text-text-muted mb-2">By Provider</p>
|
||||
<div className="space-y-2">
|
||||
{Object.entries(metrics.byProvider).map(([provider, stats]) => {
|
||||
const providerCacheRate =
|
||||
stats.inputTokens > 0 ? (stats.cachedTokens / stats.inputTokens) * 100 : 0;
|
||||
return (
|
||||
<div
|
||||
key={provider}
|
||||
className="flex items-center justify-between px-3 py-2 rounded bg-surface/30 text-xs"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-text-main capitalize w-24">{provider}</span>
|
||||
<span className="text-text-muted">{stats.requests} reqs</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4 font-mono">
|
||||
<span className="text-text-muted" title="Input tokens">
|
||||
In: {stats.inputTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400" title="Cached tokens (reads)">
|
||||
Cached: {stats.cachedTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-blue-400" title="Cache creation tokens (writes)">
|
||||
Write: {stats.cacheCreationTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400 w-12 text-right">
|
||||
{providerCacheRate.toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-sm text-text-muted">{t("loadingCacheStats")}</p>
|
||||
<p className="text-sm text-text-muted">Loading cache metrics...</p>
|
||||
)}
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -19,7 +19,10 @@ const STRATEGIES = ROUTING_STRATEGIES.filter((strategy) =>
|
||||
}));
|
||||
|
||||
export default function RoutingTab() {
|
||||
const [settings, setSettings] = useState<any>({ fallbackStrategy: "fill-first" });
|
||||
const [settings, setSettings] = useState<any>({
|
||||
fallbackStrategy: "fill-first",
|
||||
alwaysPreserveClientCache: "auto",
|
||||
});
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [aliases, setAliases] = useState([]);
|
||||
const [newPattern, setNewPattern] = useState("");
|
||||
@@ -218,6 +221,74 @@ export default function RoutingTab() {
|
||||
|
||||
{/* Fallback Chains */}
|
||||
<FallbackChainsEditor />
|
||||
|
||||
{/* Client Cache Control */}
|
||||
<Card>
|
||||
<div className="flex items-center gap-3 mb-4">
|
||||
<div className="p-2 rounded-lg bg-green-500/10 text-green-500">
|
||||
<span className="material-symbols-outlined text-[20px]" aria-hidden="true">
|
||||
cached
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-lg font-semibold">Client Cache Control</h3>
|
||||
<p className="text-sm text-text-muted">
|
||||
Configure how client-side cache_control headers are handled
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
{[
|
||||
{
|
||||
value: "auto",
|
||||
label: "Auto (Recommended)",
|
||||
desc: "Preserve cache_control only for caching-aware clients (Claude Code) with deterministic routing",
|
||||
},
|
||||
{
|
||||
value: "always",
|
||||
label: "Always Preserve",
|
||||
desc: "Always forward client cache_control headers to upstream providers",
|
||||
},
|
||||
{
|
||||
value: "never",
|
||||
label: "Never Preserve",
|
||||
desc: "Always remove client cache_control headers, let OmniRoute manage caching",
|
||||
},
|
||||
].map((option) => (
|
||||
<button
|
||||
key={option.value}
|
||||
onClick={() => updateSetting({ alwaysPreserveClientCache: option.value })}
|
||||
disabled={loading}
|
||||
className={`w-full flex flex-col items-start gap-1 p-3 rounded-lg border text-left transition-all ${
|
||||
settings.alwaysPreserveClientCache === option.value
|
||||
? "border-green-500/50 bg-green-500/5 ring-1 ring-green-500/20"
|
||||
: "border-border/50 hover:border-border hover:bg-surface/30"
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className={`material-symbols-outlined text-[16px] ${
|
||||
settings.alwaysPreserveClientCache === option.value
|
||||
? "text-green-400"
|
||||
: "text-text-muted"
|
||||
}`}
|
||||
>
|
||||
{settings.alwaysPreserveClientCache === option.value
|
||||
? "check_circle"
|
||||
: "radio_button_unchecked"}
|
||||
</span>
|
||||
<span
|
||||
className={`text-sm font-medium ${settings.alwaysPreserveClientCache === option.value ? "text-green-400" : ""}`}
|
||||
>
|
||||
{option.label}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-xs text-text-muted ml-7">{option.desc}</p>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
22
src/app/api/settings/cache-metrics/route.ts
Normal file
22
src/app/api/settings/cache-metrics/route.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { getCacheMetrics, resetCacheMetrics } from "@/lib/db/settings";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const metrics = await getCacheMetrics();
|
||||
return NextResponse.json(metrics);
|
||||
} catch (error) {
|
||||
console.error("Error getting cache metrics:", error);
|
||||
return NextResponse.json({ error: "Failed to load cache metrics" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE() {
|
||||
try {
|
||||
const metrics = await resetCacheMetrics();
|
||||
return NextResponse.json(metrics);
|
||||
} catch (error) {
|
||||
console.error("Error resetting cache metrics:", error);
|
||||
return NextResponse.json({ error: "Failed to reset cache metrics" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,12 @@ export async function PATCH(request) {
|
||||
invalidateCallLogsMaxCache();
|
||||
}
|
||||
|
||||
// Sync cache control settings to runtime cache
|
||||
if ("alwaysPreserveClientCache" in body) {
|
||||
const { invalidateCacheControlSettingsCache } = await import("@/lib/cacheControlSettings");
|
||||
invalidateCacheControlSettingsCache();
|
||||
}
|
||||
|
||||
const { password, ...safeSettings } = settings;
|
||||
return NextResponse.json(safeSettings);
|
||||
} catch (error) {
|
||||
|
||||
25
src/lib/cacheControlSettings.ts
Normal file
25
src/lib/cacheControlSettings.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Cache Control Settings
|
||||
*
|
||||
* Provides cached access to cache control settings for performance.
|
||||
* Settings are fetched once and cached to avoid repeated DB hits.
|
||||
*/
|
||||
|
||||
import { getSettings } from "./db/settings";
|
||||
import type { CacheControlMode } from "@omniroute/open-sse/utils/cacheControlPolicy";
|
||||
|
||||
let cachedSettings: CacheControlMode | null = null;
|
||||
|
||||
export async function getCacheControlSettings(): Promise<CacheControlMode> {
|
||||
if (cachedSettings !== null) {
|
||||
return cachedSettings;
|
||||
}
|
||||
|
||||
const settings = await getSettings();
|
||||
cachedSettings = (settings.alwaysPreserveClientCache as CacheControlMode) || "auto";
|
||||
return cachedSettings;
|
||||
}
|
||||
|
||||
export function invalidateCacheControlSettingsCache() {
|
||||
cachedSettings = null;
|
||||
}
|
||||
@@ -46,6 +46,7 @@ export async function getSettings() {
|
||||
stickyRoundRobinLimit: 3,
|
||||
requireLogin: true,
|
||||
hiddenSidebarItems: [],
|
||||
alwaysPreserveClientCache: "auto",
|
||||
};
|
||||
for (const row of rows) {
|
||||
const record = toRecord(row);
|
||||
@@ -486,3 +487,177 @@ export async function setProxyConfig(config: Record<string, unknown>) {
|
||||
backupDbFile("pre-write");
|
||||
return current;
|
||||
}
|
||||
|
||||
// ──────────────── Cache Control Metrics ────────────────
|
||||
// Cache metrics are now computed from usage_history table on-the-fly
|
||||
// This avoids race conditions and keeps a single source of truth for token data
|
||||
|
||||
export async function getCacheMetrics() {
|
||||
const db = getDbInstance();
|
||||
|
||||
try {
|
||||
// Aggregate totals from usage_history
|
||||
const totalsRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
COUNT(*) as totalRequests,
|
||||
SUM(tokens_input) as totalInputTokens,
|
||||
SUM(tokens_cache_read) as totalCachedTokens,
|
||||
SUM(tokens_cache_creation) as totalCacheCreationTokens
|
||||
FROM usage_history
|
||||
WHERE tokens_cache_read > 0 OR tokens_cache_creation > 0
|
||||
`
|
||||
)
|
||||
.get() as
|
||||
| {
|
||||
totalRequests: number;
|
||||
totalInputTokens: number | null;
|
||||
totalCachedTokens: number | null;
|
||||
totalCacheCreationTokens: number | null;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
// Get all requests count (including those without cache activity)
|
||||
const allRequestsRow = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT COUNT(*) as totalRequests
|
||||
FROM usage_history
|
||||
`
|
||||
)
|
||||
.get() as { totalRequests: number } | undefined;
|
||||
|
||||
// Aggregate by provider
|
||||
const byProviderRows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
provider,
|
||||
COUNT(*) as requests,
|
||||
SUM(tokens_input) as inputTokens,
|
||||
SUM(tokens_cache_read) as cachedTokens,
|
||||
SUM(tokens_cache_creation) as cacheCreationTokens
|
||||
FROM usage_history
|
||||
WHERE (tokens_cache_read > 0 OR tokens_cache_creation > 0)
|
||||
AND provider IS NOT NULL
|
||||
GROUP BY provider
|
||||
`
|
||||
)
|
||||
.all() as Array<{
|
||||
provider: string;
|
||||
requests: number;
|
||||
inputTokens: number | null;
|
||||
cachedTokens: number | null;
|
||||
cacheCreationTokens: number | null;
|
||||
}>;
|
||||
|
||||
// Aggregate by strategy
|
||||
// Since combo_strategy isn't tracked in usage_history yet, we use 'direct' for all requests
|
||||
// TODO: Add combo_strategy column to usage_history for proper strategy tracking
|
||||
const byStrategyRows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT
|
||||
'direct' as strategy,
|
||||
COUNT(*) as requests,
|
||||
SUM(tokens_input) as inputTokens,
|
||||
SUM(tokens_cache_read) as cachedTokens,
|
||||
SUM(tokens_cache_creation) as cacheCreationTokens
|
||||
FROM usage_history
|
||||
WHERE (tokens_cache_read > 0 OR tokens_cache_creation > 0)
|
||||
GROUP BY 'direct'
|
||||
`
|
||||
)
|
||||
.all() as Array<{
|
||||
strategy: string;
|
||||
requests: number;
|
||||
inputTokens: number | null;
|
||||
cachedTokens: number | null;
|
||||
cacheCreationTokens: number | null;
|
||||
}>;
|
||||
|
||||
// Calculate tokens saved (cached tokens are reused, not charged at full price)
|
||||
const tokensSaved = totalsRow?.totalCachedTokens || 0;
|
||||
|
||||
// Build byProvider object
|
||||
const byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
> = {};
|
||||
for (const row of byProviderRows) {
|
||||
byProvider[row.provider] = {
|
||||
requests: row.requests,
|
||||
inputTokens: row.inputTokens || 0,
|
||||
cachedTokens: row.cachedTokens || 0,
|
||||
cacheCreationTokens: row.cacheCreationTokens || 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Build byStrategy object
|
||||
const byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
> = {};
|
||||
for (const row of byStrategyRows) {
|
||||
byStrategy[row.strategy] = {
|
||||
requests: row.requests,
|
||||
inputTokens: row.inputTokens || 0,
|
||||
cachedTokens: row.cachedTokens || 0,
|
||||
cacheCreationTokens: row.cacheCreationTokens || 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
totalRequests: allRequestsRow?.totalRequests || totalsRow?.totalRequests || 0,
|
||||
requestsWithCacheControl: totalsRow?.totalRequests || 0,
|
||||
totalInputTokens: totalsRow?.totalInputTokens || 0,
|
||||
totalCachedTokens: totalsRow?.totalCachedTokens || 0,
|
||||
totalCacheCreationTokens: totalsRow?.totalCacheCreationTokens || 0,
|
||||
tokensSaved,
|
||||
estimatedCostSaved: 0, // Would need pricing data to calculate
|
||||
byProvider,
|
||||
byStrategy,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch cache metrics from usage_history:", error);
|
||||
return {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateCacheMetrics(_metrics: Record<string, unknown>) {
|
||||
// No-op: metrics are now computed from usage_history on-the-fly
|
||||
// The usage_history table is the single source of truth
|
||||
return getCacheMetrics();
|
||||
}
|
||||
|
||||
export async function resetCacheMetrics() {
|
||||
// No-op: cannot delete historical usage data
|
||||
// Cache metrics are computed from usage_history, so they reflect actual request history
|
||||
console.warn(
|
||||
"resetCacheMetrics is deprecated - cache metrics are now computed from usage_history"
|
||||
);
|
||||
return getCacheMetrics();
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ export const updateSettingsSchema = z.object({
|
||||
cliCompatProviders: z.array(z.string().max(100)).optional(),
|
||||
// Strip provider/model prefix at proxy layer (e.g. "openai/gpt-4" → "gpt-4")
|
||||
stripModelPrefix: z.boolean().optional(),
|
||||
// Cache control preservation mode
|
||||
alwaysPreserveClientCache: z.enum(["auto", "always", "never"]).optional(),
|
||||
// Custom CLI agent definitions for ACP
|
||||
customAgents: z
|
||||
.array(
|
||||
|
||||
@@ -275,7 +275,7 @@ export async function handleChat(request: any, clientRawRequest: any = null) {
|
||||
handleSingleModel: (b: any, m: string) =>
|
||||
handleSingleModelChat(b, m, clientRawRequest, request, combo.name, apiKeyInfo, telemetry, {
|
||||
sessionId,
|
||||
}),
|
||||
}, combo.strategy, true),
|
||||
isModelAvailable: checkModelAvailable,
|
||||
log,
|
||||
settings,
|
||||
@@ -304,7 +304,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) {
|
||||
combo.name,
|
||||
apiKeyInfo,
|
||||
telemetry,
|
||||
{ sessionId, emergencyFallbackTried: true }
|
||||
{ sessionId, emergencyFallbackTried: true },
|
||||
combo.strategy,
|
||||
true
|
||||
);
|
||||
if (fallbackResponse.ok) {
|
||||
log.info("GLOBAL_FALLBACK", `Global fallback ${fallbackModel} succeeded`);
|
||||
@@ -336,7 +338,9 @@ export async function handleChat(request: any, clientRawRequest: any = null) {
|
||||
null,
|
||||
apiKeyInfo,
|
||||
telemetry,
|
||||
{ sessionId }
|
||||
{ sessionId },
|
||||
null,
|
||||
false
|
||||
);
|
||||
recordTelemetry(telemetry);
|
||||
return withSessionHeader(response, sessionId);
|
||||
@@ -366,7 +370,9 @@ async function handleSingleModelChat(
|
||||
comboName: string | null = null,
|
||||
apiKeyInfo: any = null,
|
||||
telemetry: any = null,
|
||||
runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {}
|
||||
runtimeOptions: { emergencyFallbackTried?: boolean; sessionId?: string | null } = {},
|
||||
comboStrategy: string | null = null,
|
||||
isCombo: boolean = false
|
||||
) {
|
||||
// 1. Resolve model → provider/model
|
||||
const resolved = await resolveModelOrError(modelStr, body, clientRawRequest?.endpoint);
|
||||
@@ -443,6 +449,8 @@ async function handleSingleModelChat(
|
||||
apiKeyInfo,
|
||||
userAgent,
|
||||
comboName,
|
||||
comboStrategy,
|
||||
isCombo,
|
||||
extendedContext,
|
||||
});
|
||||
if (telemetry) telemetry.endPhase();
|
||||
@@ -512,7 +520,9 @@ async function handleSingleModelChat(
|
||||
comboName,
|
||||
apiKeyInfo,
|
||||
telemetry,
|
||||
{ ...runtimeOptions, emergencyFallbackTried: true }
|
||||
{ ...runtimeOptions, emergencyFallbackTried: true },
|
||||
null, // no strategy for emergency fallback
|
||||
Boolean(comboName) // isCombo if comboName exists
|
||||
);
|
||||
|
||||
if (fallbackResponse.ok) {
|
||||
@@ -648,6 +658,8 @@ async function executeChatWithBreaker({
|
||||
apiKeyInfo,
|
||||
userAgent,
|
||||
comboName,
|
||||
comboStrategy,
|
||||
isCombo,
|
||||
extendedContext,
|
||||
}: any): Promise<{ result: any; tlsFingerprintUsed: boolean }> {
|
||||
let tlsFingerprintUsed = false;
|
||||
@@ -665,6 +677,8 @@ async function executeChatWithBreaker({
|
||||
apiKeyInfo,
|
||||
userAgent,
|
||||
comboName,
|
||||
comboStrategy,
|
||||
isCombo,
|
||||
onCredentialsRefreshed: async (newCreds: any) => {
|
||||
await updateProviderCredentials(credentials.connectionId, {
|
||||
accessToken: newCreds.accessToken,
|
||||
|
||||
598
tests/unit/cache-control-policy.test.mjs
Normal file
598
tests/unit/cache-control-policy.test.mjs
Normal file
@@ -0,0 +1,598 @@
|
||||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
isClaudeCodeClient,
|
||||
providerSupportsCaching,
|
||||
isDeterministicStrategy,
|
||||
shouldPreserveCacheControl,
|
||||
trackCacheMetrics,
|
||||
updateCacheTokenMetrics,
|
||||
} from "../../open-sse/utils/cacheControlPolicy.ts";
|
||||
|
||||
describe("Cache Control Policy", () => {
|
||||
describe("isClaudeCodeClient", () => {
|
||||
test("detects claude-code user agent", () => {
|
||||
assert.equal(isClaudeCodeClient("claude-code/0.1.0"), true);
|
||||
assert.equal(isClaudeCodeClient("claude_code/0.1.0"), true);
|
||||
assert.equal(isClaudeCodeClient("Anthropic CLI/1.0"), true);
|
||||
});
|
||||
|
||||
test("rejects non-Claude clients", () => {
|
||||
assert.equal(isClaudeCodeClient("curl/7.68.0"), false);
|
||||
assert.equal(isClaudeCodeClient("OpenAI/1.0"), false);
|
||||
assert.equal(isClaudeCodeClient(null), false);
|
||||
assert.equal(isClaudeCodeClient(undefined), false);
|
||||
assert.equal(isClaudeCodeClient(""), false);
|
||||
});
|
||||
|
||||
test("is case-insensitive", () => {
|
||||
assert.equal(isClaudeCodeClient("Claude-Code/0.1.0"), true);
|
||||
assert.equal(isClaudeCodeClient("CLAUDE-CODE/0.1.0"), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("providerSupportsCaching", () => {
|
||||
test("detects caching providers", () => {
|
||||
assert.equal(providerSupportsCaching("claude"), true);
|
||||
assert.equal(providerSupportsCaching("anthropic"), true);
|
||||
assert.equal(providerSupportsCaching("zai"), true);
|
||||
assert.equal(providerSupportsCaching("qwen"), true);
|
||||
});
|
||||
|
||||
test("rejects non-caching providers", () => {
|
||||
assert.equal(providerSupportsCaching("openai"), false);
|
||||
assert.equal(providerSupportsCaching("gemini"), false);
|
||||
assert.equal(providerSupportsCaching("unknown"), false);
|
||||
assert.equal(providerSupportsCaching(null), false);
|
||||
assert.equal(providerSupportsCaching(undefined), false);
|
||||
});
|
||||
|
||||
test("is case-insensitive", () => {
|
||||
assert.equal(providerSupportsCaching("Claude"), true);
|
||||
assert.equal(providerSupportsCaching("ANTHROPIC"), true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isDeterministicStrategy", () => {
|
||||
test("identifies deterministic strategies", () => {
|
||||
assert.equal(isDeterministicStrategy("priority"), true);
|
||||
assert.equal(isDeterministicStrategy("cost-optimized"), true);
|
||||
});
|
||||
|
||||
test("identifies non-deterministic strategies", () => {
|
||||
assert.equal(isDeterministicStrategy("weighted"), false);
|
||||
assert.equal(isDeterministicStrategy("round-robin"), false);
|
||||
assert.equal(isDeterministicStrategy("random"), false);
|
||||
assert.equal(isDeterministicStrategy("fill-first"), false);
|
||||
assert.equal(isDeterministicStrategy("p2c"), false);
|
||||
assert.equal(isDeterministicStrategy("least-used"), false);
|
||||
assert.equal(isDeterministicStrategy("strict-random"), false);
|
||||
});
|
||||
|
||||
test("handles null/undefined", () => {
|
||||
assert.equal(isDeterministicStrategy(null), false);
|
||||
assert.equal(isDeterministicStrategy(undefined), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldPreserveCacheControl", () => {
|
||||
test("preserves for single model + Claude client + caching provider", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves for combo with priority strategy + Claude client + caching provider", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "priority",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves for combo with cost-optimized strategy + Claude client + caching provider", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "cost-optimized",
|
||||
targetProvider: "anthropic",
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects non-Claude clients", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "curl/7.68.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects non-caching providers", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "openai",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with non-deterministic strategy (weighted)", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "weighted",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with non-deterministic strategy (round-robin)", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "round-robin",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with non-deterministic strategy (random)", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "random",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with fill-first strategy", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "fill-first",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with p2c strategy", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "p2c",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with least-used strategy", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "least-used",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with strict-random strategy", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: "strict-random",
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects combo with null strategy", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: true,
|
||||
comboStrategy: null,
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects when userAgent is null", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: null,
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects when targetProvider is null", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: null,
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
describe("settings override", () => {
|
||||
test("alwaysPreserveClientCache=always overrides auto detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "curl/7.68.0", // non-Claude client
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "always" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("alwaysPreserveClientCache=never overrides auto detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0", // Claude client
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "never" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("alwaysPreserveClientCache=auto uses automatic detection", () => {
|
||||
// Should preserve for Claude client + caching provider
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
|
||||
// Should NOT preserve for non-Claude client
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "curl/7.68.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("undefined settings uses automatic detection", () => {
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: "claude-code/0.1.0",
|
||||
isCombo: false,
|
||||
targetProvider: "claude",
|
||||
settings: undefined,
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("trackCacheMetrics", () => {
|
||||
test("initializes empty metrics", () => {
|
||||
const result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics: undefined,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 500,
|
||||
cacheCreationTokens: 200,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 1);
|
||||
assert.equal(result.requestsWithCacheControl, 1);
|
||||
assert.equal(result.totalInputTokens, 1000);
|
||||
assert.equal(result.totalCachedTokens, 500);
|
||||
assert.equal(result.totalCacheCreationTokens, 200);
|
||||
assert.equal(result.tokensSaved, 500);
|
||||
});
|
||||
|
||||
test("increments total requests without cache control", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = trackCacheMetrics({
|
||||
preserved: false,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
metrics,
|
||||
inputTokens: 500,
|
||||
cachedTokens: 0,
|
||||
cacheCreationTokens: 0,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 11);
|
||||
assert.equal(result.requestsWithCacheControl, 5); // unchanged
|
||||
assert.equal(result.totalInputTokens, 5500);
|
||||
});
|
||||
|
||||
test("tracks requests with cache control preserved", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 400,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.totalRequests, 1);
|
||||
assert.equal(result.requestsWithCacheControl, 1);
|
||||
assert.equal(result.byProvider.claude.requests, 1);
|
||||
assert.equal(result.byProvider.claude.inputTokens, 1000);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 400);
|
||||
assert.equal(result.byProvider.claude.cacheCreationTokens, 100);
|
||||
assert.equal(result.byStrategy.priority.requests, 1);
|
||||
});
|
||||
|
||||
test("tracks by provider", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
let result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 300,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "zai",
|
||||
strategy: null,
|
||||
metrics: result,
|
||||
inputTokens: 800,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 50,
|
||||
});
|
||||
|
||||
assert.equal(result.byProvider.claude.requests, 1);
|
||||
assert.equal(result.byProvider.claude.inputTokens, 1000);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 300);
|
||||
assert.equal(result.byProvider.zai.requests, 1);
|
||||
assert.equal(result.byProvider.zai.inputTokens, 800);
|
||||
assert.equal(result.byProvider.zai.cachedTokens, 200);
|
||||
});
|
||||
|
||||
test("tracks by strategy", () => {
|
||||
const metrics = {
|
||||
totalRequests: 0,
|
||||
requestsWithCacheControl: 0,
|
||||
totalInputTokens: 0,
|
||||
totalCachedTokens: 0,
|
||||
totalCacheCreationTokens: 0,
|
||||
tokensSaved: 0,
|
||||
estimatedCostSaved: 0,
|
||||
byProvider: {},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
let result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
metrics,
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 300,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
result = trackCacheMetrics({
|
||||
preserved: true,
|
||||
provider: "claude",
|
||||
strategy: "cost-optimized",
|
||||
metrics: result,
|
||||
inputTokens: 800,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 50,
|
||||
});
|
||||
|
||||
assert.equal(result.byStrategy.priority.requests, 1);
|
||||
assert.equal(result.byStrategy.priority.cachedTokens, 300);
|
||||
assert.equal(result.byStrategy["cost-optimized"].requests, 1);
|
||||
assert.equal(result.byStrategy["cost-optimized"].cachedTokens, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateCacheTokenMetrics", () => {
|
||||
test("updates token counts", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 3,
|
||||
inputTokens: 3000,
|
||||
cachedTokens: 1200,
|
||||
cacheCreationTokens: 600,
|
||||
},
|
||||
},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 4,
|
||||
inputTokens: 4000,
|
||||
cachedTokens: 1600,
|
||||
cacheCreationTokens: 800,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
inputTokens: 1000,
|
||||
cachedTokens: 400,
|
||||
cacheCreationTokens: 200,
|
||||
costSaved: 0.02,
|
||||
});
|
||||
|
||||
assert.equal(result.totalInputTokens, 6000);
|
||||
assert.equal(result.totalCachedTokens, 2400);
|
||||
assert.equal(result.totalCacheCreationTokens, 1200);
|
||||
assert.equal(result.tokensSaved, 2400);
|
||||
assert.equal(result.estimatedCostSaved, 0.52);
|
||||
});
|
||||
|
||||
test("updates provider breakdown", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {
|
||||
claude: {
|
||||
requests: 3,
|
||||
inputTokens: 3000,
|
||||
cachedTokens: 1200,
|
||||
cacheCreationTokens: 600,
|
||||
},
|
||||
},
|
||||
byStrategy: {},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: null,
|
||||
inputTokens: 500,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.byProvider.claude.inputTokens, 3500);
|
||||
assert.equal(result.byProvider.claude.cachedTokens, 1400);
|
||||
assert.equal(result.byProvider.claude.cacheCreationTokens, 700);
|
||||
});
|
||||
|
||||
test("updates strategy breakdown", () => {
|
||||
const metrics = {
|
||||
totalRequests: 10,
|
||||
requestsWithCacheControl: 5,
|
||||
totalInputTokens: 5000,
|
||||
totalCachedTokens: 2000,
|
||||
totalCacheCreationTokens: 1000,
|
||||
tokensSaved: 2000,
|
||||
estimatedCostSaved: 0.5,
|
||||
byProvider: {},
|
||||
byStrategy: {
|
||||
priority: {
|
||||
requests: 4,
|
||||
inputTokens: 4000,
|
||||
cachedTokens: 1600,
|
||||
cacheCreationTokens: 800,
|
||||
},
|
||||
},
|
||||
lastUpdated: new Date().toISOString(),
|
||||
};
|
||||
|
||||
const result = updateCacheTokenMetrics({
|
||||
metrics,
|
||||
provider: "claude",
|
||||
strategy: "priority",
|
||||
inputTokens: 500,
|
||||
cachedTokens: 200,
|
||||
cacheCreationTokens: 100,
|
||||
});
|
||||
|
||||
assert.equal(result.byStrategy.priority.inputTokens, 4500);
|
||||
assert.equal(result.byStrategy.priority.cachedTokens, 1800);
|
||||
assert.equal(result.byStrategy.priority.cacheCreationTokens, 900);
|
||||
});
|
||||
});
|
||||
});
|
||||
134
tests/unit/cache-metrics.test.mjs
Normal file
134
tests/unit/cache-metrics.test.mjs
Normal file
@@ -0,0 +1,134 @@
|
||||
import { describe, test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { getCacheMetrics } from "../../src/lib/db/settings.ts";
|
||||
import { getDbInstance } from "../../src/lib/db/core.ts";
|
||||
|
||||
describe("Cache Metrics Database", () => {
|
||||
let db;
|
||||
|
||||
before(() => {
|
||||
db = getDbInstance();
|
||||
// Create usage_history table if it doesn't exist (mimicking production schema)
|
||||
db.prepare(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS usage_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT,
|
||||
model TEXT,
|
||||
connection_id TEXT,
|
||||
api_key_id TEXT,
|
||||
api_key_name TEXT,
|
||||
tokens_input INTEGER DEFAULT 0,
|
||||
tokens_output INTEGER DEFAULT 0,
|
||||
tokens_cache_read INTEGER DEFAULT 0,
|
||||
tokens_cache_creation INTEGER DEFAULT 0,
|
||||
tokens_reasoning INTEGER DEFAULT 0,
|
||||
status TEXT,
|
||||
timestamp TEXT,
|
||||
success INTEGER,
|
||||
latency_ms INTEGER DEFAULT 0,
|
||||
ttft_ms INTEGER DEFAULT 0,
|
||||
error_code TEXT
|
||||
)
|
||||
`
|
||||
).run();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
// Clean up test data
|
||||
db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run();
|
||||
});
|
||||
|
||||
describe("getCacheMetrics", () => {
|
||||
test("returns metrics even with no cache activity", async () => {
|
||||
// Verify the function works even if usage_history has data but no cache activity
|
||||
const metrics = await getCacheMetrics();
|
||||
|
||||
assert.ok(metrics.totalRequests >= 0);
|
||||
assert.ok(metrics.totalInputTokens >= 0);
|
||||
assert.ok(metrics.totalCachedTokens >= 0);
|
||||
assert.ok(metrics.totalCacheCreationTokens >= 0);
|
||||
assert.ok(metrics.tokensSaved >= 0);
|
||||
assert.ok(metrics.lastUpdated);
|
||||
});
|
||||
|
||||
test("returns aggregated metrics from usage_history", async () => {
|
||||
// Clean up any existing test data first
|
||||
db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run();
|
||||
|
||||
const now = new Date().toISOString();
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name,
|
||||
tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning,
|
||||
status, success, latency_ms, ttft_ms, error_code, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
).run(
|
||||
"test-provider",
|
||||
"test-model",
|
||||
"test-connection",
|
||||
"test-key-id",
|
||||
"test-key",
|
||||
1000, // tokens_input
|
||||
500, // tokens_output
|
||||
400, // tokens_cache_read
|
||||
200, // tokens_cache_creation
|
||||
0, // tokens_reasoning
|
||||
"200", // status
|
||||
1, // success
|
||||
100, // latency_ms
|
||||
50, // ttft_ms
|
||||
null, // error_code
|
||||
now // timestamp
|
||||
);
|
||||
|
||||
// Insert another row
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name,
|
||||
tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning,
|
||||
status, success, latency_ms, ttft_ms, error_code, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
).run(
|
||||
"test-provider",
|
||||
"test-model",
|
||||
"test-connection",
|
||||
"test-key-id",
|
||||
"test-key",
|
||||
500, // tokens_input
|
||||
300, // tokens_output
|
||||
200, // tokens_cache_read
|
||||
100, // tokens_cache_creation
|
||||
0, // tokens_reasoning
|
||||
"200", // status
|
||||
1, // success
|
||||
80, // latency_ms
|
||||
40, // ttft_ms
|
||||
null, // error_code
|
||||
now // timestamp
|
||||
);
|
||||
|
||||
const metrics = await getCacheMetrics();
|
||||
|
||||
// Should have at least the 2 test requests with cache activity
|
||||
assert.ok(metrics.requestsWithCacheControl >= 2);
|
||||
assert.ok(metrics.totalInputTokens >= 1500);
|
||||
assert.ok(metrics.totalCachedTokens >= 600);
|
||||
assert.ok(metrics.totalCacheCreationTokens >= 300);
|
||||
assert.ok(metrics.tokensSaved >= 600);
|
||||
|
||||
// Check provider breakdown
|
||||
assert.ok(metrics.byProvider["test-provider"]);
|
||||
assert.ok(metrics.byProvider["test-provider"].requests >= 2);
|
||||
assert.ok(metrics.byProvider["test-provider"].inputTokens >= 1500);
|
||||
assert.ok(metrics.byProvider["test-provider"].cachedTokens >= 600);
|
||||
assert.ok(metrics.byProvider["test-provider"].cacheCreationTokens >= 300);
|
||||
|
||||
// Clean up
|
||||
db.prepare("DELETE FROM usage_history WHERE provider = 'test-provider'").run();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user