mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix: preserve client cache_control for all Claude-protocol providers
Previously, the cache control preservation logic only recognized a hardcoded list of providers (claude, anthropic, zai, qwen, deepseek). This caused OmniRoute to inject its own cache_control markers for Claude-protocol providers not in that list (bailian-coding-plan, glm, minimax, minimax-cn, etc.), overwriting the client's cache markers. The fix checks both: 1. Known caching providers list (existing behavior) 2. Whether targetFormat === 'claude' (all Claude-protocol providers) This ensures all Claude-compatible providers properly preserve client cache_control headers when appropriate (Claude Code client, deterministic routing, etc.). Also removes unused CacheStatsCard from settings/components (duplicate of the one in cache/ page). Fixes cache token calculation for GLM, Minimax, and other Claude-compatible providers. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -697,6 +697,7 @@ export async function handleChatCore({
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider: provider,
|
||||
targetFormat,
|
||||
settings: { alwaysPreserveClientCache: cacheControlMode },
|
||||
});
|
||||
|
||||
|
||||
@@ -90,10 +90,19 @@ export function isClaudeCodeClient(userAgent: string | null | undefined): boolea
|
||||
|
||||
/**
|
||||
* Check if a provider supports prompt caching
|
||||
* Supports caching if:
|
||||
* 1. Provider is in the known caching providers list, OR
|
||||
* 2. Provider uses Claude protocol (detected via targetFormat)
|
||||
*/
|
||||
export function providerSupportsCaching(provider: string | null | undefined): boolean {
|
||||
export function providerSupportsCaching(
|
||||
provider: string | null | undefined,
|
||||
targetFormat?: string | null
|
||||
): boolean {
|
||||
if (!provider) return false;
|
||||
return CACHING_PROVIDERS.has(provider.toLowerCase());
|
||||
if (CACHING_PROVIDERS.has(provider.toLowerCase())) return true;
|
||||
// All Claude-protocol providers support prompt caching
|
||||
if (targetFormat === "claude") return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -121,12 +130,14 @@ export function shouldPreserveCacheControl({
|
||||
isCombo,
|
||||
comboStrategy,
|
||||
targetProvider,
|
||||
targetFormat,
|
||||
settings,
|
||||
}: {
|
||||
userAgent: string | null | undefined;
|
||||
isCombo: boolean;
|
||||
comboStrategy?: RoutingStrategyValue | null;
|
||||
targetProvider: string | null | undefined;
|
||||
targetFormat?: string | null;
|
||||
settings?: CacheControlSettings;
|
||||
}): boolean {
|
||||
// User override takes precedence
|
||||
@@ -144,7 +155,7 @@ export function shouldPreserveCacheControl({
|
||||
}
|
||||
|
||||
// Target provider must support caching
|
||||
if (!providerSupportsCaching(targetProvider)) {
|
||||
if (!providerSupportsCaching(targetProvider, targetFormat)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import { Card } from "@/shared/components";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
interface CacheMetrics {
|
||||
totalRequests: number;
|
||||
requestsWithCacheControl: number;
|
||||
totalInputTokens: number;
|
||||
totalCachedTokens: number;
|
||||
totalCacheCreationTokens: number;
|
||||
tokensSaved: number;
|
||||
estimatedCostSaved: number;
|
||||
byProvider: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
byStrategy: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
inputTokens: number;
|
||||
cachedTokens: number;
|
||||
cacheCreationTokens: number;
|
||||
}
|
||||
>;
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
const REFRESH_INTERVAL_MS = 10_000;
|
||||
const REFRESH_INTERVAL_SECONDS = REFRESH_INTERVAL_MS / 1000;
|
||||
|
||||
export default function CacheStatsCard() {
|
||||
const [metrics, setMetrics] = useState<CacheMetrics | null>(null);
|
||||
const [resetting, setResetting] = useState(false);
|
||||
const t = useTranslations("cache");
|
||||
|
||||
const fetchMetrics = useCallback(() => {
|
||||
fetch("/api/settings/cache-metrics")
|
||||
.then((r) => r.json())
|
||||
.then(setMetrics)
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void fetchMetrics();
|
||||
const id = setInterval(() => void fetchMetrics(), REFRESH_INTERVAL_MS);
|
||||
return () => clearInterval(id);
|
||||
}, [fetchMetrics]);
|
||||
|
||||
const handleReset = async () => {
|
||||
setResetting(true);
|
||||
try {
|
||||
await fetch("/api/settings/cache-metrics", { method: "DELETE" });
|
||||
fetchMetrics();
|
||||
} finally {
|
||||
setResetting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const cacheHitRate =
|
||||
metrics && metrics.totalInputTokens > 0
|
||||
? (metrics.totalCachedTokens / metrics.totalInputTokens) * 100
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<div className="p-5 flex flex-col gap-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<span
|
||||
className="material-symbols-outlined text-base text-text-muted"
|
||||
aria-hidden="true"
|
||||
>
|
||||
insights
|
||||
</span>
|
||||
<h2 className="font-medium text-sm">{t("cacheMetrics")}</h2>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-xs text-text-muted">
|
||||
{t("autoRefresh", { seconds: REFRESH_INTERVAL_SECONDS })}
|
||||
</span>
|
||||
<button
|
||||
onClick={handleReset}
|
||||
disabled={resetting}
|
||||
className="px-3 py-1.5 text-xs rounded-lg bg-red-500/10 text-red-400 hover:bg-red-500/20 transition-colors disabled:opacity-50"
|
||||
>
|
||||
{resetting ? t("resetting") : t("resetMetrics")}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{metrics ? (
|
||||
<div className="flex flex-col gap-4">
|
||||
{/* Overview Stats */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">{t("totalRequests")}</p>
|
||||
<p className="font-mono text-lg text-text-main">{metrics.totalRequests}</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("withCacheControl")}</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{metrics.requestsWithCacheControl}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Token Stats */}
|
||||
<div className="grid grid-cols-3 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">{t("inputTokens")}</p>
|
||||
<p className="font-mono text-lg text-text-main">
|
||||
{metrics.totalInputTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("cachedTokensRead")}</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.totalCachedTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("cacheCreationWrite")}</p>
|
||||
<p className="font-mono text-lg text-blue-400">
|
||||
{metrics.totalCacheCreationTokens.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Cache Ratio */}
|
||||
<div className="rounded-lg bg-surface/50 border border-border/30 p-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-text-main">{t("cacheReuseRatio")}</p>
|
||||
<p className="text-xs text-text-muted">{t("cacheReuseRatioDesc")}</p>
|
||||
</div>
|
||||
<p className="font-mono text-xl text-green-400">{cacheHitRate.toFixed(1)}%</p>
|
||||
</div>
|
||||
{/* Progress bar */}
|
||||
<div className="mt-2 h-2 rounded-full bg-border/30 overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-green-500 transition-all duration-300"
|
||||
style={{ width: `${Math.min(cacheHitRate, 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Savings */}
|
||||
<div className="grid grid-cols-2 gap-4 text-sm">
|
||||
<div>
|
||||
<p className="text-text-muted">{t("tokensSaved")}</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
{metrics.tokensSaved.toLocaleString()}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<p className="text-text-muted">{t("estCostSaved")}</p>
|
||||
<p className="font-mono text-lg text-green-400">
|
||||
${metrics.estimatedCostSaved.toFixed(4)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* By Provider */}
|
||||
{Object.keys(metrics.byProvider).length > 0 && (
|
||||
<div className="pt-3 border-t border-border/30">
|
||||
<p className="text-xs font-medium text-text-muted mb-2">{t("byProvider")}</p>
|
||||
<div className="space-y-2">
|
||||
{Object.entries(metrics.byProvider).map(([provider, stats]) => {
|
||||
const providerCacheRate =
|
||||
stats.inputTokens > 0 ? (stats.cachedTokens / stats.inputTokens) * 100 : 0;
|
||||
return (
|
||||
<div
|
||||
key={provider}
|
||||
className="flex items-center justify-between px-3 py-2 rounded bg-surface/30 text-xs"
|
||||
>
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-text-main capitalize w-24">{provider}</span>
|
||||
<span className="text-text-muted">
|
||||
{stats.requests} {t("requestsShort")}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-4 font-mono">
|
||||
<span className="text-text-muted" title={t("inputTokens")}>
|
||||
{t("inputShort")}: {stats.inputTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400" title={t("cachedTokensRead")}>
|
||||
{t("cachedShort")}: {stats.cachedTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-blue-400" title={t("cacheCreationWrite")}>
|
||||
{t("writeShort")}: {stats.cacheCreationTokens.toLocaleString()}
|
||||
</span>
|
||||
<span className="text-green-400 w-12 text-right">
|
||||
{providerCacheRate.toFixed(0)}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-sm text-text-muted">{t("loading")}</p>
|
||||
)}
|
||||
</div>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
141
tests/unit/cache-control-claude-providers.test.mjs
Normal file
141
tests/unit/cache-control-claude-providers.test.mjs
Normal file
@@ -0,0 +1,141 @@
|
||||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
providerSupportsCaching,
|
||||
shouldPreserveCacheControl,
|
||||
} from "../../open-sse/utils/cacheControlPolicy.ts";
|
||||
|
||||
describe("Cache Control Policy - Claude Protocol Providers", () => {
|
||||
test("providerSupportsCaching returns true for Claude-format providers", () => {
|
||||
// Known caching providers
|
||||
assert.equal(providerSupportsCaching("claude", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("anthropic", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("zai", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("qwen", "openai"), true);
|
||||
assert.equal(providerSupportsCaching("deepseek", "openai"), true);
|
||||
|
||||
// Claude-protocol providers NOT in CACHING_PROVIDERS set
|
||||
// These should be detected via targetFormat
|
||||
assert.equal(providerSupportsCaching("bailian-coding-plan", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("glm", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("minimax", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("minimax-cn", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("kimi-coding", "claude"), true);
|
||||
assert.equal(providerSupportsCaching("alicode", "claude"), true);
|
||||
|
||||
// Non-Claude providers without caching support
|
||||
assert.equal(providerSupportsCaching("openai", "openai"), false);
|
||||
assert.equal(providerSupportsCaching("gemini", "gemini"), false);
|
||||
});
|
||||
|
||||
test("shouldPreserveCacheControl preserves for Claude-format providers with Claude Code client", () => {
|
||||
const claudeCodeUA = "Claude-Code/1.0.0";
|
||||
|
||||
// Claude-protocol providers should preserve cache_control
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "bailian-coding-plan",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "glm",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "zai",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "minimax",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldPreserveCacheControl respects user override 'always'", () => {
|
||||
const regularUA = "Mozilla/5.0";
|
||||
|
||||
// Even with non-Claude Code client, 'always' should preserve
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: regularUA,
|
||||
isCombo: false,
|
||||
targetProvider: "bailian-coding-plan",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "always" },
|
||||
}),
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldPreserveCacheControl respects user override 'never'", () => {
|
||||
const claudeCodeUA = "Claude-Code/1.0.0";
|
||||
|
||||
// Even with Claude Code client, 'never' should not preserve
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "bailian-coding-plan",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "never" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldPreserveCacheControl does not preserve for non-Claude Code clients in auto mode", () => {
|
||||
const regularUA = "Mozilla/5.0";
|
||||
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: regularUA,
|
||||
isCombo: false,
|
||||
targetProvider: "bailian-coding-plan",
|
||||
targetFormat: "claude",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
test("shouldPreserveCacheControl does not preserve for non-Claude format providers", () => {
|
||||
const claudeCodeUA = "Claude-Code/1.0.0";
|
||||
|
||||
assert.equal(
|
||||
shouldPreserveCacheControl({
|
||||
userAgent: claudeCodeUA,
|
||||
isCombo: false,
|
||||
targetProvider: "openai",
|
||||
targetFormat: "openai",
|
||||
settings: { alwaysPreserveClientCache: "auto" },
|
||||
}),
|
||||
false
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user