mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-31 04:12:10 +03:00
feat(api): prompt-cache health summary endpoint and analytics tab (#8827)
Adds GET /api/usage/cache-health and a Cache Health tab under /dashboard/analytics, both backed by a pure summary over the cache columns already present in call_logs. Motivated by a production diagnosis where the aggregate ratio was actively misleading. The window read 24.8M cached tokens and wrote 9.5M — a write/read of 0.385, which reads as merely mediocre. The actual shape was very different: the median call wrote 848 tokens while 18% of the calls carried 94% of every written token, and two models in the same window sat at 0.13 (Sonnet) and 0.51 (Opus). Averaging hid all three facts. So the summary reports what the average cannot: the distribution (p50/p90/p99), the concentration (how few calls carry how much of the write), and the per-model split. The heavy-write threshold is relative to the window (10x the median, floored at 1024) because a cutoff tuned for 130k-token conversations reports nothing at all on 2k-token ones; 1024 is the minimum Anthropic bills for cache creation, below which a write carries no signal. Calls that neither read nor wrote are counted separately from thrash — a route that does not cache is an absence of caching, not a sick cache — and only successful calls are summarized, since a 4xx/5xx never reached the provider cache and would dilute the ratio. Tests cover the summary (8) and the route (6, against a real SQLite so the WHERE clause itself is exercised), including that an internal failure answers 500 without leaking the stack trace, the SQL text or a table name.
This commit is contained in:
Diego Rodrigues de Sa e Souza
committed by
GitHub
GitHub
parent
c8f1d62de5
commit
aa85fa02bb
271
src/app/(dashboard)/dashboard/analytics/CacheHealthTab.tsx
Normal file
271
src/app/(dashboard)/dashboard/analytics/CacheHealthTab.tsx
Normal file
@@ -0,0 +1,271 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useTranslations } from "next-intl";
|
||||
import Card from "@/shared/components/Card";
|
||||
import Badge from "@/shared/components/Badge";
|
||||
import { Skeleton } from "@/shared/components/Loading";
|
||||
import TimeRangeSelector from "@/shared/components/analytics/TimeRangeSelector";
|
||||
import type { UtilizationTimeRange } from "@/shared/types/utilization";
|
||||
import { cn } from "@/shared/utils/cn";
|
||||
|
||||
type CacheHealthVerdict = "healthy" | "degraded" | "thrash" | "no-data";
|
||||
|
||||
type CacheHealthModel = {
|
||||
model: string;
|
||||
calls: number;
|
||||
cacheReadTotal: number;
|
||||
cacheWriteTotal: number;
|
||||
writeReadRatio: number;
|
||||
heavyWriteCalls: number;
|
||||
};
|
||||
|
||||
type CacheHealthResponse = {
|
||||
totalCalls: number;
|
||||
cacheReadTotal: number;
|
||||
cacheWriteTotal: number;
|
||||
writeReadRatio: number;
|
||||
warmCalls: number;
|
||||
coldCalls: number;
|
||||
rewriteCalls: number;
|
||||
uncachedCalls: number;
|
||||
writeP50: number;
|
||||
writeP90: number;
|
||||
writeP99: number;
|
||||
writeMax: number;
|
||||
heavyWriteCalls: number;
|
||||
heavyWriteCallShare: number;
|
||||
heavyWriteTokenShare: number;
|
||||
heavyWriteThreshold: number;
|
||||
verdict: CacheHealthVerdict;
|
||||
byModel: CacheHealthModel[];
|
||||
timeRange: UtilizationTimeRange;
|
||||
since: string;
|
||||
truncated: boolean;
|
||||
};
|
||||
|
||||
type Translator = ((key: string, values?: Record<string, unknown>) => string) & {
|
||||
has?: (key: string) => boolean;
|
||||
};
|
||||
|
||||
function text(t: Translator, key: string, fallback: string) {
|
||||
return typeof t.has === "function" && t.has(key) ? t(key) : fallback;
|
||||
}
|
||||
|
||||
const compact = (n: number) => n.toLocaleString(undefined, { maximumFractionDigits: 0 });
|
||||
const pct = (n: number) => `${(n * 100).toFixed(1)}%`;
|
||||
|
||||
function verdictVariant(v: CacheHealthVerdict) {
|
||||
if (v === "thrash") return "error" as const;
|
||||
if (v === "degraded") return "warning" as const;
|
||||
if (v === "healthy") return "success" as const;
|
||||
return "default" as const;
|
||||
}
|
||||
|
||||
function Metric({ label, value, hint }: { label: string; value: string; hint?: string }) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
<span className="text-xs uppercase tracking-wide text-text-muted">{label}</span>
|
||||
<span className="text-2xl font-semibold text-text-main">{value}</span>
|
||||
{hint ? <span className="text-xs text-text-muted">{hint}</span> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function CacheHealthTab() {
|
||||
const t = useTranslations("analytics") as Translator;
|
||||
const [range, setRange] = useState<UtilizationTimeRange>("24h");
|
||||
const [data, setData] = useState<CacheHealthResponse | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(async (r: UtilizationTimeRange) => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await fetch(`/api/usage/cache-health?range=${r}`);
|
||||
if (!res.ok) throw new Error(String(res.status));
|
||||
setData((await res.json()) as CacheHealthResponse);
|
||||
} catch {
|
||||
setError("load-failed");
|
||||
setData(null);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void load(range);
|
||||
}, [load, range]);
|
||||
|
||||
if (loading) return <Skeleton className="h-64 w-full" />;
|
||||
|
||||
if (error || !data) {
|
||||
return (
|
||||
<Card>
|
||||
<p className="text-sm text-text-muted">
|
||||
{text(t, "cacheHealthLoadError", "Could not load the prompt-cache summary.")}
|
||||
</p>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
const hasData = data.verdict !== "no-data";
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="text-lg font-semibold text-text-main">
|
||||
{text(t, "cacheHealthTitle", "Prompt cache health")}
|
||||
</h2>
|
||||
<Badge variant={verdictVariant(data.verdict)}>
|
||||
{text(t, `cacheHealthVerdict.${data.verdict}`, data.verdict)}
|
||||
</Badge>
|
||||
</div>
|
||||
<TimeRangeSelector value={range} onChange={setRange} />
|
||||
</div>
|
||||
|
||||
<p className="text-sm text-text-muted">
|
||||
{text(
|
||||
t,
|
||||
"cacheHealthIntro",
|
||||
"A warm loop writes the prefix once and then only pays the delta. When a few calls " +
|
||||
"carry most of the writes, the prefix is being rebuilt — usually because the client " +
|
||||
"rewrote its history or alternates between two request shapes."
|
||||
)}
|
||||
</p>
|
||||
|
||||
{!hasData ? (
|
||||
<Card>
|
||||
<p className="text-sm text-text-muted">
|
||||
{text(t, "cacheHealthEmpty", "No calls with cache accounting in this window.")}
|
||||
</p>
|
||||
</Card>
|
||||
) : (
|
||||
<>
|
||||
<Card>
|
||||
<div className="grid grid-cols-2 gap-6 md:grid-cols-4">
|
||||
<Metric
|
||||
label={text(t, "cacheHealthRatio", "Write / read")}
|
||||
value={data.writeReadRatio.toFixed(2)}
|
||||
hint={text(t, "cacheHealthRatioHint", "below 0.20 is a warm loop")}
|
||||
/>
|
||||
<Metric
|
||||
label={text(t, "cacheHealthRead", "Tokens read")}
|
||||
value={compact(data.cacheReadTotal)}
|
||||
/>
|
||||
<Metric
|
||||
label={text(t, "cacheHealthWrite", "Tokens written")}
|
||||
value={compact(data.cacheWriteTotal)}
|
||||
/>
|
||||
<Metric
|
||||
label={text(t, "cacheHealthCalls", "Calls")}
|
||||
value={compact(data.totalCalls)}
|
||||
hint={
|
||||
data.truncated
|
||||
? text(t, "cacheHealthTruncated", "newest 5000 only")
|
||||
: `${compact(data.warmCalls)} warm · ${compact(data.coldCalls)} cold`
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
{/* A concentração é o achado que a média esconde — por isso ganha destaque próprio. */}
|
||||
<Card>
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex flex-wrap items-baseline justify-between gap-2">
|
||||
<h3 className="text-sm font-semibold text-text-main">
|
||||
{text(t, "cacheHealthConcentration", "Where the writes are concentrated")}
|
||||
</h3>
|
||||
<span className="text-xs text-text-muted">
|
||||
{text(t, "cacheHealthThreshold", "outlier above")} {compact(data.heavyWriteThreshold)}{" "}
|
||||
{text(t, "cacheHealthTokens", "tokens")}
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-text-main">
|
||||
<strong>{pct(data.heavyWriteCallShare)}</strong>{" "}
|
||||
{text(t, "cacheHealthOfCalls", "of the calls carry")}{" "}
|
||||
<strong>{pct(data.heavyWriteTokenShare)}</strong>{" "}
|
||||
{text(t, "cacheHealthOfWrites", "of all written tokens")} (
|
||||
{compact(data.heavyWriteCalls)} {text(t, "cacheHealthCallsLower", "calls")}).
|
||||
</p>
|
||||
<div className="h-2 w-full overflow-hidden rounded-full bg-bg-subtle">
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full",
|
||||
data.heavyWriteTokenShare > 0.75 ? "bg-red-500" : "bg-amber-500"
|
||||
)}
|
||||
style={{ width: `${Math.min(100, data.heavyWriteTokenShare * 100)}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-4 pt-1 md:grid-cols-4">
|
||||
<Metric label="p50" value={compact(data.writeP50)} />
|
||||
<Metric label="p90" value={compact(data.writeP90)} />
|
||||
<Metric label="p99" value={compact(data.writeP99)} />
|
||||
<Metric label="max" value={compact(data.writeMax)} />
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<h3 className="mb-3 text-sm font-semibold text-text-main">
|
||||
{text(t, "cacheHealthByModel", "By model (worst ratio first)")}
|
||||
</h3>
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[560px] text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-left text-xs uppercase text-text-muted">
|
||||
<th className="pb-2 pr-4 font-medium">{text(t, "cacheHealthModel", "Model")}</th>
|
||||
<th className="pb-2 pr-4 text-right font-medium">
|
||||
{text(t, "cacheHealthCalls", "Calls")}
|
||||
</th>
|
||||
<th className="pb-2 pr-4 text-right font-medium">
|
||||
{text(t, "cacheHealthRead", "Read")}
|
||||
</th>
|
||||
<th className="pb-2 pr-4 text-right font-medium">
|
||||
{text(t, "cacheHealthWrite", "Written")}
|
||||
</th>
|
||||
<th className="pb-2 pr-4 text-right font-medium">W/R</th>
|
||||
<th className="pb-2 text-right font-medium">
|
||||
{text(t, "cacheHealthHeavy", "Heavy")}
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{data.byModel.map((m) => (
|
||||
<tr key={m.model} className="border-b border-border/50 last:border-0">
|
||||
<td className="py-2 pr-4 font-mono text-xs text-text-main">{m.model}</td>
|
||||
<td className="py-2 pr-4 text-right text-text-muted">{compact(m.calls)}</td>
|
||||
<td className="py-2 pr-4 text-right text-text-muted">
|
||||
{compact(m.cacheReadTotal)}
|
||||
</td>
|
||||
<td className="py-2 pr-4 text-right text-text-muted">
|
||||
{compact(m.cacheWriteTotal)}
|
||||
</td>
|
||||
<td
|
||||
className={cn(
|
||||
"py-2 pr-4 text-right font-medium",
|
||||
m.writeReadRatio > 1
|
||||
? "text-red-500"
|
||||
: m.writeReadRatio > 0.2
|
||||
? "text-amber-500"
|
||||
: "text-text-main"
|
||||
)}
|
||||
>
|
||||
{m.writeReadRatio.toFixed(2)}
|
||||
</td>
|
||||
<td className="py-2 text-right text-text-muted">
|
||||
{compact(m.heavyWriteCalls)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { useTranslations } from "next-intl";
|
||||
import { UsageAnalytics, CardSkeleton } from "@/shared/components";
|
||||
import { cn } from "@/shared/utils/cn";
|
||||
import EvalsTab from "../usage/components/EvalsTab";
|
||||
import CacheHealthTab from "./CacheHealthTab";
|
||||
import ComboHealthTab from "./ComboHealthTab";
|
||||
import ProviderUtilizationTab from "./ProviderUtilizationTab";
|
||||
import RouteExplainabilityTab from "./RouteExplainabilityTab";
|
||||
@@ -13,7 +14,13 @@ import SearchAnalyticsTab from "./SearchAnalyticsTab";
|
||||
import DiversityScoreCard from "./components/DiversityScoreCard";
|
||||
|
||||
type AnalyticsTab =
|
||||
"overview" | "evals" | "search" | "utilization" | "combo-health" | "route-trace";
|
||||
| "overview"
|
||||
| "evals"
|
||||
| "search"
|
||||
| "utilization"
|
||||
| "combo-health"
|
||||
| "cache-health"
|
||||
| "route-trace";
|
||||
|
||||
const ANALYTICS_TABS: Array<{
|
||||
id: AnalyticsTab;
|
||||
@@ -31,6 +38,12 @@ const ANALYTICS_TABS: Array<{
|
||||
label: "Combo Health",
|
||||
icon: "health_and_safety",
|
||||
},
|
||||
{
|
||||
id: "cache-health",
|
||||
labelKey: "cacheHealth",
|
||||
label: "Cache Health",
|
||||
icon: "database",
|
||||
},
|
||||
{ id: "route-trace", labelKey: "routeTrace", label: "Route Trace", icon: "alt_route" },
|
||||
];
|
||||
|
||||
@@ -44,7 +57,13 @@ function analyticsText(t: AnalyticsTranslator, key: string, fallback: string) {
|
||||
|
||||
function normalizeTab(tab: string | null): AnalyticsTab {
|
||||
if (tab === "route-trace" || tab === "route-explain") return "route-trace";
|
||||
if (tab === "evals" || tab === "search" || tab === "utilization" || tab === "combo-health") {
|
||||
if (
|
||||
tab === "evals" ||
|
||||
tab === "search" ||
|
||||
tab === "utilization" ||
|
||||
tab === "combo-health" ||
|
||||
tab === "cache-health"
|
||||
) {
|
||||
return tab;
|
||||
}
|
||||
return "overview";
|
||||
@@ -118,6 +137,7 @@ function AnalyticsPageContent() {
|
||||
{activeTab === "search" ? <SearchAnalyticsTab /> : null}
|
||||
{activeTab === "utilization" ? <ProviderUtilizationTab /> : null}
|
||||
{activeTab === "combo-health" ? <ComboHealthTab /> : null}
|
||||
{activeTab === "cache-health" ? <CacheHealthTab /> : null}
|
||||
{activeTab === "route-trace" ? (
|
||||
<RouteExplainabilityTab initialRequestId={initialRequestId} />
|
||||
) : null}
|
||||
|
||||
33
src/app/api/usage/cache-health/route.ts
Normal file
33
src/app/api/usage/cache-health/route.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { z } from "zod";
|
||||
|
||||
import { buildCacheHealthResponse } from "@/lib/usage/cacheHealth";
|
||||
|
||||
const querySchema = z.object({
|
||||
range: z.enum(["1h", "24h", "7d", "30d"]).default("24h"),
|
||||
model: z.string().min(1).max(200).optional(),
|
||||
});
|
||||
|
||||
export async function GET(request: Request) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const parsed = querySchema.safeParse({
|
||||
range: searchParams.get("range") ?? undefined,
|
||||
model: searchParams.get("model") || undefined,
|
||||
});
|
||||
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: parsed.error.issues[0]?.message ?? "Invalid query parameters" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json(buildCacheHealthResponse(parsed.data));
|
||||
} catch (error) {
|
||||
// Never echo the error: the message can carry the DATA_DIR path and the
|
||||
// SQL text. Details go to the server log, the caller gets a fixed string.
|
||||
console.error("Error building cache health summary:", error);
|
||||
return NextResponse.json({ error: "Failed to build cache health summary" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
290
src/lib/usage/cacheHealth.ts
Normal file
290
src/lib/usage/cacheHealth.ts
Normal file
@@ -0,0 +1,290 @@
|
||||
/**
|
||||
* Prompt-cache health — turns the raw cache columns of `call_logs` into a
|
||||
* summary an operator can act on.
|
||||
*
|
||||
* Why this exists: a production diagnosis on 2026-07-27 showed the aggregate
|
||||
* ratio is close to useless on its own. The window read 24.8M cached tokens and
|
||||
* wrote 9.5M (`write/read = 0.385`) — mediocre-looking but unremarkable. The
|
||||
* real story was the CONCENTRATION: 18% of the calls carried 94% of the write,
|
||||
* while the median call wrote 848 tokens. Two models in the same window sat at
|
||||
* 0.13 (Sonnet) and 0.51 (Opus).
|
||||
*
|
||||
* So the summary reports three things the average hides:
|
||||
* - the distribution (p50/p90/p99), because the median being cheap is exactly
|
||||
* what makes the tail invisible;
|
||||
* - the concentration (how few calls carry how much of the write), which is
|
||||
* what tells you whether to chase a pattern or accept the cost;
|
||||
* - the per-model split, because a healthy model averages away a sick one.
|
||||
*/
|
||||
|
||||
import { getDbInstance } from "@/lib/db/core";
|
||||
import type { UtilizationTimeRange } from "@/shared/types/utilization";
|
||||
|
||||
/** One `call_logs` row, already narrowed to the cache columns. Both counters are nullable in the schema. */
|
||||
export interface CacheHealthRow {
|
||||
model: string;
|
||||
cacheRead: number | null;
|
||||
cacheCreation: number | null;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface CacheHealthModelSummary {
|
||||
model: string;
|
||||
calls: number;
|
||||
cacheReadTotal: number;
|
||||
cacheWriteTotal: number;
|
||||
writeReadRatio: number;
|
||||
heavyWriteCalls: number;
|
||||
}
|
||||
|
||||
export type CacheHealthVerdict = "healthy" | "degraded" | "thrash" | "no-data";
|
||||
|
||||
export interface CacheHealthSummary {
|
||||
totalCalls: number;
|
||||
cacheReadTotal: number;
|
||||
cacheWriteTotal: number;
|
||||
/** `write / max(read, 1)`. Below ~0.2 is a warm loop; above 1 the prefix is being repaid. */
|
||||
writeReadRatio: number;
|
||||
/** Read something and read at least as much as it wrote — the loop is working. */
|
||||
warmCalls: number;
|
||||
/** Wrote without reading: a genuinely new prefix (first turn, or the cache expired). */
|
||||
coldCalls: number;
|
||||
/** Read, but rewrote more than it read — the prefix moved under it. */
|
||||
rewriteCalls: number;
|
||||
/** Neither read nor wrote: this route simply does not cache. Not a fault. */
|
||||
uncachedCalls: number;
|
||||
writeP50: number;
|
||||
writeP90: number;
|
||||
writeP99: number;
|
||||
writeMax: number;
|
||||
/** Calls whose write is an outlier for THIS window (see `heavyWriteThreshold`). */
|
||||
heavyWriteCalls: number;
|
||||
heavyWriteCallShare: number;
|
||||
/** Share of all written tokens those few calls account for. This is the number that matters. */
|
||||
heavyWriteTokenShare: number;
|
||||
heavyWriteThreshold: number;
|
||||
verdict: CacheHealthVerdict;
|
||||
byModel: CacheHealthModelSummary[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Anthropic pads cache-creation up to a 1024-token minimum (see the note on
|
||||
* issue #2215 in `claude-to-openai.ts`), so a write below that carries no
|
||||
* signal — it is the floor, not a decision. Used as the lower bound of the
|
||||
* outlier threshold so a window of tiny conversations cannot make everything
|
||||
* look "heavy".
|
||||
*/
|
||||
const MIN_MEANINGFUL_WRITE = 1024;
|
||||
|
||||
/**
|
||||
* How many times the median a write must be to count as an outlier. Relative on
|
||||
* purpose: a fixed cutoff tuned for 130k-token conversations reports nothing at
|
||||
* all on 2k-token ones, and the whole point is to find the tail of whatever
|
||||
* window the operator is looking at.
|
||||
*/
|
||||
const HEAVY_WRITE_MEDIAN_FACTOR = 10;
|
||||
|
||||
const num = (v: number | null | undefined): number =>
|
||||
typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 0;
|
||||
|
||||
/** Nearest-rank percentile over an ascending array. */
|
||||
function percentile(sortedAsc: number[], p: number): number {
|
||||
if (sortedAsc.length === 0) return 0;
|
||||
const idx = Math.min(sortedAsc.length - 1, Math.floor(sortedAsc.length * p));
|
||||
return sortedAsc[idx];
|
||||
}
|
||||
|
||||
function ratio(write: number, read: number): number {
|
||||
// max(read, 1) keeps this JSON-safe: a window that only ever wrote would divide
|
||||
// by zero, and Infinity does not survive JSON.stringify.
|
||||
return write / Math.max(read, 1);
|
||||
}
|
||||
|
||||
export function summarizeCacheHealth(rows: CacheHealthRow[]): CacheHealthSummary {
|
||||
const empty: CacheHealthSummary = {
|
||||
totalCalls: 0,
|
||||
cacheReadTotal: 0,
|
||||
cacheWriteTotal: 0,
|
||||
writeReadRatio: 0,
|
||||
warmCalls: 0,
|
||||
coldCalls: 0,
|
||||
rewriteCalls: 0,
|
||||
uncachedCalls: 0,
|
||||
writeP50: 0,
|
||||
writeP90: 0,
|
||||
writeP99: 0,
|
||||
writeMax: 0,
|
||||
heavyWriteCalls: 0,
|
||||
heavyWriteCallShare: 0,
|
||||
heavyWriteTokenShare: 0,
|
||||
heavyWriteThreshold: 0,
|
||||
verdict: "no-data",
|
||||
byModel: [],
|
||||
};
|
||||
if (rows.length === 0) return empty;
|
||||
|
||||
let readTotal = 0;
|
||||
let writeTotal = 0;
|
||||
let warm = 0;
|
||||
let cold = 0;
|
||||
let rewrite = 0;
|
||||
let uncached = 0;
|
||||
const writes: number[] = [];
|
||||
const perModel = new Map<string, { calls: number; read: number; write: number; heavy: number }>();
|
||||
|
||||
for (const r of rows) {
|
||||
const read = num(r.cacheRead);
|
||||
const write = num(r.cacheCreation);
|
||||
readTotal += read;
|
||||
writeTotal += write;
|
||||
writes.push(write);
|
||||
|
||||
if (read === 0 && write === 0) uncached++;
|
||||
else if (read === 0) cold++;
|
||||
else if (read >= write) warm++;
|
||||
else rewrite++;
|
||||
|
||||
const key = r.model || "unknown";
|
||||
const m = perModel.get(key) || { calls: 0, read: 0, write: 0, heavy: 0 };
|
||||
m.calls++;
|
||||
m.read += read;
|
||||
m.write += write;
|
||||
perModel.set(key, m);
|
||||
}
|
||||
|
||||
const sorted = [...writes].sort((a, b) => a - b);
|
||||
const median = percentile(sorted, 0.5);
|
||||
const heavyWriteThreshold = Math.max(median * HEAVY_WRITE_MEDIAN_FACTOR, MIN_MEANINGFUL_WRITE);
|
||||
|
||||
let heavyCalls = 0;
|
||||
let heavyTokens = 0;
|
||||
for (const r of rows) {
|
||||
const write = num(r.cacheCreation);
|
||||
if (write <= heavyWriteThreshold) continue;
|
||||
heavyCalls++;
|
||||
heavyTokens += write;
|
||||
const m = perModel.get(r.model || "unknown");
|
||||
if (m) m.heavy++;
|
||||
}
|
||||
|
||||
// Only calls that touched the cache at all get a say in the verdict — a route
|
||||
// that never caches is an absence of caching, not a sick cache.
|
||||
const cacheTouching = warm + cold + rewrite;
|
||||
const warmShare = cacheTouching > 0 ? warm / cacheTouching : 1;
|
||||
const verdict: CacheHealthVerdict =
|
||||
warmShare >= 0.6 ? "healthy" : warmShare >= 0.3 ? "degraded" : "thrash";
|
||||
|
||||
const byModel: CacheHealthModelSummary[] = [...perModel.entries()]
|
||||
.map(([model, m]) => ({
|
||||
model,
|
||||
calls: m.calls,
|
||||
cacheReadTotal: m.read,
|
||||
cacheWriteTotal: m.write,
|
||||
writeReadRatio: ratio(m.write, m.read),
|
||||
heavyWriteCalls: m.heavy,
|
||||
}))
|
||||
.sort((a, b) => b.writeReadRatio - a.writeReadRatio || b.cacheWriteTotal - a.cacheWriteTotal);
|
||||
|
||||
return {
|
||||
totalCalls: rows.length,
|
||||
cacheReadTotal: readTotal,
|
||||
cacheWriteTotal: writeTotal,
|
||||
writeReadRatio: ratio(writeTotal, readTotal),
|
||||
warmCalls: warm,
|
||||
coldCalls: cold,
|
||||
rewriteCalls: rewrite,
|
||||
uncachedCalls: uncached,
|
||||
writeP50: median,
|
||||
writeP90: percentile(sorted, 0.9),
|
||||
writeP99: percentile(sorted, 0.99),
|
||||
writeMax: sorted[sorted.length - 1] ?? 0,
|
||||
heavyWriteCalls: heavyCalls,
|
||||
heavyWriteCallShare: heavyCalls / rows.length,
|
||||
heavyWriteTokenShare: writeTotal > 0 ? heavyTokens / writeTotal : 0,
|
||||
heavyWriteThreshold,
|
||||
verdict,
|
||||
byModel,
|
||||
};
|
||||
}
|
||||
|
||||
const RANGE_MS: Record<UtilizationTimeRange, number> = {
|
||||
"1h": 60 * 60 * 1000,
|
||||
"24h": 24 * 60 * 60 * 1000,
|
||||
"7d": 7 * 24 * 60 * 60 * 1000,
|
||||
"30d": 30 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Cap on rows pulled into memory for one summary. A busy box logs thousands of
|
||||
* calls a day; the summary is statistical, so the newest N is representative
|
||||
* and bounded. `truncated` in the response tells the caller when this bit.
|
||||
*/
|
||||
const MAX_ROWS = 5000;
|
||||
|
||||
type CacheHealthDbRow = {
|
||||
model: string | null;
|
||||
requested_model: string | null;
|
||||
tokens_cache_read: number | null;
|
||||
tokens_cache_creation: number | null;
|
||||
timestamp: string | null;
|
||||
};
|
||||
|
||||
export interface CacheHealthResponse extends CacheHealthSummary {
|
||||
timeRange: UtilizationTimeRange;
|
||||
since: string;
|
||||
/** True when the window held more calls than `MAX_ROWS` and only the newest were summarized. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the cache columns of `call_logs` for the window and summarizes them.
|
||||
*
|
||||
* Only successful calls count: a 4xx/5xx never reached the provider cache, so
|
||||
* including them would dilute the ratio with requests that never had a chance
|
||||
* to hit. Rows where both counters are NULL are excluded at the SQL level —
|
||||
* those pre-date cache accounting and would show up as fake "uncached" calls.
|
||||
*/
|
||||
export function buildCacheHealthResponse(opts: {
|
||||
range: UtilizationTimeRange;
|
||||
model?: string;
|
||||
now?: number;
|
||||
}): CacheHealthResponse {
|
||||
const since = new Date((opts.now ?? Date.now()) - RANGE_MS[opts.range]).toISOString();
|
||||
const db = getDbInstance();
|
||||
|
||||
const params: (string | number)[] = [since];
|
||||
let modelFilter = "";
|
||||
if (opts.model) {
|
||||
modelFilter = " AND (model = ? OR requested_model = ?)";
|
||||
params.push(opts.model, opts.model);
|
||||
}
|
||||
params.push(MAX_ROWS + 1);
|
||||
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT model, requested_model, tokens_cache_read, tokens_cache_creation, timestamp
|
||||
FROM call_logs
|
||||
WHERE timestamp >= ?
|
||||
AND status = 200
|
||||
AND (tokens_cache_read IS NOT NULL OR tokens_cache_creation IS NOT NULL)
|
||||
${modelFilter}
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT ?`
|
||||
)
|
||||
.all(...params) as CacheHealthDbRow[];
|
||||
|
||||
const truncated = rows.length > MAX_ROWS;
|
||||
const summary = summarizeCacheHealth(
|
||||
rows.slice(0, MAX_ROWS).map((r) => ({
|
||||
// requested_model carries the alias the caller actually asked for
|
||||
// (e.g. a quota-share `qtSd/...` id); model is what the upstream saw.
|
||||
// The alias is the useful grouping key for an operator.
|
||||
model: r.requested_model || r.model || "unknown",
|
||||
cacheRead: r.tokens_cache_read,
|
||||
cacheCreation: r.tokens_cache_creation,
|
||||
timestamp: r.timestamp || "",
|
||||
}))
|
||||
);
|
||||
|
||||
return { ...summary, timeRange: opts.range, since, truncated };
|
||||
}
|
||||
123
tests/unit/usage-cache-health-route.test.ts
Normal file
123
tests/unit/usage-cache-health-route.test.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* Route contract for GET /api/usage/cache-health.
|
||||
*
|
||||
* Uses a real on-disk SQLite so the SQL itself is exercised — a summary that
|
||||
* silently returns zeros because the WHERE clause is wrong is worse than no
|
||||
* endpoint at all, and only a real query catches that.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-cache-health-"));
|
||||
process.env.DATA_DIR = tmpDir;
|
||||
|
||||
// Cleanup on exit, not in test.after(): a root after-hook fires as soon as the
|
||||
// first top-level test settles and would delete the directory out from under
|
||||
// the later ones.
|
||||
process.on("exit", () => {
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
});
|
||||
|
||||
const { getDbInstance, resetDbInstance } = await import("@/lib/db/core");
|
||||
const { buildCacheHealthResponse } = await import("@/lib/usage/cacheHealth");
|
||||
const { GET } = await import("@/app/api/usage/cache-health/route");
|
||||
|
||||
const NOW = Date.parse("2026-07-28T03:00:00.000Z");
|
||||
|
||||
function seed() {
|
||||
const db = getDbInstance();
|
||||
db.prepare("DELETE FROM call_logs").run();
|
||||
const insert = db.prepare(
|
||||
`INSERT INTO call_logs (id, timestamp, status, model, requested_model,
|
||||
tokens_cache_read, tokens_cache_creation)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
);
|
||||
const iso = (minsAgo: number) => new Date(NOW - minsAgo * 60_000).toISOString();
|
||||
|
||||
// dentro da janela de 1h: um write inicial + dois reads (padrão saudável)
|
||||
insert.run("a1", iso(10), 200, "claude-sonnet-5", "claude-sonnet-5", 0, 11660);
|
||||
insert.run("a2", iso(9), 200, "claude-sonnet-5", "claude-sonnet-5", 11660, 14);
|
||||
insert.run("a3", iso(8), 200, "claude-sonnet-5", "claude-sonnet-5", 11674, 15);
|
||||
// uma chamada cara do mesmo período, modelo diferente
|
||||
insert.run("a4", iso(7), 200, "claude-opus-4-8", "claude-opus-4-8", 37509, 91878);
|
||||
// ruído que NÃO pode entrar: erro, fora da janela, e linha sem contabilidade de cache
|
||||
insert.run("b1", iso(5), 500, "claude-sonnet-5", "claude-sonnet-5", 999999, 999999);
|
||||
insert.run("b2", iso(600), 200, "claude-sonnet-5", "claude-sonnet-5", 888888, 888888);
|
||||
insert.run("b3", iso(4), 200, "claude-sonnet-5", "claude-sonnet-5", null, null);
|
||||
}
|
||||
|
||||
test("resume a janela ignorando erro, fora-da-janela e linhas sem contabilidade", () => {
|
||||
seed();
|
||||
const r = buildCacheHealthResponse({ range: "1h", now: NOW });
|
||||
|
||||
assert.equal(r.totalCalls, 4, "só as 4 chamadas 200 dentro de 1h com colunas de cache");
|
||||
assert.equal(r.cacheReadTotal, 0 + 11660 + 11674 + 37509);
|
||||
assert.equal(r.cacheWriteTotal, 11660 + 14 + 15 + 91878);
|
||||
assert.equal(r.timeRange, "1h");
|
||||
assert.equal(r.truncated, false);
|
||||
assert.ok(
|
||||
!JSON.stringify(r).includes("999999") && !JSON.stringify(r).includes("888888"),
|
||||
"o 500 e a linha antiga não podem vazar para o resumo"
|
||||
);
|
||||
});
|
||||
|
||||
test("filtra por modelo quando pedido", () => {
|
||||
seed();
|
||||
const r = buildCacheHealthResponse({ range: "1h", model: "claude-opus-4-8", now: NOW });
|
||||
|
||||
assert.equal(r.totalCalls, 1);
|
||||
assert.equal(r.byModel.length, 1);
|
||||
assert.equal(r.byModel[0].model, "claude-opus-4-8");
|
||||
});
|
||||
|
||||
test("GET responde 200 com o resumo", async () => {
|
||||
seed();
|
||||
const res = await GET(new Request("http://localhost/api/usage/cache-health?range=1h"));
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const body = await res.json();
|
||||
assert.equal(body.timeRange, "1h");
|
||||
assert.equal(typeof body.writeReadRatio, "number");
|
||||
assert.ok(Array.isArray(body.byModel));
|
||||
assert.ok(["healthy", "degraded", "thrash", "no-data"].includes(body.verdict));
|
||||
});
|
||||
|
||||
test("GET rejeita range inválido com 400", async () => {
|
||||
const res = await GET(new Request("http://localhost/api/usage/cache-health?range=99y"));
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
|
||||
test("GET assume 24h quando o range é omitido", async () => {
|
||||
seed();
|
||||
const res = await GET(new Request("http://localhost/api/usage/cache-health"));
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal((await res.json()).timeRange, "24h");
|
||||
});
|
||||
|
||||
test("erro interno responde 500 sem vazar stack trace nem SQL", async () => {
|
||||
// Gatilho determinístico: sem a tabela, o prepare() lança SqliteError. Um
|
||||
// DATA_DIR inválido NÃO serve — o driver fica preso tentando criar o arquivo
|
||||
// em vez de falhar, e o teste passaria sem exercitar o catch.
|
||||
seed();
|
||||
const db = getDbInstance();
|
||||
db.prepare("DROP TABLE call_logs").run();
|
||||
try {
|
||||
const res = await GET(new Request("http://localhost/api/usage/cache-health?range=1h"));
|
||||
assert.equal(res.status, 500, "o caminho de erro precisa ser realmente exercitado");
|
||||
|
||||
const text = JSON.stringify(await res.json());
|
||||
assert.ok(!text.includes("at /"), "sem stack trace no corpo");
|
||||
assert.ok(!/\.ts:\d+/.test(text), "sem caminho de arquivo no corpo");
|
||||
assert.ok(!/SELECT|call_logs/i.test(text), "sem o texto do SQL nem nome de tabela no corpo");
|
||||
} finally {
|
||||
resetDbInstance();
|
||||
}
|
||||
});
|
||||
147
tests/unit/usage-cache-health.test.ts
Normal file
147
tests/unit/usage-cache-health.test.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Prompt-cache health summary.
|
||||
*
|
||||
* Context: production diagnosis on 2026-07-27 found that 18% of the Opus calls
|
||||
* carried 94% of all cache_creation tokens, while the median call wrote only
|
||||
* 848 tokens. A plain average hid this completely — `write/read = 0.385` looks
|
||||
* merely mediocre, and the aggregate says nothing about WHERE the waste is.
|
||||
* The summary therefore has to expose the concentration (heavy-write share),
|
||||
* not just the ratio, and split per model, because Sonnet sat at 0.13 while
|
||||
* Opus sat at 0.51 in the very same window.
|
||||
*
|
||||
* The numbers in these fixtures are the real shape observed on the box.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { summarizeCacheHealth, type CacheHealthRow } from "@/lib/usage/cacheHealth";
|
||||
|
||||
function row(
|
||||
model: string,
|
||||
cacheRead: number,
|
||||
cacheCreation: number,
|
||||
timestamp = "2026-07-28T02:00:00.000Z"
|
||||
): CacheHealthRow {
|
||||
return { model, cacheRead, cacheCreation, timestamp };
|
||||
}
|
||||
|
||||
test("uma conversa saudável: escreve o prefixo uma vez e depois só o delta", () => {
|
||||
const rows = [
|
||||
row("claude-sonnet-5", 0, 11660),
|
||||
row("claude-sonnet-5", 11660, 14),
|
||||
row("claude-sonnet-5", 11674, 15),
|
||||
row("claude-sonnet-5", 11689, 12),
|
||||
];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
|
||||
assert.equal(s.totalCalls, 4);
|
||||
assert.equal(s.cacheReadTotal, 35023);
|
||||
assert.equal(s.cacheWriteTotal, 11701);
|
||||
// O write inicial é inevitável; o que importa é que os turnos seguintes leem.
|
||||
assert.equal(s.warmCalls, 3);
|
||||
assert.equal(s.coldCalls, 1);
|
||||
assert.equal(s.verdict, "healthy");
|
||||
});
|
||||
|
||||
test("thrash: repaga o prefixo a cada turno em vez de ler", () => {
|
||||
const rows = [
|
||||
row("claude-opus-4-8", 0, 200000),
|
||||
row("claude-opus-4-8", 0, 205000),
|
||||
row("claude-opus-4-8", 0, 210000),
|
||||
];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
|
||||
assert.equal(s.verdict, "thrash");
|
||||
assert.ok(s.writeReadRatio > 1, "razão write/read acima de 1 é a assinatura do thrash");
|
||||
assert.equal(s.warmCalls, 0);
|
||||
});
|
||||
|
||||
test("expõe a CONCENTRAÇÃO do write, que a média esconde", () => {
|
||||
// 20 chamadas: 18 baratas + 2 caras. A média mente, a concentração não.
|
||||
const rows = [
|
||||
...Array.from({ length: 18 }, () => row("claude-opus-4-8", 127000, 800)),
|
||||
row("claude-opus-4-8", 37509, 91878),
|
||||
row("claude-opus-4-8", 37509, 87949),
|
||||
];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
|
||||
assert.equal(s.heavyWriteCalls, 2);
|
||||
assert.equal(s.heavyWriteCallShare, 0.1, "2 de 20 chamadas");
|
||||
assert.ok(
|
||||
s.heavyWriteTokenShare > 0.9,
|
||||
`essas 2 chamadas concentram >90% do write, não 10% (obtido: ${s.heavyWriteTokenShare})`
|
||||
);
|
||||
// A mediana continua barata — é exatamente por isso que o p50 sozinho engana.
|
||||
assert.equal(s.writeP50, 800);
|
||||
assert.ok(s.writeP90 > 50000, "o p90 é onde a dor aparece");
|
||||
});
|
||||
|
||||
test("quebra por modelo — Opus e Sonnet na mesma janela têm saúde diferente", () => {
|
||||
const rows = [
|
||||
...Array.from({ length: 8 }, () => row("claude-sonnet-5", 100000, 1000)),
|
||||
...Array.from({ length: 6 }, () => row("claude-opus-4-8", 100000, 1000)),
|
||||
...Array.from({ length: 2 }, () => row("claude-opus-4-8", 37509, 90000)),
|
||||
];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
const byModel = Object.fromEntries(s.byModel.map((m) => [m.model, m]));
|
||||
|
||||
assert.equal(s.byModel.length, 2);
|
||||
assert.ok(
|
||||
byModel["claude-opus-4-8"].writeReadRatio > byModel["claude-sonnet-5"].writeReadRatio,
|
||||
"o modelo problemático tem de se destacar no relatório"
|
||||
);
|
||||
// Ordenado pelo pior primeiro: é o que o operador precisa ver de cara.
|
||||
assert.equal(s.byModel[0].model, "claude-opus-4-8");
|
||||
});
|
||||
|
||||
test("chamadas sem cache nenhum são contadas à parte, não como thrash", () => {
|
||||
// Um modelo/rota que não cacheia não é um cache doente — é ausência de cache.
|
||||
const rows = [row("gpt-4o", 0, 0), row("gpt-4o", 0, 0), row("claude-sonnet-5", 9000, 20)];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
|
||||
assert.equal(s.uncachedCalls, 2);
|
||||
assert.equal(s.warmCalls, 1);
|
||||
assert.equal(s.verdict, "healthy", "ausência de cache não pode ser reportada como thrash");
|
||||
});
|
||||
|
||||
test("janela vazia não divide por zero nem inventa veredicto", () => {
|
||||
const s = summarizeCacheHealth([]);
|
||||
|
||||
assert.equal(s.totalCalls, 0);
|
||||
assert.equal(s.writeReadRatio, 0);
|
||||
assert.equal(s.heavyWriteTokenShare, 0);
|
||||
assert.equal(s.verdict, "no-data");
|
||||
assert.deepEqual(s.byModel, []);
|
||||
});
|
||||
|
||||
test("tolera linhas com null vindas do banco (colunas são nullable)", () => {
|
||||
const rows = [
|
||||
{ model: "claude-sonnet-5", cacheRead: null, cacheCreation: 5000, timestamp: "x" },
|
||||
{ model: "claude-sonnet-5", cacheRead: 5000, cacheCreation: null, timestamp: "x" },
|
||||
] as unknown as CacheHealthRow[];
|
||||
|
||||
const s = summarizeCacheHealth(rows);
|
||||
|
||||
assert.equal(s.cacheReadTotal, 5000);
|
||||
assert.equal(s.cacheWriteTotal, 5000);
|
||||
assert.equal(s.totalCalls, 2);
|
||||
});
|
||||
|
||||
test("o limiar de 'heavy' acompanha a janela em vez de ser um número mágico fixo", () => {
|
||||
// Conversas pequenas (cache de ~2k) não podem ser julgadas pelo mesmo corte
|
||||
// absoluto de uma conversa de 130k, senão nada é 'heavy' e o relatório cala.
|
||||
const pequenas = [
|
||||
...Array.from({ length: 9 }, () => row("m", 2000, 20)),
|
||||
row("m", 500, 1800),
|
||||
];
|
||||
|
||||
const s = summarizeCacheHealth(pequenas);
|
||||
|
||||
assert.equal(s.heavyWriteCalls, 1, "a chamada cara relativa à janela tem de ser detectada");
|
||||
});
|
||||
Reference in New Issue
Block a user