From 9d3eb480a22443bb4b4aec04c765088c32cc314e Mon Sep 17 00:00:00 2001 From: Jan Leon Date: Fri, 8 May 2026 11:16:14 +0000 Subject: [PATCH] feat(usage): account for codex fast tier analytics --- open-sse/handlers/chatCore.ts | 41 +++- .../dashboard/providers/[id]/page.tsx | 81 ++++---- .../providers/components/ProviderCard.tsx | 25 ++- src/app/api/usage/analytics/route.ts | 187 +++++++++++++++--- src/lib/db/core.ts | 2 + src/lib/db/migrations/001_initial_schema.sql | 2 + .../051_usage_history_service_tier.sql | 4 + src/lib/usage/costCalculator.ts | 49 ++++- src/lib/usage/usageHistory.ts | 13 +- src/lib/usage/usageStats.ts | 9 +- src/shared/components/Toggle.tsx | 6 +- src/shared/components/UsageAnalytics.tsx | 18 +- src/shared/components/analytics/charts.tsx | 64 ++++++ src/shared/components/analytics/index.tsx | 1 + tests/unit/usage-analytics-route.test.ts | 55 +++++- tests/unit/usage-analytics.test.ts | 16 ++ tests/unit/usage-history-db.test.ts | 28 +++ 17 files changed, 496 insertions(+), 105 deletions(-) create mode 100644 src/lib/db/migrations/051_usage_history_service_tier.sql diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 28a8c98882..3fdc212c58 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -82,6 +82,10 @@ import { import { getCacheMetrics } from "@/lib/db/settings.ts"; import { getCachedSettings } from "@/lib/db/readCache"; import { applyCodexGlobalFastServiceTier } from "@/lib/providers/codexFastTier"; +import { + getCodexRequestDefaults, + normalizeCodexServiceTier, +} from "@/lib/providers/requestDefaults"; import { cacheReasoningFromAssistantMessage } from "../services/reasoningCache.ts"; import { sanitizeOpenAITool } from "../services/toolSchemaSanitizer.ts"; @@ -981,6 +985,21 @@ export async function handleChatCore({ log?.info?.("STAGE_TRACE", `${traceId} ${label} t=${elapsed}ms${suffix}`); }; let tokensCompressed: number | null = null; + let effectiveServiceTier: "standard" | "priority" = "standard"; + const resolveEffectiveServiceTier = (requestBody?: unknown): "standard" | "priority" => { + if (provider !== "codex") return "standard"; + const requestRecord = + requestBody && typeof requestBody === "object" && !Array.isArray(requestBody) + ? (requestBody as Record) + : {}; + const rawServiceTier = requestRecord.service_tier; + if (typeof rawServiceTier === "string" && rawServiceTier.trim().length > 0) { + return normalizeCodexServiceTier(rawServiceTier) ? "priority" : "standard"; + } + return getCodexRequestDefaults(credentials?.providerSpecificData).serviceTier === "priority" + ? "priority" + : "standard"; + }; const persistFailureUsage = (statusCode: number, errorCode?: string | null) => { saveRequestUsage({ provider: provider || "unknown", @@ -995,6 +1014,7 @@ export async function handleChatCore({ connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, + serviceTier: effectiveServiceTier, }).catch(() => {}); }; @@ -1084,7 +1104,9 @@ export async function handleChatCore({ | undefined) : undefined; const idempotentCost = idempotentUsage - ? await calculateCost(provider, model, idempotentUsage as Record) + ? await calculateCost(provider, model, idempotentUsage as Record, { + serviceTier: effectiveServiceTier, + }) : 0; return { success: true, @@ -1404,6 +1426,7 @@ export async function handleChatCore({ : resolveStreamFlag(body?.stream, acceptHeader); const settings = await getCachedSettings(); credentials = applyCodexGlobalFastServiceTier(provider, credentials, settings); + effectiveServiceTier = resolveEffectiveServiceTier(body); setGeminiThoughtSignatureMode(settings.antigravitySignatureCacheMode); const semanticCacheEnabled = settings.semanticCacheEnabled !== false; @@ -1441,7 +1464,9 @@ export async function handleChatCore({ extractUsageFromResponse(cached as Record, provider) || ((cached as Record)?.usage as Record | undefined); const cachedCost = cachedUsage - ? await calculateCost(provider, model, cachedUsage as Record) + ? await calculateCost(provider, model, cachedUsage as Record, { + serviceTier: effectiveServiceTier, + }) : 0; persistAttemptLogs({ status: 200, @@ -1896,7 +1921,8 @@ export async function handleChatCore({ effectiveModel ?? "", { input: tokensSaved, - } + }, + { serviceTier: effectiveServiceTier } ); insertCompressionAnalyticsRow({ timestamp: new Date().toISOString(), @@ -2945,6 +2971,7 @@ export async function handleChatCore({ providerUrl = result.url; providerHeaders = result.headers; finalBody = result.transformedBody; + effectiveServiceTier = resolveEffectiveServiceTier(finalBody); claudePromptCacheLogMeta = buildClaudePromptCacheLogMeta( targetFormat, finalBody, @@ -3685,6 +3712,7 @@ export async function handleChatCore({ connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, + serviceTier: effectiveServiceTier, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); @@ -3817,7 +3845,9 @@ export async function handleChatCore({ (translatedResponse?.usage && typeof translatedResponse.usage === "object" ? translatedResponse.usage : null); - const estimatedCost = responseUsage ? await calculateCost(provider, model, responseUsage) : 0; + const estimatedCost = responseUsage + ? await calculateCost(provider, model, responseUsage, { serviceTier: effectiveServiceTier }) + : 0; if (postCallGuardrails.blocked) { const guardrailMessage = postCallGuardrails.message || "Response blocked by guardrail"; @@ -4047,6 +4077,7 @@ export async function handleChatCore({ connectionId: connectionId || undefined, apiKeyId: apiKeyInfo?.id || undefined, apiKeyName: apiKeyInfo?.name || undefined, + serviceTier: effectiveServiceTier, }).catch((err) => { console.error("Failed to save usage stats:", err.message); }); @@ -4065,7 +4096,7 @@ export async function handleChatCore({ }); if (apiKeyInfo?.id && streamUsage) { - calculateCost(provider, model, streamUsage) + calculateCost(provider, model, streamUsage, { serviceTier: effectiveServiceTier }) .then((estimatedCost) => { if (estimatedCost > 0) recordCost(apiKeyInfo.id, estimatedCost); }) diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx index bc514e4bd5..d0ca329f72 100644 --- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx @@ -2855,8 +2855,8 @@ export default function ProviderDetailPage() { {/* Connections */} {!isUpstreamProxyProvider && ( -
-
+
+

{t("connections")}

{providerId === "codex" && (
@@ -2865,8 +2865,9 @@ export default function ProviderDetailPage() { checked={codexGlobalFastServiceTier} onChange={handleToggleCodexGlobalFastServiceTier} disabled={savingCodexGlobalFastServiceTier} - label="Fast" - className="rounded-lg border border-border bg-bg-subtle px-2 py-1" + label="Fast default" + ariaLabel="Toggle Codex Fast default" + className="rounded-lg border border-sky-500/20 bg-sky-500/5 px-2 py-1" />
)} @@ -2898,42 +2899,44 @@ export default function ProviderDetailPage() { : t("providerProxy")}
- {connections.length > 1 && ( - - )} - {!isCompatible ? ( -
- - {providerId === "qoder" && ( - + )} + {!isCompatible ? ( + <> + - )} -
- ) : ( - connections.length === 0 && ( - - ) - )} + {providerId === "qoder" && ( + + )} + + ) : ( + connections.length === 0 && ( + + ) + )} +
{connections.length === 0 ? ( diff --git a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx index 9c609e231f..ed6c405d32 100644 --- a/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx +++ b/src/app/(dashboard)/dashboard/providers/components/ProviderCard.tsx @@ -57,7 +57,8 @@ function getStatusDisplay( connected: number, error: number, errorCode: string | null | undefined, - t: ReturnType + t: ReturnType, + afterConnected?: ReactNode ) { const parts: ReactNode[] = []; if (connected > 0) { @@ -66,6 +67,7 @@ function getStatusDisplay( {t("connected", { count: connected })} ); + if (afterConnected) parts.push(afterConnected); } if (error > 0) { const errText = errorCode @@ -98,6 +100,17 @@ export default function ProviderCard({ const isCompatible = isOpenAICompatibleProvider(providerId); const isCcCompatible = isClaudeCodeCompatibleProvider(providerId); const isAnthropicCompatible = isAnthropicCompatibleProvider(providerId) && !isCcCompatible; + const codexFastChip = + providerId === "codex" && stats.codexFastActive ? ( + + bolt + Fast + + ) : null; const dotLabels: Record = { free: tc("free"), @@ -184,7 +197,7 @@ export default function ProviderCard({ ) : ( <> - {getStatusDisplay(connected, error, stats.errorCode, t)} + {getStatusDisplay(connected, error, stats.errorCode, t, codexFastChip)} {(authType === "free" || provider.hasFree === true) && ( )} - {providerId === "codex" && stats.codexFastActive && ( - - - bolt - Fast - - - )} {isCompatible && ( {provider.apiType === "responses" ? t("responses") : t("chat")} diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts index 1eeec256b8..004fe0d967 100644 --- a/src/app/api/usage/analytics/route.ts +++ b/src/app/api/usage/analytics/route.ts @@ -35,7 +35,8 @@ const WEEKDAY_LABELS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; type PricingByProvider = Record>>; type ComputeCostFromPricing = ( pricing: Record | null | undefined, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options?: Record ) => number; function toNumber(value: unknown): number { @@ -55,6 +56,15 @@ function roundCost(value: number): number { return Math.round(value * 1_000_000) / 1_000_000; } +function normalizeServiceTier(value: unknown): "standard" | "priority" { + const tier = typeof value === "string" ? value.trim().toLowerCase() : ""; + return tier === "priority" || tier === "fast" ? "priority" : "standard"; +} + +function getServiceTierLabel(serviceTier: string): string { + return normalizeServiceTier(serviceTier) === "priority" ? "Fast" : "Standard"; +} + function appendWhereCondition(whereClause: string, condition: string): string { return whereClause ? `${whereClause} AND (${condition})` : `WHERE (${condition})`; } @@ -195,6 +205,7 @@ function computeUsageRowCost( const provider = toStringValue(row.provider); const model = toStringValue(row.model); if (!provider || !model) return 0; + const serviceTier = normalizeServiceTier(row.serviceTier ?? row.service_tier); const pricing = resolveModelPricing( pricingByProvider, @@ -205,13 +216,21 @@ function computeUsageRowCost( ); if (!pricing) return 0; - return computeCostFromPricing(pricing, { - input: toNumber(row.promptTokens), - output: toNumber(row.completionTokens), - cacheRead: toNumber(row.cacheReadTokens), - cacheCreation: toNumber(row.cacheCreationTokens), - reasoning: toNumber(row.reasoningTokens), - }); + return computeCostFromPricing( + pricing, + { + input: toNumber(row.promptTokens), + output: toNumber(row.completionTokens), + cacheRead: toNumber(row.cacheReadTokens), + cacheCreation: toNumber(row.cacheCreationTokens), + reasoning: toNumber(row.reasoningTokens), + }, + { + provider, + model, + serviceTier, + } + ); } function formatUtcDate(date: Date): string { @@ -331,6 +350,7 @@ export async function GET(request: Request) { DATE(timestamp) as date, LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, @@ -338,7 +358,7 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens FROM usage_history ${whereClause} - GROUP BY DATE(timestamp), LOWER(provider), LOWER(model) + GROUP BY DATE(timestamp), LOWER(provider), LOWER(model), serviceTier ORDER BY date ASC ` ) @@ -384,6 +404,7 @@ export async function GET(request: Request) { SELECT LOWER(model) as model, LOWER(provider) as provider, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COUNT(*) as requests, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, @@ -396,9 +417,8 @@ export async function GET(request: Request) { COALESCE(MAX(timestamp), '') as lastUsed FROM usage_history ${whereClause} - GROUP BY LOWER(model), LOWER(provider) + GROUP BY LOWER(model), LOWER(provider), serviceTier ORDER BY requests DESC - LIMIT 50 ` ) .all(params) as Array>; @@ -409,6 +429,7 @@ export async function GET(request: Request) { SELECT LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, @@ -416,7 +437,7 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens FROM usage_history ${whereClause} - GROUP BY LOWER(provider), LOWER(model) + GROUP BY LOWER(provider), LOWER(model), serviceTier ` ) .all(params) as Array>; @@ -447,6 +468,7 @@ export async function GET(request: Request) { COALESCE(NULLIF(c.display_name, ''), NULLIF(c.email, ''), NULLIF(c.name, ''), usage_history.connection_id, 'unknown') as account, LOWER(usage_history.provider) as provider, LOWER(usage_history.model) as model, + COALESCE(NULLIF(usage_history.service_tier, ''), 'standard') as serviceTier, COALESCE(SUM(usage_history.tokens_input), 0) as promptTokens, COALESCE(SUM(usage_history.tokens_output), 0) as completionTokens, COALESCE(SUM(usage_history.tokens_cache_read), 0) as cacheReadTokens, @@ -455,7 +477,7 @@ export async function GET(request: Request) { FROM usage_history LEFT JOIN provider_connections c ON c.id = usage_history.connection_id ${whereClause.replace(/timestamp/g, "usage_history.timestamp").replace(/api_key_/g, "usage_history.api_key_")} - GROUP BY account, LOWER(usage_history.provider), LOWER(usage_history.model) + GROUP BY account, LOWER(usage_history.provider), LOWER(usage_history.model), serviceTier ` ) .all(params) as Array>; @@ -493,6 +515,7 @@ export async function GET(request: Request) { COALESCE(NULLIF(api_key_name, ''), NULLIF(api_key_id, ''), 'Unknown API key') as apiKeyName, LOWER(provider) as provider, LOWER(model) as model, + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, COUNT(*) as requests, COALESCE(SUM(tokens_input), 0) as promptTokens, COALESCE(SUM(tokens_output), 0) as completionTokens, @@ -502,7 +525,28 @@ export async function GET(request: Request) { COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens FROM usage_history ${apiKeyWhereClause} - GROUP BY api_key_id, api_key_name, LOWER(provider), LOWER(model) + GROUP BY api_key_id, api_key_name, LOWER(provider), LOWER(model), serviceTier + ` + ) + .all(params) as Array>; + + const serviceTierRows = db + .prepare( + ` + SELECT + COALESCE(NULLIF(service_tier, ''), 'standard') as serviceTier, + LOWER(provider) as provider, + LOWER(model) as model, + COUNT(*) as requests, + COALESCE(SUM(tokens_input), 0) as promptTokens, + COALESCE(SUM(tokens_output), 0) as completionTokens, + COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens, + COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens, + COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens, + COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens + FROM usage_history + ${whereClause} + GROUP BY serviceTier, LOWER(provider), LOWER(model) ` ) .all(params) as Array>; @@ -584,6 +628,11 @@ export async function GET(request: Request) { firstRequest: summaryRow?.firstRequest || "", lastRequest: summaryRow?.lastRequest || "", fallbackCount: Number(fallbackRow?.fallbacks || 0), + fastRequests: 0, + standardRequests: 0, + fastCost: 0, + standardCost: 0, + fastRequestSharePct: 0, fallbackRatePct: Number(fallbackRow?.fallback_eligible || 0) > 0 ? Number( @@ -648,14 +697,11 @@ export async function GET(request: Request) { } summary.streak = computeActivityStreak(activityMap); - const byModel = modelRows.map((row) => { + const modelMap = new Map>(); + for (const row of modelRows) { const model = row.model as string; const provider = row.provider as string; const short = normalizeModelName(model); - const tokens = { - input: Number(row.promptTokens) || 0, - output: Number(row.completionTokens) || 0, - }; const cost = computeUsageRowCost( row, pricingByProvider, @@ -663,23 +709,59 @@ export async function GET(request: Request) { normalizeModelName, computeCostFromPricing ); - return { + const key = `${provider}::${model}`; + const existing = modelMap.get(key) || { model: short, provider, rawModel: model, + requests: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + latencyWeightedTotal: 0, + successfulRequests: 0, + lastUsed: "", + cost: 0, + }; + const requests = Number(row.requests) || 0; + existing.requests = Number(existing.requests || 0) + requests; + existing.promptTokens = Number(existing.promptTokens || 0) + Number(row.promptTokens || 0); + existing.completionTokens = + Number(existing.completionTokens || 0) + Number(row.completionTokens || 0); + existing.totalTokens = Number(existing.totalTokens || 0) + Number(row.totalTokens || 0); + existing.latencyWeightedTotal = + Number(existing.latencyWeightedTotal || 0) + Number(row.avgLatencyMs || 0) * requests; + existing.successfulRequests = + Number(existing.successfulRequests || 0) + Number(row.successfulRequests || 0); + if (!existing.lastUsed || String(row.lastUsed || "") > String(existing.lastUsed || "")) { + existing.lastUsed = row.lastUsed; + } + existing.cost = Number(existing.cost || 0) + cost; + modelMap.set(key, existing); + } + + const byModel = Array.from(modelMap.values()) + .map((row) => ({ + model: row.model, + provider: row.provider, + rawModel: row.rawModel, requests: Number(row.requests), - promptTokens: tokens.input, - completionTokens: tokens.output, + promptTokens: Number(row.promptTokens), + completionTokens: Number(row.completionTokens), totalTokens: Number(row.totalTokens), - avgLatencyMs: Math.round(Number(row.avgLatencyMs)), + avgLatencyMs: + Number(row.requests) > 0 + ? Math.round(Number(row.latencyWeightedTotal || 0) / Number(row.requests)) + : 0, successRatePct: Number(row.requests) > 0 - ? Number((Number(row.successfulRequests) / Number(row.requests)) * 100).toFixed(2) + ? Number((Number(row.successfulRequests || 0) / Number(row.requests)) * 100).toFixed(2) : 0, lastUsed: row.lastUsed, - cost: roundCost(cost), - }; - }); + cost: roundCost(Number(row.cost || 0)), + })) + .sort((left, right) => Number(right.requests) - Number(left.requests)) + .slice(0, 50); const totalCost = Array.from(dailyCostByDate.values()).reduce((sum, cost) => sum + cost, 0); summary.totalCost = roundCost(totalCost); @@ -781,6 +863,56 @@ export async function GET(request: Request) { .map((row) => ({ ...row, cost: roundCost(row.cost) })) .sort((left, right) => right.cost - left.cost); + const serviceTierMap = new Map< + string, + { + serviceTier: "standard" | "priority"; + label: string; + requests: number; + promptTokens: number; + completionTokens: number; + totalTokens: number; + cost: number; + } + >(); + for (const row of serviceTierRows) { + const serviceTier = normalizeServiceTier(row.serviceTier); + const existing = serviceTierMap.get(serviceTier) || { + serviceTier, + label: getServiceTierLabel(serviceTier), + requests: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + cost: 0, + }; + existing.requests += Number(row.requests || 0); + existing.promptTokens += Number(row.promptTokens || 0); + existing.completionTokens += Number(row.completionTokens || 0); + existing.totalTokens += Number(row.totalTokens || 0); + existing.cost += computeUsageRowCost( + row, + pricingByProvider, + PROVIDER_ID_TO_ALIAS, + normalizeModelName, + computeCostFromPricing + ); + serviceTierMap.set(serviceTier, existing); + } + const byServiceTier = Array.from(serviceTierMap.values()) + .map((row) => ({ ...row, cost: roundCost(row.cost) })) + .sort((left, right) => (left.serviceTier === "priority" ? -1 : 1)); + const fastTier = serviceTierMap.get("priority"); + const standardTier = serviceTierMap.get("standard"); + summary.fastRequests = fastTier?.requests || 0; + summary.fastCost = roundCost(fastTier?.cost || 0); + summary.standardRequests = standardTier?.requests || 0; + summary.standardCost = roundCost(standardTier?.cost || 0); + summary.fastRequestSharePct = + summary.totalRequests > 0 + ? Number(((Number(summary.fastRequests) / Number(summary.totalRequests)) * 100).toFixed(2)) + : 0; + const weeklyTokens = [0, 0, 0, 0, 0, 0, 0]; const weeklyCounts = [0, 0, 0, 0, 0, 0, 0]; const weeklyPattern = WEEKDAY_LABELS.map((day) => ({ @@ -816,6 +948,7 @@ export async function GET(request: Request) { byProvider, byApiKey, byAccount, + byServiceTier, weeklyPattern, weeklyTokens, weeklyCounts, diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 62d8a67d70..c9da9f0793 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -250,6 +250,7 @@ const SCHEMA_SQL = ` tokens_cache_read INTEGER DEFAULT 0, tokens_cache_creation INTEGER DEFAULT 0, tokens_reasoning INTEGER DEFAULT 0, + service_tier TEXT DEFAULT 'standard', status TEXT, success INTEGER DEFAULT 1, latency_ms INTEGER DEFAULT 0, @@ -260,6 +261,7 @@ const SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_uh_timestamp ON usage_history(timestamp); CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); + CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); CREATE TABLE IF NOT EXISTS call_logs ( id TEXT PRIMARY KEY, diff --git a/src/lib/db/migrations/001_initial_schema.sql b/src/lib/db/migrations/001_initial_schema.sql index 2cfc72eabe..d366cde3c8 100644 --- a/src/lib/db/migrations/001_initial_schema.sql +++ b/src/lib/db/migrations/001_initial_schema.sql @@ -97,6 +97,7 @@ CREATE TABLE IF NOT EXISTS usage_history ( tokens_cache_read INTEGER DEFAULT 0, tokens_cache_creation INTEGER DEFAULT 0, tokens_reasoning INTEGER DEFAULT 0, + service_tier TEXT DEFAULT 'standard', status TEXT, success INTEGER DEFAULT 1, latency_ms INTEGER DEFAULT 0, @@ -107,6 +108,7 @@ CREATE TABLE IF NOT EXISTS usage_history ( CREATE INDEX IF NOT EXISTS idx_uh_timestamp ON usage_history(timestamp); CREATE INDEX IF NOT EXISTS idx_uh_provider ON usage_history(provider); CREATE INDEX IF NOT EXISTS idx_uh_model ON usage_history(model); +CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); CREATE TABLE IF NOT EXISTS call_logs ( id TEXT PRIMARY KEY, diff --git a/src/lib/db/migrations/051_usage_history_service_tier.sql b/src/lib/db/migrations/051_usage_history_service_tier.sql new file mode 100644 index 0000000000..0f18aea384 --- /dev/null +++ b/src/lib/db/migrations/051_usage_history_service_tier.sql @@ -0,0 +1,4 @@ +-- Migration 051: Track effective service tier for usage analytics +-- Used to distinguish Codex Fast (priority) requests from standard requests. +ALTER TABLE usage_history ADD COLUMN service_tier TEXT DEFAULT 'standard'; +CREATE INDEX IF NOT EXISTS idx_uh_service_tier ON usage_history(service_tier); diff --git a/src/lib/usage/costCalculator.ts b/src/lib/usage/costCalculator.ts index a34010017b..31c2267e87 100644 --- a/src/lib/usage/costCalculator.ts +++ b/src/lib/usage/costCalculator.ts @@ -22,6 +22,12 @@ export function normalizeModelName(model: string): string { return parts[parts.length - 1]; } +export type CostCalculationOptions = { + provider?: string | null; + model?: string | null; + serviceTier?: string | null; +}; + function toNumber(value: unknown, fallback = 0): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { @@ -31,6 +37,35 @@ function toNumber(value: unknown, fallback = 0): number { return fallback; } +function normalizeServiceTier(value: unknown): string { + return typeof value === "string" ? value.trim().toLowerCase() : ""; +} + +function stripCodexEffortSuffix(model: string): string { + return model.replace(/-(?:xhigh|high|medium|low|none)$/i, ""); +} + +export function getCodexFastCostMultiplier( + provider: string | null | undefined, + model: string | null | undefined, + serviceTier: string | null | undefined +): number { + const providerKey = normalizeServiceTier(provider); + const tier = normalizeServiceTier(serviceTier); + if ( + (providerKey !== "codex" && providerKey !== "cx") || + (tier !== "priority" && tier !== "fast") + ) { + return 1; + } + + const modelKey = stripCodexEffortSuffix(normalizeModelName(String(model || "")).toLowerCase()); + const compactModelKey = modelKey.replace(/-/g, ""); + if (modelKey === "gpt-5.5" || compactModelKey === "gpt5.5") return 2.5; + if (modelKey === "gpt-5.4" || compactModelKey === "gpt5.4") return 2; + return 1; +} + /** * Calculate cost for a usage entry. * @@ -45,7 +80,8 @@ function toNumber(value: unknown, fallback = 0): number { */ export function computeCostFromPricing( pricing: Record | null | undefined, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options: CostCalculationOptions = {} ): number { if (!pricing || !tokens) return 0; const inputPrice = toNumber(pricing.input, 0); @@ -71,13 +107,14 @@ export function computeCostFromPricing( const cacheCreationTokens = tokens.cacheCreation ?? tokens.cache_creation_input_tokens ?? 0; if (cacheCreationTokens > 0) cost += cacheCreationTokens * (cacheCreationPrice / 1_000_000); - return cost; + return cost * getCodexFastCostMultiplier(options.provider, options.model, options.serviceTier); } export async function calculateCost( provider: string, model: string, - tokens: Record | null | undefined + tokens: Record | null | undefined, + options: CostCalculationOptions = {} ): Promise { if (!tokens || !provider || !model) return 0; @@ -98,7 +135,11 @@ export async function calculateCost( pricing && typeof pricing === "object" && !Array.isArray(pricing) ? (pricing as Record) : {}; - return computeCostFromPricing(pricingRecord, tokens); + return computeCostFromPricing(pricingRecord, tokens, { + provider, + model, + ...options, + }); } catch (error) { console.error("Error calculating cost:", error); return 0; diff --git a/src/lib/usage/usageHistory.ts b/src/lib/usage/usageHistory.ts index 5ee4c75b0b..fab526009e 100644 --- a/src/lib/usage/usageHistory.ts +++ b/src/lib/usage/usageHistory.ts @@ -44,6 +44,11 @@ function toStringOrNull(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value : null; } +function normalizeServiceTier(value: unknown): string { + const tier = typeof value === "string" ? value.trim().toLowerCase() : ""; + return tier === "priority" || tier === "fast" ? "priority" : "standard"; +} + function toNumber(value: unknown): number { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string" && value.trim().length > 0) { @@ -299,6 +304,7 @@ export async function getUsageDb(sinceIso?: string | null, limit?: number, curso connectionId: toStringOrNull(r.connection_id), apiKeyId: toStringOrNull(r.api_key_id), apiKeyName: toStringOrNull(r.api_key_name), + serviceTier: normalizeServiceTier(r.service_tier), tokens: { input: toNumber(r.tokens_input), output: toNumber(r.tokens_output), @@ -332,13 +338,14 @@ export async function saveRequestUsage(entry: any) { try { const db = getDbInstance(); const timestamp = entry.timestamp || new Date().toISOString(); + const serviceTier = normalizeServiceTier(entry.serviceTier ?? entry.service_tier); db.prepare( ` INSERT INTO usage_history (provider, model, connection_id, api_key_id, api_key_name, tokens_input, tokens_output, tokens_cache_read, tokens_cache_creation, tokens_reasoning, - status, success, latency_ms, ttft_ms, error_code, timestamp) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + service_tier, status, success, latency_ms, ttft_ms, error_code, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` ).run( entry.provider || null, @@ -351,6 +358,7 @@ export async function saveRequestUsage(entry: any) { getPromptCacheReadTokens(entry.tokens), getPromptCacheCreationTokens(entry.tokens), getReasoningTokens(entry.tokens), + serviceTier, entry.status || null, entry.success === false ? 0 : 1, Number.isFinite(Number(entry.latencyMs)) ? Number(entry.latencyMs) : 0, @@ -409,6 +417,7 @@ export async function getUsageHistory(filter: any = {}) { connectionId: toStringOrNull(r.connection_id), apiKeyId: toStringOrNull(r.api_key_id), apiKeyName: toStringOrNull(r.api_key_name), + serviceTier: normalizeServiceTier(r.service_tier), tokens: { input: toNumber(r.tokens_input), output: toNumber(r.tokens_output), diff --git a/src/lib/usage/usageStats.ts b/src/lib/usage/usageStats.ts index 1138ae605b..1a3081712f 100644 --- a/src/lib/usage/usageStats.ts +++ b/src/lib/usage/usageStats.ts @@ -81,7 +81,8 @@ export async function getUsageStats() { tokens_output, tokens_cache_read, tokens_cache_creation, - tokens_reasoning + tokens_reasoning, + service_tier FROM usage_history WHERE DATE(timestamp) >= ? @@ -98,7 +99,8 @@ export async function getUsageStats() { total_output_tokens as tokens_output, 0 as tokens_cache_read, 0 as tokens_cache_creation, - 0 as tokens_reasoning + 0 as tokens_reasoning, + 'standard' as service_tier FROM daily_usage_summary WHERE date < ? @@ -193,6 +195,7 @@ export async function getUsageStats() { const connectionId = toStringOrEmpty(row.connection_id) || null; const apiKeyId = toStringOrEmpty(row.api_key_id) || null; const apiKeyName = toStringOrEmpty(row.api_key_name) || null; + const serviceTier = toStringOrEmpty(row.service_tier) || "standard"; const promptTokens = toNumber(row.tokens_input); const completionTokens = toNumber(row.tokens_output); @@ -205,7 +208,7 @@ export async function getUsageStats() { cacheCreation: toNumber(row.tokens_cache_creation), reasoning: toNumber(row.tokens_reasoning), }; - const entryCost = await calculateCost(provider, model, entryTokens); + const entryCost = await calculateCost(provider, model, entryTokens, { serviceTier }); stats.totalPromptTokens += promptTokens; stats.totalCompletionTokens += completionTokens; diff --git a/src/shared/components/Toggle.tsx b/src/shared/components/Toggle.tsx index 66cff8c905..e8b0e79cbf 100644 --- a/src/shared/components/Toggle.tsx +++ b/src/shared/components/Toggle.tsx @@ -11,6 +11,7 @@ interface ToggleProps { size?: "sm" | "md" | "lg"; className?: string; title?: string; + ariaLabel?: string; } export default function Toggle({ @@ -21,6 +22,8 @@ export default function Toggle({ disabled = false, size = "md", className, + title, + ariaLabel, }: ToggleProps) { const sizes = { sm: { @@ -58,7 +61,8 @@ export default function Toggle({ type="button" role="switch" aria-checked={checked} - aria-label={!label ? description || "Toggle" : undefined} + aria-label={ariaLabel || label || description || title || "Toggle"} + title={title} disabled={disabled} onClick={handleClick} className={cn( diff --git a/src/shared/components/UsageAnalytics.tsx b/src/shared/components/UsageAnalytics.tsx index 0ab5838dc9..bda130c41a 100644 --- a/src/shared/components/UsageAnalytics.tsx +++ b/src/shared/components/UsageAnalytics.tsx @@ -18,6 +18,7 @@ import { ProviderCostDonut, ModelOverTimeChart, ProviderTable, + ServiceTierBreakdown, ApiKeyFilterDropdown, CustomRangePicker, } from "./analytics"; @@ -311,10 +312,10 @@ export default function UsageAnalytics() { color: "text-violet-500", }, { - icon: "swap_horiz", - label: "Fallback Rate", - value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`, - color: "text-amber-500", + icon: "bolt", + label: "Fast Requests", + value: fmt(s.fastRequests || 0), + color: "text-sky-500", }, ], }, @@ -331,6 +332,12 @@ export default function UsageAnalytics() { value: `${providerDiversity.toFixed(1)}%`, color: "text-sky-500", }, + { + icon: "swap_horiz", + label: "Fallback Rate", + value: `${Number(s.fallbackRatePct || 0).toFixed(1)}%`, + color: "text-amber-500", + }, ], }, ]} @@ -351,6 +358,9 @@ export default function UsageAnalytics() {
+ {/* Fast / Standard service tier split */} + + {/* Model Usage Over Time (stacked area) */} byServiceTier || [], [byServiceTier]); + const totalRequests = Number(summary?.totalRequests || 0); + const totalCost = Number(summary?.totalCost || 0); + + if (!data.length) { + return null; + } + + return ( + +
+

+ Service Tier +

+ Fast / Standard cost split +
+
+ {data.map((tier) => { + const isFast = tier.serviceTier === "priority"; + const requestPct = + totalRequests > 0 + ? ((Number(tier.requests || 0) / totalRequests) * 100).toFixed(1) + : "0"; + const costPct = + totalCost > 0 ? ((Number(tier.cost || 0) / totalCost) * 100).toFixed(1) : "0"; + return ( +
+
+
+ + {isFast ? "bolt" : "speed"} + +
+
{tier.label}
+
+ {fmtFull(tier.requests)} requests ยท {fmt(tier.totalTokens)} tokens +
+
+
+
+
+ {fmtCost(tier.cost)} +
+
{costPct}% of cost
+
+
+
+
+
+
+ ); + })} +
+ + ); + } const sorted = useMemo(() => { const arr = [...(byModel || [])]; arr.sort((a, b) => { diff --git a/src/shared/components/analytics/index.tsx b/src/shared/components/analytics/index.tsx index fb14864249..fb9d2be56a 100644 --- a/src/shared/components/analytics/index.tsx +++ b/src/shared/components/analytics/index.tsx @@ -25,6 +25,7 @@ export { ProviderCostDonut, ModelOverTimeChart, ProviderTable, + ServiceTierBreakdown, } from "./charts"; export { default as ApiKeyFilterDropdown } from "./ApiKeyFilterDropdown"; diff --git a/tests/unit/usage-analytics-route.test.ts b/tests/unit/usage-analytics-route.test.ts index 66935d5819..9c44f37099 100644 --- a/tests/unit/usage-analytics-route.test.ts +++ b/tests/unit/usage-analytics-route.test.ts @@ -150,21 +150,56 @@ test("GET /api/usage/analytics resolves Codex GPT-5.5 pricing through provider a assertClose(body.byModel[0].cost, 0.02); }); +test("GET /api/usage/analytics applies Codex Fast tier multipliers and exposes tier split", async () => { + const db = core.getDbInstance(); + const timestamp = new Date().toISOString(); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.5", "codex-fast", 1000, 500, 1, 250, "priority", timestamp); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.5", "codex-standard", 1000, 500, 1, 250, "standard", timestamp); + + const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); + const body = await response.json(); + + assert.equal(response.status, 200); + assertClose(body.summary.totalCost, 0.07); + assert.equal(body.summary.fastRequests, 1); + assert.equal(body.summary.standardRequests, 1); + assertClose(body.summary.fastCost, 0.05); + assertClose(body.summary.standardCost, 0.02); + assert.equal(body.byServiceTier.length, 2); + assertClose(body.byProvider[0].cost, 0.07); + assertClose(body.byModel[0].cost, 0.07); +}); + +test("GET /api/usage/analytics applies Codex GPT-5.4 Fast multiplier", async () => { + await localDb.updatePricing({ + codex: { "gpt-5.4": { input: 5, output: 30 } }, + }); + const db = core.getDbInstance(); + db.prepare( + `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, service_tier, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run("codex", "gpt-5.4", "codex-fast", 1000, 500, 1, 250, "priority", new Date().toISOString()); + + const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); + const body = await response.json(); + + assert.equal(response.status, 200); + assertClose(body.summary.totalCost, 0.04); + assertClose(body.summary.fastCost, 0.04); +}); + test("GET /api/usage/analytics maps Codex auto-review usage to GPT-5.5 pricing", async () => { const db = core.getDbInstance(); db.prepare( `INSERT INTO usage_history (provider, model, connection_id, tokens_input, tokens_output, success, latency_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ).run( - "codex", - "codex-auto-review", - "codex-conn", - 1000, - 500, - 1, - 250, - new Date().toISOString() - ); + ).run("codex", "codex-auto-review", "codex-conn", 1000, 500, 1, 250, new Date().toISOString()); const response = await analyticsRoute.GET(makeRequest("http://localhost/api/usage/analytics")); const body = await response.json(); diff --git a/tests/unit/usage-analytics.test.ts b/tests/unit/usage-analytics.test.ts index 8ff5f6662c..54ad4ecae4 100644 --- a/tests/unit/usage-analytics.test.ts +++ b/tests/unit/usage-analytics.test.ts @@ -275,6 +275,22 @@ test("getUsageStats aggregates totals, buckets, pending requests, and cost break assert.equal(recentBucketTotal, 1); }); +test("Codex Fast service tier applies documented GPT-5.5 and GPT-5.4 cost multipliers", async () => { + await localDb.updatePricing({ + codex: { + "gpt-5.5": { input: 5, output: 30 }, + "gpt-5.4": { input: 5, output: 30 }, + }, + }); + + const tokens = { input: 1000, output: 500 }; + + assert.equal(await calculateCost("codex", "gpt-5.5", tokens), 0.02); + assert.equal(await calculateCost("codex", "gpt-5.5", tokens, { serviceTier: "priority" }), 0.05); + assert.equal(await calculateCost("codex", "gpt-5.4-high", tokens, { serviceTier: "fast" }), 0.04); + assert.equal(await calculateCost("openai", "gpt-5.5", tokens, { serviceTier: "priority" }), 0); +}); + test("recent request summaries are generated from SQLite call logs", async () => { const connection = await providersDb.createProviderConnection({ provider: "log-provider", diff --git a/tests/unit/usage-history-db.test.ts b/tests/unit/usage-history-db.test.ts index d3e44fc1d1..e97972b29a 100644 --- a/tests/unit/usage-history-db.test.ts +++ b/tests/unit/usage-history-db.test.ts @@ -28,6 +28,7 @@ async function seedUsageEntries( success?: boolean; latencyMs?: number; minutesAgo?: number; + serviceTier?: string; }> ) { for (const [i, e] of entries.entries()) { @@ -41,6 +42,7 @@ async function seedUsageEntries( success: e.success !== false, latencyMs: e.latencyMs || 100, timestamp: new Date(Date.now() - (e.minutesAgo || i) * 60 * 1000).toISOString(), + serviceTier: e.serviceTier, }); } } @@ -89,6 +91,32 @@ test("getUsageDb filters by sinceIso date", async () => { assert.equal(result.data.history[0].provider, "new"); }); +test("usage history persists service tier and defaults to standard", async () => { + await usageHistory.saveRequestUsage({ + provider: "codex", + model: "gpt-5.5", + tokens: { input: 100, output: 50 }, + success: true, + serviceTier: "priority", + timestamp: new Date().toISOString(), + }); + await usageHistory.saveRequestUsage({ + provider: "openai", + model: "gpt-4o", + tokens: { input: 10, output: 5 }, + success: true, + timestamp: new Date(Date.now() + 1).toISOString(), + }); + + const history = await usageHistory.getUsageHistory({ provider: "codex" }); + const usageDb = await usageHistory.getUsageDb(); + + assert.equal(history.length, 1); + assert.equal(history[0].serviceTier, "priority"); + assert.equal(usageDb.data.history[0].serviceTier, "priority"); + assert.equal(usageDb.data.history[1].serviceTier, "standard"); +}); + test("getUsageDb provides nextCursor when rows exceed MAX_ROWS", async () => { const db = core.getDbInstance(); // Insert exactly MAX_ROWS + 1 = 10001 rows so getUsageDb returns a cursor