fix(analytics): robust model pricing resolution, dark mode charts and SQL aggregation fixes (#1896)

Integrated into release/v3.7.9 (migration renumbered to 044)
2026-07-26 09:52:11 +03:00 · 2026-05-03 22:19:08 +03:00
parent 9dc377e1d8
commit 1d38900f17
3 changed files with 196 additions and 82 deletions
--- a/src/app/api/usage/analytics/route.ts
+++ b/src/app/api/usage/analytics/route.ts
@@ -30,12 +30,6 @@ function getRangeStartIso(range: string): string | null {
  return start.toISOString();
 }

-function shortModelName(model: string | null): string {
-  if (!model) return "-";
-  const parts = model.split(/[/:-]/);
-  return parts[parts.length - 1] || model;
-}
-
 const WEEKDAY_LABELS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];

 type PricingByProvider = Record<string, Record<string, Record<string, unknown>>>;
@@ -65,28 +59,76 @@ function appendWhereCondition(whereClause: string, condition: string): string {
  return whereClause ? `${whereClause} AND (${condition})` : `WHERE (${condition})`;
 }

+function findKeyInsensitive(obj: Record<string, any> | undefined | null, key: string): any {
+  if (!obj || !key) return undefined;
+  return obj[key.toLowerCase()];
+}
+
 function resolveModelPricing(
  pricingByProvider: PricingByProvider,
-  provider: string,
+  providerAliasMap: Record<string, string>,
+  providerRaw: string,
  model: string,
  normalizeModelName: (model: string) => string
 ): Record<string, unknown> | null {
-  const providerPricing = pricingByProvider[provider];
-  if (!providerPricing) return null;
+  const pLower = (providerRaw || "").toLowerCase();

-  const normalizedModel = normalizeModelName(model);
-  const shortModel = shortModelName(model);
-  return (
-    providerPricing[model] ||
-    providerPricing[normalizedModel] ||
-    providerPricing[shortModel] ||
-    null
-  );
+  let providerPricing = findKeyInsensitive(pricingByProvider, pLower);
+  if (!providerPricing) {
+    const alias = providerAliasMap[pLower];
+    if (alias) {
+      providerPricing = findKeyInsensitive(pricingByProvider, alias);
+    } else {
+      const np = pLower.replace(/-cn$/, "");
+      if (np && np !== pLower) {
+        providerPricing = findKeyInsensitive(pricingByProvider, np);
+      }
+    }
+  }
+
+  // Hardcoded known fallbacks
+  if (!providerPricing) {
+    if (pLower === "antigravity") providerPricing = findKeyInsensitive(pricingByProvider, "ag");
+  }
+
+  const normalizedModel = normalizeModelName(model).toLowerCase();
+  const shortModel = normalizedModel; // normalizeModelName behaves exactly like shortModelName
+  const hyphenModel = model.toLowerCase().replace(/\./g, "-");
+  const hyphenNormalized = normalizedModel.replace(/\./g, "-");
+  const lowerModel = model.toLowerCase();
+
+  const tryFind = (prov: Record<string, unknown> | null | undefined) => {
+    if (!prov || typeof prov !== "object") return null;
+    return (
+      findKeyInsensitive(prov as Record<string, unknown>, lowerModel) ||
+      findKeyInsensitive(prov as Record<string, unknown>, normalizedModel) ||
+      findKeyInsensitive(prov as Record<string, unknown>, shortModel) ||
+      findKeyInsensitive(prov as Record<string, unknown>, hyphenModel) ||
+      findKeyInsensitive(prov as Record<string, unknown>, hyphenNormalized) ||
+      null
+    );
+  };
+
+  let pricing = providerPricing ? tryFind(providerPricing) : null;
+
+  if (!pricing) {
+    // Global fallback: search all providers for this exact model (helps with aliases)
+    for (const prov of Object.values(pricingByProvider)) {
+      const found = tryFind(prov as Record<string, unknown>);
+      if (found) {
+        pricing = found;
+        break;
+      }
+    }
+  }
+
+  return pricing as Record<string, unknown> | null;
 }

 function computeUsageRowCost(
  row: Record<string, unknown>,
  pricingByProvider: PricingByProvider,
+  providerAliasMap: Record<string, string>,
  normalizeModelName: (model: string) => string,
  computeCostFromPricing: ComputeCostFromPricing
 ): number {
@@ -94,7 +136,13 @@ function computeUsageRowCost(
  const model = toStringValue(row.model);
  if (!provider || !model) return 0;

-  const pricing = resolveModelPricing(pricingByProvider, provider, model, normalizeModelName);
+  const pricing = resolveModelPricing(
+    pricingByProvider,
+    providerAliasMap,
+    provider,
+    model,
+    normalizeModelName
+  );
  if (!pricing) return 0;

  return computeCostFromPricing(pricing, {
@@ -163,9 +211,20 @@ export async function GET(request: Request) {

    // Fetch pricing data for cost calculation (no rows loaded)
    const { getPricing } = await import("@/lib/db/settings");
-    const pricingByProvider = (await getPricing()) as PricingByProvider;
+    const rawPricingByProvider = (await getPricing()) as PricingByProvider;
+
+    // Pre-process pricing data to lowercase keys for O(1) lookups
+    const pricingByProvider: PricingByProvider = {};
+    for (const [providerKey, providerVal] of Object.entries(rawPricingByProvider || {})) {
+      const lowerProvider = {};
+      for (const [modelKey, modelVal] of Object.entries(providerVal || {})) {
+        (lowerProvider as any)[modelKey.toLowerCase()] = modelVal;
+      }
+      pricingByProvider[providerKey.toLowerCase()] = lowerProvider;
+    }
    const { computeCostFromPricing, normalizeModelName } =
      await import("@/lib/usage/costCalculator");
+    const { PROVIDER_ID_TO_ALIAS } = await import("@omniroute/open-sse/config/providerModels");

    const summaryRow = db
      .prepare(
@@ -210,8 +269,8 @@ export async function GET(request: Request) {
        `
        SELECT
          DATE(timestamp) as date,
-          provider,
-          model,
+          LOWER(provider) as provider,
+          LOWER(model) as model,
          COALESCE(SUM(tokens_input), 0) as promptTokens,
          COALESCE(SUM(tokens_output), 0) as completionTokens,
          COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
@@ -219,7 +278,7 @@ export async function GET(request: Request) {
          COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
        FROM usage_history
        ${whereClause}
-        GROUP BY DATE(timestamp), provider, model
+        GROUP BY DATE(timestamp), LOWER(provider), LOWER(model)
        ORDER BY date ASC
      `
      )
@@ -263,8 +322,8 @@ export async function GET(request: Request) {
      .prepare(
        `
        SELECT
-          model,
-          provider,
+          LOWER(model) as model,
+          LOWER(provider) as provider,
          COUNT(*) as requests,
          COALESCE(SUM(tokens_input), 0) as promptTokens,
          COALESCE(SUM(tokens_output), 0) as completionTokens,
@@ -277,7 +336,7 @@ export async function GET(request: Request) {
          COALESCE(MAX(timestamp), '') as lastUsed
        FROM usage_history
        ${whereClause}
-        GROUP BY model, provider
+        GROUP BY LOWER(model), LOWER(provider)
        ORDER BY requests DESC
        LIMIT 50
      `
@@ -288,8 +347,8 @@ export async function GET(request: Request) {
      .prepare(
        `
        SELECT
-          provider,
-          model,
+          LOWER(provider) as provider,
+          LOWER(model) as model,
          COALESCE(SUM(tokens_input), 0) as promptTokens,
          COALESCE(SUM(tokens_output), 0) as completionTokens,
          COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
@@ -297,7 +356,7 @@ export async function GET(request: Request) {
          COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
        FROM usage_history
        ${whereClause}
-        GROUP BY provider, model
+        GROUP BY LOWER(provider), LOWER(model)
      `
      )
      .all(params) as Array<Record<string, unknown>>;
@@ -306,7 +365,7 @@ export async function GET(request: Request) {
      .prepare(
        `
        SELECT
-          provider,
+          LOWER(provider) as provider,
          COUNT(*) as requests,
          COALESCE(SUM(tokens_input), 0) as promptTokens,
          COALESCE(SUM(tokens_output), 0) as completionTokens,
@@ -315,7 +374,7 @@ export async function GET(request: Request) {
          COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) as successfulRequests
        FROM usage_history
        ${whereClause}
-        GROUP BY provider
+        GROUP BY LOWER(provider)
        ORDER BY requests DESC
      `
      )
@@ -325,17 +384,18 @@ export async function GET(request: Request) {
      .prepare(
        `
        SELECT
-          connection_id as account,
-          provider,
-          model,
-          COALESCE(SUM(tokens_input), 0) as promptTokens,
-          COALESCE(SUM(tokens_output), 0) as completionTokens,
-          COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
-          COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens,
-          COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
+          COALESCE(NULLIF(c.display_name, ''), NULLIF(c.email, ''), NULLIF(c.name, ''), usage_history.connection_id, 'unknown') as account,
+          LOWER(usage_history.provider) as provider,
+          LOWER(usage_history.model) as model,
+          COALESCE(SUM(usage_history.tokens_input), 0) as promptTokens,
+          COALESCE(SUM(usage_history.tokens_output), 0) as completionTokens,
+          COALESCE(SUM(usage_history.tokens_cache_read), 0) as cacheReadTokens,
+          COALESCE(SUM(usage_history.tokens_cache_creation), 0) as cacheCreationTokens,
+          COALESCE(SUM(usage_history.tokens_reasoning), 0) as reasoningTokens
        FROM usage_history
-        ${whereClause}
-        GROUP BY connection_id, provider, model
+        LEFT JOIN provider_connections c ON c.id = usage_history.connection_id
+        ${whereClause.replace(/timestamp/g, "usage_history.timestamp").replace(/api_key_/g, "usage_history.api_key_")}
+        GROUP BY account, LOWER(usage_history.provider), LOWER(usage_history.model)
      `
      )
      .all(params) as Array<Record<string, unknown>>;
@@ -344,31 +404,35 @@ export async function GET(request: Request) {
      .prepare(
        `
        SELECT
-          connection_id as account,
-          COUNT(*) as requests,
-          COALESCE(SUM(tokens_input), 0) as promptTokens,
-          COALESCE(SUM(tokens_output), 0) as completionTokens,
-          COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens,
-          COALESCE(AVG(latency_ms), 0) as avgLatencyMs,
-          COALESCE(MAX(timestamp), '') as lastUsed
+          COALESCE(NULLIF(c.display_name, ''), NULLIF(c.email, ''), NULLIF(c.name, ''), usage_history.connection_id, 'unknown') as account,
+          COUNT(usage_history.id) as requests,
+          COALESCE(SUM(usage_history.tokens_input), 0) as promptTokens,
+          COALESCE(SUM(usage_history.tokens_output), 0) as completionTokens,
+          COALESCE(SUM(usage_history.tokens_input + usage_history.tokens_output), 0) as totalTokens,
+          COALESCE(AVG(usage_history.latency_ms), 0) as avgLatencyMs,
+          COALESCE(MAX(usage_history.timestamp), '') as lastUsed
        FROM usage_history
-        ${whereClause}
-        GROUP BY connection_id
+        LEFT JOIN provider_connections c ON c.id = usage_history.connection_id
+        ${whereClause.replace(/timestamp/g, "usage_history.timestamp").replace(/api_key_/g, "usage_history.api_key_")}
+        GROUP BY account
        ORDER BY requests DESC
        LIMIT 50
      `
      )
      .all(params) as Array<Record<string, unknown>>;

-    const apiKeyWhereClause = whereClause;
+    const apiKeyWhereClause = appendWhereCondition(
+      whereClause,
+      "(api_key_id IS NOT NULL AND api_key_id != '') OR (api_key_name IS NOT NULL AND api_key_name != '')"
+    );
    const apiKeyRows = db
      .prepare(
        `
        SELECT
          api_key_id as apiKeyId,
          COALESCE(NULLIF(api_key_name, ''), NULLIF(api_key_id, ''), 'Unknown API key') as apiKeyName,
-          provider,
-          model,
+          LOWER(provider) as provider,
+          LOWER(model) as model,
          COUNT(*) as requests,
          COALESCE(SUM(tokens_input), 0) as promptTokens,
          COALESCE(SUM(tokens_output), 0) as completionTokens,
@@ -378,7 +442,7 @@ export async function GET(request: Request) {
          COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens
        FROM usage_history
        ${apiKeyWhereClause}
-        GROUP BY api_key_id, api_key_name, provider, model
+        GROUP BY api_key_id, api_key_name, LOWER(provider), LOWER(model)
      `
      )
      .all(params) as Array<Record<string, unknown>>;
@@ -471,17 +535,31 @@ export async function GET(request: Request) {
      streak: 0,
    };

+    const dailyByModelMap: Record<string, Record<string, number>> = {};
+    const allModels = new Set<string>();
+
    const dailyCostByDate = new Map<string, number>();
    for (const row of dailyCostRows) {
      const date = toStringValue(row.date);
      if (!date) continue;
+
+      // Calculate costs
      const cost = computeUsageRowCost(
        row,
        pricingByProvider,
+        PROVIDER_ID_TO_ALIAS,
        normalizeModelName,
        computeCostFromPricing
      );
      dailyCostByDate.set(date, (dailyCostByDate.get(date) || 0) + cost);
+
+      // Group tokens by model for the day
+      const model = normalizeModelName(row.model as string);
+      const tokens = Number(row.promptTokens) + Number(row.completionTokens);
+
+      if (!dailyByModelMap[date]) dailyByModelMap[date] = {};
+      dailyByModelMap[date][model] = (dailyByModelMap[date][model] || 0) + tokens;
+      allModels.add(model);
    }

    const dailyTrend = dailyRows.map((row) => ({
@@ -502,7 +580,7 @@ export async function GET(request: Request) {
    const byModel = modelRows.map((row) => {
      const model = row.model as string;
      const provider = row.provider as string;
-      const short = shortModelName(model);
+      const short = normalizeModelName(model);
      const tokens = {
        input: Number(row.promptTokens) || 0,
        output: Number(row.completionTokens) || 0,
@@ -510,6 +588,7 @@ export async function GET(request: Request) {
      const cost = computeUsageRowCost(
        row,
        pricingByProvider,
+        PROVIDER_ID_TO_ALIAS,
        normalizeModelName,
        computeCostFromPricing
      );
@@ -541,6 +620,7 @@ export async function GET(request: Request) {
      const cost = computeUsageRowCost(
        row,
        pricingByProvider,
+        PROVIDER_ID_TO_ALIAS,
        normalizeModelName,
        computeCostFromPricing
      );
@@ -567,6 +647,7 @@ export async function GET(request: Request) {
      const cost = computeUsageRowCost(
        row,
        pricingByProvider,
+        PROVIDER_ID_TO_ALIAS,
        normalizeModelName,
        computeCostFromPricing
      );
@@ -619,6 +700,7 @@ export async function GET(request: Request) {
      existing.cost += computeUsageRowCost(
        row,
        pricingByProvider,
+        PROVIDER_ID_TO_ALIAS,
        normalizeModelName,
        computeCostFromPricing
      );
@@ -650,6 +732,11 @@ export async function GET(request: Request) {
      }
    }

+    const dailyByModel = Object.keys(dailyByModelMap)
+      .sort()
+      .map((date) => ({ date, ...dailyByModelMap[date] }));
+    const modelNames = Array.from(allModels);
+
    const analytics = {
      summary,
      dailyTrend,
@@ -661,6 +748,8 @@ export async function GET(request: Request) {
      weeklyPattern,
      weeklyTokens,
      weeklyCounts,
+      dailyByModel,
+      modelNames,
      range,
    } as any;

@@ -699,8 +788,8 @@ export async function GET(request: Request) {
          .prepare(
            `
            SELECT
-              model,
-              provider,
+              LOWER(model) as model,
+              LOWER(provider) as provider,
              COALESCE(SUM(tokens_input), 0) as promptTokens,
              COALESCE(SUM(tokens_output), 0) as completionTokens,
              COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
@@ -708,7 +797,7 @@ export async function GET(request: Request) {
              COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
            FROM usage_history
            ${presetWhere}
-            GROUP BY model, provider
+            GROUP BY LOWER(model), LOWER(provider)
          `
          )
          .all(presetParams) as Array<Record<string, unknown>>;
@@ -718,6 +807,7 @@ export async function GET(request: Request) {
          presetTotalCost += computeUsageRowCost(
            row,
            pricingByProvider,
+            PROVIDER_ID_TO_ALIAS,
            normalizeModelName,
            computeCostFromPricing
          );
--- a/src/lib/db/migrations/044_usage_history_api_key_backfill.sql
+++ b/src/lib/db/migrations/044_usage_history_api_key_backfill.sql
@@ -0,0 +1,27 @@
+-- Usage History API Key Backfill
+-- Backfills missing API key names and IDs in usage_history using the connection_id
+
+UPDATE usage_history
+SET 
+  api_key_name = (
+    SELECT api_key_name
+    FROM usage_history AS uh2
+    WHERE uh2.connection_id = usage_history.connection_id
+      AND uh2.api_key_name IS NOT NULL
+      AND uh2.api_key_name != ''
+    GROUP BY uh2.api_key_name
+    ORDER BY COUNT(*) DESC
+    LIMIT 1
+  ),
+  api_key_id = (
+    SELECT api_key_id
+    FROM usage_history AS uh2
+    WHERE uh2.connection_id = usage_history.connection_id
+      AND uh2.api_key_id IS NOT NULL
+      AND uh2.api_key_id != ''
+    GROUP BY uh2.api_key_id
+    ORDER BY COUNT(*) DESC
+    LIMIT 1
+  )
+WHERE (api_key_name IS NULL OR api_key_name = '')
+  AND connection_id IS NOT NULL;
--- a/src/shared/components/analytics/charts.tsx
+++ b/src/shared/components/analytics/charts.tsx
@@ -267,10 +267,7 @@ export function ActivityHeatmap({ activityMap }) {
        </span>
      </div>

-      <div
-        ref={scrollRef}
-        className="overflow-x-auto"
-      >
+      <div ref={scrollRef} className="overflow-x-auto">
        <div className="w-max">
          <div className="flex gap-[3px] mb-1 ml-6" style={{ fontSize: "10px" }}>
            {monthLabels.map((m, i) => (
@@ -300,19 +297,19 @@ export function ActivityHeatmap({ activityMap }) {
            </div>

            {weeks.map((week, wi) => (
-            <div key={wi} className="flex flex-col gap-[3px]">
-              {week.map((day, di) => (
-                <div
-                  key={di}
-                  title={day ? `${day.date}: ${fmtFull(day.value)} tokens` : ""}
-                  className={`w-[10px] h-[10px] rounded-[2px] ${day ? getCellColor(day.value) : "bg-transparent"}`}
-                />
-              ))}
-            </div>
-          ))}
+              <div key={wi} className="flex flex-col gap-[3px]">
+                {week.map((day, di) => (
+                  <div
+                    key={di}
+                    title={day ? `${day.date}: ${fmtFull(day.value)} tokens` : ""}
+                    className={`w-[10px] h-[10px] rounded-[2px] ${day ? getCellColor(day.value) : "bg-transparent"}`}
+                  />
+                ))}
+              </div>
+            ))}
+          </div>
        </div>
      </div>
-      </div>

      <div className="flex items-center gap-1 mt-2 ml-6 text-[10px] text-text-muted">
        <span>Less</span>
@@ -364,7 +361,7 @@ export function DailyTrendChart({ dailyTrend }) {
        >
          <XAxis
            dataKey="date"
-            tick={{ fontSize: 9, fill: "var(--text-muted)" }}
+            tick={{ fontSize: 9, fill: "var(--color-text-muted)" }}
            axisLine={false}
            tickLine={false}
            interval={Math.max(Math.floor(chartData.length / 6), 0)}
@@ -784,7 +781,7 @@ export function WeeklyPattern({ weeklyPattern }) {
        <BarChart data={chartData} margin={{ top: 0, right: 0, left: 0, bottom: 0 }}>
          <XAxis
            dataKey="day"
-            tick={{ fontSize: 9, fill: "var(--text-muted)" }}
+            tick={{ fontSize: 9, fill: "var(--color-text-muted)" }}
            axisLine={false}
            tickLine={false}
          />
@@ -794,7 +791,7 @@ export function WeeklyPattern({ weeklyPattern }) {
          />
          <Bar
            dataKey="Tokens"
-            fill="var(--text-muted)"
+            fill="var(--color-text-muted)"
            opacity={0.3}
            radius={[3, 3, 0, 0]}
            animationDuration={400}
@@ -847,7 +844,7 @@ export function MostActiveDay7d({ activityMap }) {
    <Card className="p-4 flex flex-col justify-center" style={{ flex: 1, minHeight: 0 }}>
      <h3
        className="text-xs font-semibold uppercase tracking-wider mb-2"
-        style={{ color: "var(--text-muted)" }}
+        style={{ color: "var(--color-text-muted)" }}
      >
        Most Active Day
      </h3>
@@ -856,12 +853,12 @@ export function MostActiveDay7d({ activityMap }) {
          <span className="text-xl font-bold capitalize" style={{ lineHeight: 1.2 }}>
            {data.weekday}
          </span>
-          <span className="text-xs mt-1" style={{ color: "var(--text-muted)" }}>
+          <span className="text-xs mt-1" style={{ color: "var(--color-text-muted)" }}>
            {data.label} · {fmt(data.tokens)} tokens
          </span>
        </>
      ) : (
-        <span className="text-xs" style={{ color: "var(--text-muted)" }}>
+        <span className="text-xs" style={{ color: "var(--color-text-muted)" }}>
          No data in the last 7 days
        </span>
      )}
@@ -913,7 +910,7 @@ export function WeeklySquares7d({ activityMap }) {
    <Card className="p-4 flex flex-col justify-center" style={{ flex: 1, minHeight: 0 }}>
      <h3
        className="text-xs font-semibold uppercase tracking-wider mb-3"
-        style={{ color: "var(--text-muted)" }}
+        style={{ color: "var(--color-text-muted)" }}
      >
        Weekly
      </h3>
@@ -938,7 +935,7 @@ export function WeeklySquares7d({ activityMap }) {
              style={{
                fontSize: 9,
                fontWeight: 600,
-                color: "var(--text-muted)",
+                color: "var(--color-text-muted)",
                letterSpacing: "0.03em",
              }}
            >
@@ -1238,13 +1235,13 @@ export function ModelOverTimeChart({ dailyByModel, modelNames }) {
        <AreaChart data={chartData} margin={{ top: 4, right: 4, left: 0, bottom: 0 }}>
          <XAxis
            dataKey="dateLabel"
-            tick={{ fontSize: 10, fill: "var(--text-muted)" }}
+            tick={{ fontSize: 10, fill: "var(--color-text-muted)" }}
            axisLine={false}
            tickLine={false}
            interval="preserveStartEnd"
          />
          <YAxis
-            tick={{ fontSize: 10, fill: "var(--text-muted)" }}
+            tick={{ fontSize: 10, fill: "var(--color-text-muted)" }}
            tickFormatter={(v) => fmt(v)}
            axisLine={false}
            tickLine={false}