diff --git a/src/app/(dashboard)/dashboard/provider-stats/page.tsx b/src/app/(dashboard)/dashboard/provider-stats/page.tsx new file mode 100644 index 0000000000..9be2cb0a5a --- /dev/null +++ b/src/app/(dashboard)/dashboard/provider-stats/page.tsx @@ -0,0 +1,499 @@ +"use client"; + +/** + * Provider Stats Dashboard + * + * Shows per-provider and per-model statistics aggregated from call_logs, + * plus in-memory combo metrics and telemetry data. + */ + +import { useState, useEffect, useCallback, Fragment } from "react"; +import { Card } from "@/shared/components"; + +interface ProviderStat { + provider: string; + totalRequests: number; + successfulRequests: number; + avgLatencyMs: number; + totalTokensIn: number; + totalTokensOut: number; +} + +interface ModelStat { + provider: string; + model: string; + requests: number; + avgLatencyMs: number; + successfulRequests: number; +} + +interface ToolLatencyStat { + avgTtftAfterToolMs: number; + avgGapAfterToolMs: number; + measurementCount: number; +} + +type SortKey = "totalRequests" | "successfulRequests" | "avgLatencyMs" | "totalTokensIn" | "totalTokensOut" | "avgTtftAfterToolMs" | "avgGapAfterToolMs"; +type SortDir = "asc" | "desc"; + +function formatNumber(n: number | null): string { + if (n == null) return "0"; + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`; + return n.toString(); +} + +function formatLatency(ms: number | null): string { + if (ms == null) return "—"; + if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`; + return `${Math.round(ms)}ms`; +} + +function successRate(successful: number, total: number): string { + if (total === 0) return "—"; + return `${((successful / total) * 100).toFixed(1)}%`; +} + +export default function ProviderStatsPage() { + const [data, setData] = useState<{ + providers: ProviderStat[]; + models: ModelStat[]; + comboMetrics: Record; + telemetry: Record; + toolLatency: Record; + } | null>(null); + const [error, setError] = useState(null); + const [lastRefresh, setLastRefresh] = useState(null); + const [sortKey, setSortKey] = useState("totalRequests"); + const [sortDir, setSortDir] = useState("desc"); + const [expandedProvider, setExpandedProvider] = useState(null); + + const fetchData = useCallback(async () => { + try { + const res = await fetch("/api/provider-stats"); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const json = await res.json(); + setData(json); + setError(null); + setLastRefresh(new Date()); + } catch (err) { + setError(err instanceof Error ? err.message : "Unknown error"); + } + }, []); + + useEffect(() => { + fetchData(); + const interval = setInterval(fetchData, 30000); + return () => clearInterval(interval); + }, [fetchData]); + + const handleSort = (key: SortKey) => { + if (sortKey === key) { + setSortDir(sortDir === "desc" ? "asc" : "desc"); + } else { + setSortKey(key); + setSortDir("desc"); + } + }; + + // Compute summary stats + const totalRequests = data?.providers.reduce((s, p) => s + (p.totalRequests || 0), 0) ?? 0; + const totalSuccessful = data?.providers.reduce((s, p) => s + (p.successfulRequests || 0), 0) ?? 0; + const avgLatency = + data?.providers.length + ? Math.round( + data.providers.reduce((s, p) => s + (p.avgLatencyMs || 0), 0) / data.providers.length + ) + : 0; + const activeProviders = data?.providers.length ?? 0; + + // Sorted providers + const sortedProviders = [...(data?.providers ?? [])].sort((a, b) => { + if (sortKey === "avgTtftAfterToolMs" || sortKey === "avgGapAfterToolMs") { + const aLatency = data?.toolLatency?.[a.provider] ?? null; + const bLatency = data?.toolLatency?.[b.provider] ?? null; + const va = aLatency ? (aLatency[sortKey] as number) ?? 0 : 0; + const vb = bLatency ? (bLatency[sortKey] as number) ?? 0 : 0; + return sortDir === "desc" ? vb - va : va - vb; + } + const va = (a[sortKey] as number) ?? 0; + const vb = (b[sortKey] as number) ?? 0; + return sortDir === "desc" ? vb - va : va - vb; + }); + + // Group models by provider + const modelsByProvider = new Map(); + for (const m of data?.models ?? []) { + if (!modelsByProvider.has(m.provider)) modelsByProvider.set(m.provider, []); + modelsByProvider.get(m.provider)!.push(m); + } + + // Sort helper icon + const SortIcon = ({ column }: { column: SortKey }) => { + if (sortKey !== column) return null; + return ( + + {sortDir === "desc" ? "arrow_downward" : "arrow_upward"} + + ); + }; + + if (!data && !error) { + return ( +
+
+
+

Loading provider stats...

+
+
+ ); + } + + if (error && !data) { + return ( +
+
+ error +

Failed to load provider stats: {error}

+ +
+
+ ); + } + + return ( +
+ {/* Header */} +
+ {lastRefresh && ( + + Updated {lastRefresh.toLocaleTimeString()} + + )} + +
+ + {/* Summary Cards */} +
+ +
+
+ analytics +
+ Total Requests +
+

{formatNumber(totalRequests)}

+ + + +
+
+ timer +
+ Avg Latency +
+

{formatLatency(avgLatency)}

+ + + +
+
+ check_circle +
+ Success Rate +
+

+ {successRate(totalSuccessful, totalRequests)} +

+ + + +
+
+ dns +
+ Active Providers +
+

{activeProviders}

+ +
+ + {/* Provider Table */} + +
+

+ table_chart + Provider Breakdown +

+ {activeProviders} providers +
+ +
+ + + + + + + + + + + + + + + + {sortedProviders.map((p) => { + const isExpanded = expandedProvider === p.provider; + const models = modelsByProvider.get(p.provider) ?? []; + const rate = p.totalRequests > 0 ? (p.successfulRequests / p.totalRequests) * 100 : 0; + return ( + + + setExpandedProvider(isExpanded ? null : models.length > 0 ? p.provider : null) + } + > + + + + + + + + + + + + {isExpanded && models.length > 0 && ( + + + + )} + + ); + })} + {sortedProviders.length === 0 && ( + + + + )} + +
Provider handleSort("totalRequests")} + > + Requests + handleSort("successfulRequests")} + > + Success + Rate handleSort("avgLatencyMs")} + > + Avg Latency + handleSort("totalTokensIn")} + > + Tokens In + handleSort("totalTokensOut")} + > + Tokens Out + handleSort("avgTtftAfterToolMs")} + > + TTFT After Tool + handleSort("avgGapAfterToolMs")} + > + Gap After Tool + +
+ {p.provider} + + {formatNumber(p.totalRequests)} + + {formatNumber(p.successfulRequests)} + + = 99 + ? "text-green-500" + : rate >= 95 + ? "text-amber-500" + : "text-red-500" + } + > + {rate.toFixed(1)}% + + + {formatLatency(p.avgLatencyMs)} + + {formatNumber(p.totalTokensIn)} + + {formatNumber(p.totalTokensOut)} + + {formatLatency(data?.toolLatency?.[p.provider]?.avgTtftAfterToolMs ?? null)} + + {formatLatency(data?.toolLatency?.[p.provider]?.avgGapAfterToolMs ?? null)} + + {models.length > 0 && ( + + chevron_right + + )} +
+
+ + + + + + + + + + + + {models.map((m) => { + const mRate = + m.requests > 0 ? (m.successfulRequests / m.requests) * 100 : 0; + return ( + + + + + + + + ); + })} + +
ModelRequestsSuccessRateAvg Latency +
+ {m.model} + + {formatNumber(m.requests)} + + {formatNumber(m.successfulRequests)} + + = 99 + ? "text-green-500" + : mRate >= 95 + ? "text-amber-500" + : "text-red-500" + } + > + {mRate.toFixed(1)}% + + + {formatLatency(m.avgLatencyMs)} + +
+
+
+ No provider data recorded yet. +
+
+ + + {/* In-Memory Metrics */} + {data?.comboMetrics && Object.keys(data.comboMetrics).length > 0 && ( + +
+ + + + + + + + + + + + {Object.entries(data.comboMetrics).map(([name, m]: [string, any]) => ( + + + + + + + + ))} + +
ComboRequestsAvg TTFTAvg TotalSuccess
{name} + {formatNumber(m.requestCount ?? m.totalRequests ?? 0)} + + {formatLatency(m.avgTtft ?? m.ttftMs ?? null)} + + {formatLatency(m.avgLatency ?? m.avgTotalMs ?? null)} + + {typeof m.successRate === "number" ? `${(m.successRate * 100).toFixed(1)}%` : "—"} +
+
+ + )} + + {/* Telemetry Phase Breakdown */} + {data?.telemetry && Object.keys(data.telemetry).length > 0 && ( + +
+ {Object.entries(data.telemetry).map(([key, val]: [string, any]) => { + if (val == null || typeof val === "object") return null; + return ( +
+

+ {key.replace(/([A-Z])/g, " $1").replace(/_/g, " ")} +

+

+ {typeof val === "number" ? formatNumber(val) : String(val)} +

+
+ ); + })} +
+ + )} +
+ ); +} diff --git a/src/app/api/provider-stats/route.ts b/src/app/api/provider-stats/route.ts new file mode 100644 index 0000000000..738b9f4747 --- /dev/null +++ b/src/app/api/provider-stats/route.ts @@ -0,0 +1,54 @@ +import { NextResponse } from "next/server"; +import { getProviderCallStats, getModelCallStats } from "@/lib/db/providerStats"; +import { AI_PROVIDERS } from "@/shared/constants/providers"; + +export async function GET() { + try { + // Hard Rule #5: SQL lives in src/lib/db/providerStats.ts, not inline here. + const providerStats = getProviderCallStats(); + const modelStats = getModelCallStats(); + + let comboMetrics: Record = {}; + try { + const { getAllComboMetrics } = await import( + "@omniroute/open-sse/services/comboMetrics.ts" + ); + comboMetrics = getAllComboMetrics() as Record; + } catch {} + + let telemetry: Record = {}; + try { + const { getTelemetrySummary } = await import("@/shared/utils/requestTelemetry"); + telemetry = getTelemetrySummary(300000) as Record; + } catch {} + + let toolLatency: Record = {}; + try { + const { getToolLatencyByProvider } = await import( + "@omniroute/open-sse/services/toolLatencyTracker" + ); + toolLatency = getToolLatencyByProvider() as Record; + } catch {} + + const resolveName = (provider: string, nodeName: string | null) => { + if (nodeName?.trim()) return nodeName.trim(); + const info = AI_PROVIDERS[provider as keyof typeof AI_PROVIDERS]; + return info?.name || provider; + }; + + const providers = providerStats.map((p: any) => ({ + ...p, + provider: resolveName(p.provider, p.nodeName), + })); + + const models = modelStats.map((m: any) => ({ + ...m, + provider: resolveName(m.provider, m.nodeName), + })); + + return NextResponse.json({ providers, models, comboMetrics, telemetry, toolLatency }); + } catch (error) { + console.error("Error fetching provider stats:", error); + return NextResponse.json({ error: "Failed to fetch provider stats" }, { status: 500 }); + } +} diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json index f4299c15c2..767ab319f7 100644 --- a/src/i18n/messages/en.json +++ b/src/i18n/messages/en.json @@ -1027,6 +1027,8 @@ "analyticsCompressionSubtitle": "Token savings stats", "analyticsSearchSubtitle": "Search tool analytics", "analyticsEvalsSubtitle": "Eval suite results", + "providerStats": "Provider Stats", + "providerStatsSubtitle": "Latency and performance metrics", "logsSubtitle": "Application logs", "logsProxySubtitle": "Proxy traffic logs", "consoleLogsSubtitle": "Console output", diff --git a/src/lib/db/providerStats.ts b/src/lib/db/providerStats.ts new file mode 100644 index 0000000000..65d7dc8a4c --- /dev/null +++ b/src/lib/db/providerStats.ts @@ -0,0 +1,68 @@ +import { getDbInstance } from "./core"; + +/** + * Provider/model call statistics aggregated from `call_logs`. + * + * Hard Rule #5: routes must not embed raw SQL — these queries live here so the + * /api/provider-stats route can delegate. Read-only aggregation; no writes. + */ + +export interface ProviderCallStat { + provider: string; + nodeName: string | null; + totalRequests: number; + successfulRequests: number; + avgLatencyMs: number | null; + totalTokensIn: number | null; + totalTokensOut: number | null; +} + +export interface ModelCallStat { + provider: string; + nodeName: string | null; + model: string; + requests: number; + avgLatencyMs: number | null; + successfulRequests: number; +} + +export function getProviderCallStats(): ProviderCallStat[] { + const db = getDbInstance(); + return db + .prepare( + `SELECT + c.provider, + pn.name AS nodeName, + COUNT(*) AS totalRequests, + SUM(CASE WHEN c.status >= 200 AND c.status < 400 THEN 1 ELSE 0 END) AS successfulRequests, + ROUND(AVG(c.duration)) AS avgLatencyMs, + SUM(c.tokens_in) AS totalTokensIn, + SUM(c.tokens_out) AS totalTokensOut + FROM call_logs c + LEFT JOIN provider_nodes pn ON pn.id = c.provider + WHERE c.provider IS NOT NULL AND c.provider != '-' + GROUP BY c.provider + ORDER BY totalRequests DESC` + ) + .all() as ProviderCallStat[]; +} + +export function getModelCallStats(): ModelCallStat[] { + const db = getDbInstance(); + return db + .prepare( + `SELECT + c.provider, + pn.name AS nodeName, + c.model, + COUNT(*) AS requests, + ROUND(AVG(c.duration)) AS avgLatencyMs, + SUM(CASE WHEN c.status >= 200 AND c.status < 400 THEN 1 ELSE 0 END) AS successfulRequests + FROM call_logs c + LEFT JOIN provider_nodes pn ON pn.id = c.provider + WHERE c.provider IS NOT NULL AND c.model IS NOT NULL + GROUP BY c.provider, c.model + ORDER BY c.provider, requests DESC` + ) + .all() as ModelCallStat[]; +} diff --git a/src/shared/constants/sidebarVisibility.ts b/src/shared/constants/sidebarVisibility.ts index 42aed0cfdc..5dd9d328e4 100644 --- a/src/shared/constants/sidebarVisibility.ts +++ b/src/shared/constants/sidebarVisibility.ts @@ -35,6 +35,7 @@ export const HIDEABLE_SIDEBAR_ITEM_IDS = [ "analytics-compression", "analytics-search", "analytics-evals", + "provider-stats", // Monitoring — flat "activity", "logs", @@ -374,6 +375,13 @@ const ANALYTICS_ITEMS: readonly SidebarItemDefinition[] = [ subtitleKey: "analyticsEvalsSubtitle", icon: "labs", }, + { + id: "provider-stats", + href: "/dashboard/provider-stats", + i18nKey: "providerStats", + subtitleKey: "providerStatsSubtitle", + icon: "speed", + }, ]; const MONITORING_ITEMS: readonly SidebarItemDefinition[] = [ diff --git a/tests/unit/db-provider-stats.test.ts b/tests/unit/db-provider-stats.test.ts new file mode 100644 index 0000000000..398351084a --- /dev/null +++ b/tests/unit/db-provider-stats.test.ts @@ -0,0 +1,104 @@ +/** + * #3175 — provider/model call statistics aggregated from call_logs. + * Guards the SQL extracted out of the /api/provider-stats route (Hard Rule #5). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omni-db-provider-stats-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const stats = await import("../../src/lib/db/providerStats.ts"); + +function insertCallLog(row: Record) { + const db = core.getDbInstance(); + const full = { + method: "POST", + path: "/v1/chat/completions", + status: 200, + model: "openai/gpt-4.1", + requested_model: null, + provider: "openai", + account: null, + connection_id: null, + duration: 100, + tokens_in: 10, + tokens_out: 20, + cache_source: "upstream", + source_format: null, + target_format: null, + api_key_id: null, + api_key_name: null, + combo_name: null, + combo_step_id: null, + combo_execution_key: null, + error_summary: null, + detail_state: "none", + artifact_relpath: null, + artifact_size_bytes: null, + artifact_sha256: null, + has_request_body: 0, + has_response_body: 0, + has_pipeline_details: 0, + request_summary: null, + ...row, + id: row.id ?? `log-${Math.random().toString(16).slice(2)}`, + timestamp: row.timestamp ?? new Date().toISOString(), + }; + db.prepare( + `INSERT INTO call_logs ( + id, timestamp, method, path, status, model, requested_model, provider, account, + connection_id, duration, tokens_in, tokens_out, cache_source, source_format, target_format, + api_key_id, api_key_name, combo_name, combo_step_id, combo_execution_key, + error_summary, detail_state, artifact_relpath, artifact_size_bytes, artifact_sha256, + has_request_body, has_response_body, has_pipeline_details, request_summary + ) VALUES ( + @id, @timestamp, @method, @path, @status, @model, @requested_model, @provider, @account, + @connection_id, @duration, @tokens_in, @tokens_out, @cache_source, @source_format, @target_format, + @api_key_id, @api_key_name, @combo_name, @combo_step_id, @combo_execution_key, + @error_summary, @detail_state, @artifact_relpath, @artifact_size_bytes, @artifact_sha256, + @has_request_body, @has_response_body, @has_pipeline_details, @request_summary + )` + ).run(full); +} + +test.before(() => { + core.resetDbInstance(); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("#3175 getProviderCallStats aggregates totals, success and latency per provider", () => { + insertCallLog({ provider: "openai", status: 200, duration: 100, tokens_in: 10, tokens_out: 20 }); + insertCallLog({ provider: "openai", status: 500, duration: 300, tokens_in: 5, tokens_out: 0 }); + insertCallLog({ provider: "anthropic", status: 200, duration: 50, tokens_in: 1, tokens_out: 2 }); + // excluded: provider '-' / null + insertCallLog({ provider: "-", status: 200 }); + + const rows = stats.getProviderCallStats(); + const openai = rows.find((r) => r.provider === "openai"); + assert.ok(openai, "openai stats present"); + assert.equal(openai.totalRequests, 2); + assert.equal(openai.successfulRequests, 1); // only the 200 + assert.equal(openai.avgLatencyMs, 200); // (100+300)/2 + assert.equal(openai.totalTokensIn, 15); + assert.equal(openai.totalTokensOut, 20); + assert.ok(!rows.some((r) => r.provider === "-"), "provider '-' is excluded"); + // ordered by totalRequests desc → openai (2) before anthropic (1) + assert.equal(rows[0].provider, "openai"); +}); + +test("#3175 getModelCallStats groups by provider+model", () => { + const rows = stats.getModelCallStats(); + const m = rows.find((r) => r.model === "openai/gpt-4.1" && r.provider === "openai"); + assert.ok(m, "model row present"); + assert.equal(m.requests, 2); + assert.equal(m.successfulRequests, 1); +});