From 9a8404b7339da7c084b398a9402fded00dddf573 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Thu, 23 Apr 2026 14:40:01 -0300 Subject: [PATCH] fix(evals): persist run history and secure eval management routes Store eval executions with target metadata, expose aggregated scorecard and recent run history endpoints, and return dashboard-ready eval data including target options and API key metadata. Also require management auth for eval read endpoints and preserve per-case latency, errors, and output snippets so historical results are more reliable and easier to inspect. --- .../dashboard/usage/components/EvalsTab.tsx | 1169 +++++++++++------ src/app/api/evals/[suiteId]/route.ts | 6 +- src/app/api/evals/route.ts | 89 +- src/app/api/evals/scorecard/route.ts | 22 + src/lib/evals/evalRunner.ts | 22 +- src/lib/evals/runtime.ts | 311 +++++ src/lib/localDb.ts | 15 + tests/unit/evals-history.test.ts | 100 ++ tests/unit/evals-route.test.ts | 95 ++ 9 files changed, 1441 insertions(+), 388 deletions(-) create mode 100644 src/app/api/evals/scorecard/route.ts create mode 100644 src/lib/evals/runtime.ts create mode 100644 tests/unit/evals-history.test.ts create mode 100644 tests/unit/evals-route.test.ts diff --git a/src/app/(dashboard)/dashboard/usage/components/EvalsTab.tsx b/src/app/(dashboard)/dashboard/usage/components/EvalsTab.tsx index bb3af0ae16..915ce5376b 100644 --- a/src/app/(dashboard)/dashboard/usage/components/EvalsTab.tsx +++ b/src/app/(dashboard)/dashboard/usage/components/EvalsTab.tsx @@ -1,20 +1,109 @@ "use client"; +import { useEffect, useState } from "react"; import { useTranslations } from "next-intl"; - -/** - * EvalsTab — Batch F - * - * Lists evaluation suites, runs evals against real LLM endpoints, - * and shows results. - * API: GET/POST /api/evals, GET /api/evals/[suiteId] - */ - -import { useState, useEffect, useCallback } from "react"; -import { Card, Button, EmptyState, DataTable, FilterBar } from "@/shared/components"; +import { Card, Button, EmptyState, DataTable, FilterBar, Select } from "@/shared/components"; import { useNotificationStore } from "@/store/notificationStore"; -// ── Strategy config for visual legend ────────────────────────────────── +type EvalTargetType = "suite-default" | "model" | "combo"; + +interface EvalTargetOption { + key: string; + type: EvalTargetType; + id: string | null; + label: string; + description: string; +} + +interface EvalApiKeyOption { + id: string; + name: string; + isActive: boolean; +} + +interface EvalCasePreview { + id: string; + name: string; + model?: string; + input?: { + messages?: Array<{ role: string; content: string }>; + }; + expected?: { + strategy?: string; + value?: string; + }; + tags?: string[]; +} + +interface EvalSuite { + id: string; + name: string; + description?: string; + caseCount?: number; + cases?: EvalCasePreview[]; +} + +interface EvalResult { + caseId: string; + caseName: string; + passed: boolean; + durationMs: number; + error?: string; + details?: { + expected?: string; + actual?: string; + actualSnippet?: string; + searchTerm?: string; + pattern?: string; + }; +} + +interface EvalRunSummary { + total: number; + passed: number; + failed: number; + passRate: number; +} + +interface EvalRun { + id: string; + runGroupId: string | null; + suiteId: string; + suiteName: string; + target: { + type: EvalTargetType; + id: string | null; + key: string; + label: string; + }; + avgLatencyMs: number; + summary: EvalRunSummary; + results: EvalResult[]; + outputs: Record; + createdAt: string; +} + +interface EvalScorecard { + suites: number; + totalCases: number; + totalPassed: number; + overallPassRate: number; + perSuite: Array<{ id: string; name: string; passRate: number }>; +} + +interface EvalSuiteRunState { + runs: EvalRun[]; + scorecard: EvalScorecard | null; +} + +interface EvalsDashboardPayload { + suites: EvalSuite[]; + recentRuns: EvalRun[]; + scorecard: EvalScorecard | null; + targets: EvalTargetOption[]; + apiKeys: EvalApiKeyOption[]; +} + const STRATEGIES = [ { name: "contains", @@ -50,154 +139,278 @@ const STRATEGIES = [ }, ]; +const RESULT_COLUMNS = [ + { key: "caseName", labelKey: "columnCase" }, + { key: "status", labelKey: "columnStatus" }, + { key: "durationMs", labelKey: "columnLatency" }, + { key: "details", labelKey: "columnDetails" }, +]; + +const HISTORY_COLUMNS = [ + { key: "suiteName", labelKey: "historyColumnSuiteName" }, + { key: "target", labelKey: "historyColumnTarget" }, + { key: "passRate", labelKey: "historyColumnPassRate" }, + { key: "avgLatencyMs", labelKey: "historyColumnAvgLatencyMs" }, + { key: "createdAt", labelKey: "historyColumnCreatedAt" }, +]; + +const NO_COMPARE_TARGET = "__none__"; +const AUTO_API_KEY = "__auto__"; + +function getTargetLabel( + target: { type: EvalTargetType; id: string | null }, + t: (key: string, values?: Record) => string +): string { + if (target.type === "combo") { + return `${t("targetTypeCombo")}: ${target.id || "—"}`; + } + + if (target.type === "model") { + return `${t("targetTypeModel")}: ${target.id || "—"}`; + } + + return t("targetSuiteDefaults"); +} + +function parseTargetKey(value: string): { type: EvalTargetType; id: string | null } { + const [rawType, ...rawId] = value.split(":"); + const idValue = rawId.join(":"); + + if (rawType === "combo") { + return { type: "combo", id: idValue || null }; + } + + if (rawType === "model") { + return { type: "model", id: idValue || null }; + } + + return { type: "suite-default", id: null }; +} + +function formatTimestamp(value: string): string { + try { + return new Intl.DateTimeFormat(undefined, { + dateStyle: "short", + timeStyle: "short", + }).format(new Date(value)); + } catch { + return value; + } +} + +function getResultDetails( + result: EvalResult, + t: (key: string, values?: Record) => string +): string { + if (result.error) { + return `${t("resultErrorLabel")}: ${result.error}`; + } + + if (result.details?.searchTerm) { + return t("detailsContains", { term: result.details.searchTerm }); + } + + if (result.details?.pattern) { + return t("detailsRegex", { pattern: result.details.pattern }); + } + + if (result.details?.expected) { + return t("detailsExpected", { + expected: String(result.details.expected).slice(0, 60), + }); + } + + if (result.details?.actualSnippet) { + return t("actualOutputLabel", { + value: String(result.details.actualSnippet).slice(0, 60), + }); + } + + return "—"; +} + export default function EvalsTab() { const t = useTranslations("usage"); - const [suites, setSuites] = useState([]); - const [apiKey, setApiKey] = useState(null); - const [loading, setLoading] = useState(true); - const [running, setRunning] = useState(null); - const [progress, setProgress] = useState({ current: 0, total: 0 }); - const [results, setResults] = useState({}); - const [search, setSearch] = useState(""); - const [expanded, setExpanded] = useState(null); - const [showHowItWorks, setShowHowItWorks] = useState(false); const notify = useNotificationStore(); - - const fetchSuites = useCallback(async () => { - try { - const res = await fetch("/api/evals"); - if (res.ok) { - const data = await res.json(); - setSuites(Array.isArray(data) ? data : data.suites || []); - } - } catch { - // silent - } finally { - setLoading(false); - } - }, []); - - const fetchApiKey = useCallback(async () => { - try { - const res = await fetch("/api/keys"); - if (!res.ok) return; - const data = await res.json(); - const firstKey = data?.keys?.[0]?.key || null; - setApiKey(firstKey); - } catch { - // silent - } - }, []); + const [suites, setSuites] = useState([]); + const [recentRuns, setRecentRuns] = useState([]); + const [scorecard, setScorecard] = useState(null); + const [targetOptions, setTargetOptions] = useState([]); + const [apiKeys, setApiKeys] = useState([]); + const [selectedTargetKey, setSelectedTargetKey] = useState("suite-default:__default__"); + const [compareTargetKey, setCompareTargetKey] = useState(""); + const [selectedApiKeyId, setSelectedApiKeyId] = useState(""); + const [suiteRuns, setSuiteRuns] = useState>({}); + const [loading, setLoading] = useState(true); + const [running, setRunning] = useState(null); + const [search, setSearch] = useState(""); + const [expanded, setExpanded] = useState(null); + const [showHowItWorks, setShowHowItWorks] = useState(false); useEffect(() => { - fetchSuites(); - fetchApiKey(); - }, [fetchSuites, fetchApiKey]); + let isMounted = true; - /** - * Call the proxy LLM endpoint for a single eval case. - * Returns the assistant's response text. - */ - const callLLM = async (evalCase) => { - try { - const headers: any = { "Content-Type": "application/json" }; - if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + async function loadDashboard() { + try { + const response = await fetch("/api/evals"); + if (!response.ok) { + throw new Error(t("notifyEvalRunFailed")); + } - const res = await fetch("/v1/chat/completions", { - method: "POST", - headers, - body: JSON.stringify({ - model: evalCase.model || "gpt-4o", - messages: evalCase.input?.messages || [], - max_tokens: 512, - stream: false, - }), - }); - if (!res.ok) { - return `[ERROR: HTTP ${res.status}]`; + const payload = (await response.json()) as EvalsDashboardPayload; + if (!isMounted) return; + + setSuites(Array.isArray(payload.suites) ? payload.suites : []); + setRecentRuns(Array.isArray(payload.recentRuns) ? payload.recentRuns : []); + setScorecard(payload.scorecard || null); + setTargetOptions(Array.isArray(payload.targets) ? payload.targets : []); + setApiKeys(Array.isArray(payload.apiKeys) ? payload.apiKeys : []); + } catch { + if (isMounted) { + notify.error(t("notifyEvalLoadFailed")); + } + } finally { + if (isMounted) { + setLoading(false); + } } - const data = await res.json(); - return data.choices?.[0]?.message?.content || "[No content returned]"; - } catch (err) { - return `[ERROR: ${err.message}]`; } - }; - /** - * Run all cases: call LLM for each, then submit outputs for evaluation. - */ - const handleRunEval = async (suite) => { + loadDashboard(); + + return () => { + isMounted = false; + }; + }, [notify, t]); + + useEffect(() => { + if (targetOptions.length === 0) return; + if (targetOptions.some((option) => option.key === selectedTargetKey)) return; + setSelectedTargetKey(targetOptions[0]?.key || "suite-default:__default__"); + }, [selectedTargetKey, targetOptions]); + + useEffect(() => { + if (!compareTargetKey) return; + if (compareTargetKey === selectedTargetKey) { + setCompareTargetKey(""); + } + }, [compareTargetKey, selectedTargetKey]); + + const filteredSuites = !search.trim() + ? suites + : suites.filter((suite) => { + const term = search.toLowerCase(); + return ( + suite.name?.toLowerCase().includes(term) || + suite.id?.toLowerCase().includes(term) || + suite.description?.toLowerCase().includes(term) + ); + }); + + const totalCases = suites.reduce( + (sum, suite) => sum + (suite.cases?.length || suite.caseCount || 0), + 0 + ); + + const uniqueModels = [ + ...new Set( + suites + .flatMap((suite) => suite.cases || []) + .map((evalCase) => evalCase.model) + .filter((model): model is string => typeof model === "string" && model.trim().length > 0) + ), + ]; + + const compareOptions = targetOptions.filter((option) => option.key !== selectedTargetKey); + + async function refreshDashboard() { + const response = await fetch("/api/evals"); + if (!response.ok) { + throw new Error(t("notifyEvalLoadFailed")); + } + const payload = (await response.json()) as EvalsDashboardPayload; + setRecentRuns(Array.isArray(payload.recentRuns) ? payload.recentRuns : []); + setScorecard(payload.scorecard || null); + setTargetOptions(Array.isArray(payload.targets) ? payload.targets : []); + setApiKeys(Array.isArray(payload.apiKeys) ? payload.apiKeys : []); + setSuites(Array.isArray(payload.suites) ? payload.suites : []); + } + + async function handleRunEval(suite: EvalSuite) { const cases = suite.cases || []; if (cases.length === 0) { notify.warning(t("notifyNoTestCases")); return; } + if (compareTargetKey && compareTargetKey === selectedTargetKey) { + notify.warning(t("notifySelectDifferentCompareTarget")); + return; + } + setRunning(suite.id); - setProgress({ current: 0, total: cases.length }); try { - // Step 1: Call LLM for each case and collect outputs - const outputs = {}; - for (let i = 0; i < cases.length; i++) { - setProgress({ current: i + 1, total: cases.length }); - const response = await callLLM(cases[i]); - outputs[cases[i].id] = response; - } - - // Step 2: Submit outputs for evaluation - const res = await fetch("/api/evals", { + const response = await fetch("/api/evals", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ suiteId: suite.id, - outputs, + target: parseTargetKey(selectedTargetKey), + ...(compareTargetKey ? { compareTarget: parseTargetKey(compareTargetKey) } : {}), + ...(selectedApiKeyId ? { apiKeyId: selectedApiKeyId } : {}), }), }); - const data = await res.json(); - setResults((prev) => ({ ...prev, [suite.id]: data })); - // Notify with results - if (data.summary) { - const { passed, failed, total } = data.summary; - if (failed === 0) { - notify.success( - t("notifyAllCasesPassed", { total }), - t("notifyEvalTitle", { name: suite.name || suite.id }) - ); - } else { - notify.warning( - t("notifySomeCasesFailed", { passed, total, failed }), - t("notifyEvalTitle", { name: suite.name || suite.id }) - ); - } + const payload = await response.json(); + if (!response.ok) { + throw new Error( + payload?.error?.message || payload?.error || payload?.message || t("notifyEvalRunFailed") + ); } - // Auto-expand to show results + const runs = Array.isArray(payload.runs) ? (payload.runs as EvalRun[]) : []; + const comparisonScorecard = (payload.scorecard || null) as EvalScorecard | null; + setSuiteRuns((prev) => ({ + ...prev, + [suite.id]: { + runs, + scorecard: comparisonScorecard, + }, + })); setExpanded(suite.id); - } catch { - notify.error(t("notifyEvalRunFailed")); + + if (Array.isArray(payload.recentRuns)) { + setRecentRuns(payload.recentRuns as EvalRun[]); + } else { + await refreshDashboard(); + } + + if (payload.historyScorecard) { + setScorecard(payload.historyScorecard as EvalScorecard); + } + + const primaryRun = runs[0]; + if (primaryRun) { + const score = primaryRun.summary.passRate; + notify.success( + compareTargetKey + ? t("compareCompletedWithScore", { score }) + : t("runCompletedWithScore", { score }), + t("notifyEvalTitle", { name: suite.name || suite.id }) + ); + } + } catch (error: any) { + notify.error( + t("notifyEvalRunFailedWithReason", { + reason: error?.message || t("notAvailableSymbol"), + }), + t("notifyEvalTitle", { name: suite.name || suite.id }) + ); } finally { setRunning(null); - setProgress({ current: 0, total: 0 }); } - }; - - const filtered = suites.filter((s) => { - if (!search) return true; - return ( - s.name?.toLowerCase().includes(search.toLowerCase()) || - s.id?.toLowerCase().includes(search.toLowerCase()) - ); - }); - - // Count total cases and unique models across all suites - const totalCases = suites.reduce((sum, s) => sum + (s.cases?.length || s.caseCount || 0), 0); - const uniqueModels: string[] = [ - ...new Set( - suites.flatMap((s: any) => (s.cases || []).map((c: any) => c.model)).filter(Boolean) - ), - ]; + } if (loading) { return ( @@ -211,7 +424,6 @@ export default function EvalsTab() { if (suites.length === 0) { return (
- {/* Hero Section — always visible */} - {/* Hero Section */} - {/* Stats Bar */}
@@ -264,10 +467,133 @@ export default function EvalsTab() {
- {/* {t("howItWorks")} — Collapsible */} + +
+
+ route +
+
+

{t("evalControlsTitle")}

+

{t("evalControlsHint")}

+
+
+ +
+ + setCompareTargetKey( + event.target.value === NO_COMPARE_TARGET ? "" : event.target.value + ) + } + options={[ + { + value: NO_COMPARE_TARGET, + label: t("evalCompareOptional"), + }, + ...compareOptions.map((option) => ({ + value: option.key, + label: getTargetLabel(option, t), + })), + ]} + hint={t("evalCompareHint")} + /> +