diff --git a/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx b/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx new file mode 100644 index 0000000000..5c9ff6c064 --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx @@ -0,0 +1,115 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx + +import type { StreamMetrics } from "@/shared/schemas/playground"; +import MarkdownMessage from "./MarkdownMessage"; +import ProviderMetrics from "./ProviderMetrics"; + +export type ColumnStatus = "idle" | "streaming" | "done" | "error"; + +export interface CompareColumnData { + id: string; + model: string; + status: ColumnStatus; + metrics: StreamMetrics; + response: string; + errorMessage?: string; +} + +interface CompareColumnProps { + column: CompareColumnData; + onCancel: (id: string) => void; + onRemove: (id: string) => void; +} + +/** + * CompareColumn — a single column in the Compare tab. + * Shows the model name, streaming response (via MarkdownMessage), and ProviderMetrics. + */ +export default function CompareColumn({ column, onCancel, onRemove }: CompareColumnProps) { + const { id, model, status, metrics, response, errorMessage } = column; + + return ( +
+ {/* Column header */} +
+
+ {/* Status indicator */} + + + {model || No model} + +
+ +
+ {status === "streaming" && ( + + )} + +
+
+ + {/* Metrics bar */} + {(status === "streaming" || status === "done") && ( +
+ +
+ )} + + {/* Response content */} +
+ {status === "idle" && ( +

+ Ready to run. +

+ )} + + {status === "error" && ( +
+ Error: + {errorMessage ?? "Unknown error occurred."} +
+ )} + + {status === "streaming" && response === "" && ( +
+ + Waiting for response… +
+ )} + + {(status === "streaming" || status === "done") && response !== "" && ( + + )} +
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx b/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx new file mode 100644 index 0000000000..481f8f191f --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx @@ -0,0 +1,60 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx + +import type { StreamMetrics } from "@/shared/schemas/playground"; + +interface ProviderMetricsProps { + metrics: StreamMetrics; +} + +function formatMs(ms: number | null): string { + if (ms == null) return "—"; + if (ms < 1000) return `${ms.toFixed(0)}ms`; + return `${(ms / 1000).toFixed(2)}s`; +} + +function formatTps(tps: number | null): string { + if (tps == null) return "—"; + return `${tps.toFixed(1)} t/s`; +} + +function formatCost(usd: number | null): string { + if (usd == null) return "—"; + if (usd < 0.001) return `$${(usd * 1000).toFixed(3)}m`; + return `$${usd.toFixed(4)}`; +} + +/** + * ProviderMetrics — displays TTFT, TPS, token counts, and estimated cost. + * + * All metrics are client-perceived (D12): measured from the first SSE chunk + * to the last. Labeled "(estimated)" as required by D13. + */ +export default function ProviderMetrics({ metrics }: ProviderMetricsProps) { + const { ttftMs, tps, tokensIn, tokensOut, costUsd } = metrics; + + return ( +
+ {/* TTFT */} + + TTFT {formatMs(ttftMs)} + + + {/* TPS */} + + · {formatTps(tps)} + + + {/* Token counts */} + + · {tokensIn}↑ {tokensOut}↓ + + + {/* Cost */} + + · {formatCost(costUsd)} (estimated) + +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx b/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx new file mode 100644 index 0000000000..6719b7fc40 --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx @@ -0,0 +1,395 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx + +import { useEffect, useRef, useState } from "react"; +import CompareColumn, { type CompareColumnData, type ColumnStatus } from "../CompareColumn"; +import type { ConfigState } from "../StudioConfigPane"; +import type { StreamMetrics } from "@/shared/schemas/playground"; + +interface CompareTabProps { + configState: ConfigState; +} + +const MAX_COLUMNS = 4; // D10: cap at 4 columns + +interface ColumnState { + id: string; + model: string; + status: ColumnStatus; + metrics: StreamMetrics; + response: string; + errorMessage?: string; +} + +const INITIAL_METRICS: StreamMetrics = { + ttftMs: null, + totalMs: null, + tokensIn: 0, + tokensOut: 0, + tps: null, + costUsd: null, +}; + +/** + * ColumnMetricsTracker — plain (non-React) imperative object for tracking per-column stream metrics. + * Uses simple mutable state (no React hooks) so it can be stored in a ref and called safely + * from async callbacks without violating Rules of Hooks. + */ +class ColumnMetricsTracker { + private startedAt: number | null = null; + private firstChunkAt: number | null = null; + private finishedAt: number | null = null; + private tokensOut = 0; + private tokensIn = 0; + + reset() { + this.startedAt = null; + this.firstChunkAt = null; + this.finishedAt = null; + this.tokensOut = 0; + this.tokensIn = 0; + } + + start() { + this.startedAt = Date.now(); + } + + onFirstChunk() { + if (this.firstChunkAt == null) { + this.firstChunkAt = Date.now(); + } + } + + onChunk(n: number) { + this.tokensOut += n; + } + + finish(usage?: { prompt_tokens?: number; completion_tokens?: number }) { + this.finishedAt = Date.now(); + if (usage?.prompt_tokens != null) this.tokensIn = usage.prompt_tokens; + if (usage?.completion_tokens != null) this.tokensOut = usage.completion_tokens; + } + + getMetrics(): StreamMetrics { + const { startedAt, firstChunkAt, finishedAt, tokensOut, tokensIn } = this; + const ttftMs = startedAt != null && firstChunkAt != null ? firstChunkAt - startedAt : null; + const totalMs = startedAt != null && finishedAt != null ? finishedAt - startedAt : null; + const tps = + totalMs != null && totalMs > 0 && tokensOut > 0 ? (tokensOut / totalMs) * 1000 : null; + return { ttftMs, totalMs, tokensIn, tokensOut, tps, costUsd: null }; + } +} + +/** + * CompareTab — up to 4 parallel streaming columns (D10 + D19 + D22). + * + * Streaming uses native fetch + ReadableStream (not EventSource). + * Each column has its own AbortController. + * Cmd+K (or Ctrl+K) focuses the "add column" model input. + */ +export default function CompareTab({ configState }: CompareTabProps) { + const [columns, setColumns] = useState(() => [ + { + id: crypto.randomUUID(), + model: configState.model, + status: "idle", + metrics: INITIAL_METRICS, + response: "", + }, + ]); + + // AbortControllers per column id + const controllersRef = useRef>(new Map()); + // Metrics trackers per column id (plain class instances, no React hooks) + const metricsTrackersRef = useRef>(new Map()); + + // Input for model name when adding a column + const [newModel, setNewModel] = useState(""); + const addInputRef = useRef(null); + + // Cmd+K / Ctrl+K shortcut to focus model input (D10) + useEffect(() => { + function onKeyDown(e: KeyboardEvent) { + if ((e.metaKey || e.ctrlKey) && e.key === "k") { + e.preventDefault(); + addInputRef.current?.focus(); + } + } + document.addEventListener("keydown", onKeyDown); + return () => document.removeEventListener("keydown", onKeyDown); + }, []); + + function getOrCreateTracker(id: string): ColumnMetricsTracker { + if (!metricsTrackersRef.current.has(id)) { + metricsTrackersRef.current.set(id, new ColumnMetricsTracker()); + } + return metricsTrackersRef.current.get(id)!; + } + + function addColumn() { + if (columns.length >= MAX_COLUMNS) return; + const model = newModel.trim() || configState.model; + const id = crypto.randomUUID(); + setColumns((prev) => [ + ...prev, + { id, model, status: "idle", metrics: INITIAL_METRICS, response: "" }, + ]); + setNewModel(""); + } + + function removeColumn(id: string) { + controllersRef.current.get(id)?.abort(); + controllersRef.current.delete(id); + metricsTrackersRef.current.delete(id); + setColumns((prev) => prev.filter((c) => c.id !== id)); + } + + function cancelColumn(id: string) { + controllersRef.current.get(id)?.abort(); + } + + function cancelAll() { + for (const ctrl of controllersRef.current.values()) { + ctrl.abort(); + } + } + + function updateColumn(id: string, patch: Partial) { + setColumns((prev) => prev.map((c) => (c.id === id ? { ...c, ...patch } : c))); + } + + /** + * Stream a single column's SSE request (D19). + */ + async function streamColumn(col: ColumnState): Promise { + const tracker = getOrCreateTracker(col.id); + tracker.reset(); + + const controller = new AbortController(); + controllersRef.current.set(col.id, controller); + + updateColumn(col.id, { status: "streaming", response: "", metrics: INITIAL_METRICS }); + tracker.start(); + + const body: Record = { + model: col.model, + stream: true, + messages: [ + ...(configState.systemPrompt + ? [{ role: "system", content: configState.systemPrompt }] + : []), + ], + }; + + const { params } = configState; + if (params.temperature != null) body["temperature"] = params.temperature; + if (params.max_tokens != null) body["max_tokens"] = params.max_tokens; + if (params.top_p != null) body["top_p"] = params.top_p; + if (params.presence_penalty != null) body["presence_penalty"] = params.presence_penalty; + if (params.frequency_penalty != null) body["frequency_penalty"] = params.frequency_penalty; + if (params.seed != null) body["seed"] = params.seed; + if (params.stop != null) body["stop"] = params.stop; + + let accumulated = ""; + let firstChunk = true; + + try { + const res = await fetch(`${configState.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal, + }); + + if (!res.ok) { + const text = await res.text().catch(() => "Unknown error"); + throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`); + } + + if (!res.body) throw new Error("No response body"); + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice("data: ".length).trim(); + if (data === "[DONE]") continue; + + let parsed: Record; + try { + parsed = JSON.parse(data) as Record; + } catch { + continue; + } + + const choices = + (parsed["choices"] as Array> | undefined) ?? []; + const delta = choices[0]?.["delta"] as Record | undefined; + const content = delta?.["content"]; + + if (typeof content === "string" && content !== "") { + if (firstChunk) { + tracker.onFirstChunk(); + firstChunk = false; + } + accumulated += content; + tracker.onChunk(1); + + updateColumn(col.id, { + response: accumulated, + metrics: tracker.getMetrics(), + }); + } + + const usage = parsed["usage"] as + | { prompt_tokens?: number; completion_tokens?: number } + | undefined; + if (usage != null) { + tracker.finish(usage); + } + } + } + + tracker.finish(); + updateColumn(col.id, { + status: "done", + metrics: tracker.getMetrics(), + }); + } catch (err) { + if (controller.signal.aborted) { + updateColumn(col.id, { status: "idle" }); + return; + } + const message = err instanceof Error ? err.message : String(err); + updateColumn(col.id, { + status: "error", + errorMessage: message, + metrics: tracker.getMetrics(), + }); + } + } + + /** + * Run all columns in parallel (D10). + */ + async function runAll() { + cancelAll(); + await Promise.allSettled(columns.map((col) => streamColumn(col))); + } + + const isAnyStreaming = columns.some((c) => c.status === "streaming"); + const atColumnLimit = columns.length >= MAX_COLUMNS; + + const displayColumns: CompareColumnData[] = columns.map((c) => ({ + id: c.id, + model: c.model, + status: c.status, + metrics: c.metrics, + response: c.response, + errorMessage: c.errorMessage, + })); + + return ( +
+ {/* Compare toolbar */} +
+ {/* Run / Cancel all */} + {isAnyStreaming ? ( + + ) : ( + + )} + + {/* Add column */} +
+ setNewModel(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") addColumn(); + }} + placeholder="Model (Cmd+K)…" + disabled={atColumnLimit} + className="text-xs bg-surface border border-border rounded px-2 py-1.5 focus:outline-none focus:ring-1 focus:ring-primary text-text-main w-40 disabled:opacity-40" + aria-label="Model name for new column" + /> + +
+ + {/* Column count indicator */} + + {columns.length}/{MAX_COLUMNS} columns + +
+ + {/* Columns area */} +
+ {displayColumns.map((col) => ( + + ))} + + {columns.length === 0 && ( +
+
+ + compare + +

Add a model column to compare

+

+ Up to {MAX_COLUMNS} models simultaneously +

+
+
+ )} +
+
+ ); +}