From 05a0dceba73797d9e3179211e4bb51d570839b20 Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Thu, 28 May 2026 11:20:06 -0300 Subject: [PATCH 1/7] feat(playground): add CompareTab with parallel streams + metrics --- .../playground/components/CompareColumn.tsx | 115 +++++ .../playground/components/ProviderMetrics.tsx | 60 +++ .../playground/components/tabs/CompareTab.tsx | 395 ++++++++++++++++++ 3 files changed, 570 insertions(+) create mode 100644 src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx create mode 100644 src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx create mode 100644 src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx diff --git a/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx b/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx new file mode 100644 index 0000000000..5c9ff6c064 --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx @@ -0,0 +1,115 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/CompareColumn.tsx + +import type { StreamMetrics } from "@/shared/schemas/playground"; +import MarkdownMessage from "./MarkdownMessage"; +import ProviderMetrics from "./ProviderMetrics"; + +export type ColumnStatus = "idle" | "streaming" | "done" | "error"; + +export interface CompareColumnData { + id: string; + model: string; + status: ColumnStatus; + metrics: StreamMetrics; + response: string; + errorMessage?: string; +} + +interface CompareColumnProps { + column: CompareColumnData; + onCancel: (id: string) => void; + onRemove: (id: string) => void; +} + +/** + * CompareColumn — a single column in the Compare tab. + * Shows the model name, streaming response (via MarkdownMessage), and ProviderMetrics. + */ +export default function CompareColumn({ column, onCancel, onRemove }: CompareColumnProps) { + const { id, model, status, metrics, response, errorMessage } = column; + + return ( +
+ {/* Column header */} +
+
+ {/* Status indicator */} + + + {model || No model} + +
+ +
+ {status === "streaming" && ( + + )} + +
+
+ + {/* Metrics bar */} + {(status === "streaming" || status === "done") && ( +
+ +
+ )} + + {/* Response content */} +
+ {status === "idle" && ( +

+ Ready to run. +

+ )} + + {status === "error" && ( +
+ Error: + {errorMessage ?? "Unknown error occurred."} +
+ )} + + {status === "streaming" && response === "" && ( +
+ + Waiting for response… +
+ )} + + {(status === "streaming" || status === "done") && response !== "" && ( + + )} +
+
+ ); +} diff --git a/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx b/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx new file mode 100644 index 0000000000..481f8f191f --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx @@ -0,0 +1,60 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/ProviderMetrics.tsx + +import type { StreamMetrics } from "@/shared/schemas/playground"; + +interface ProviderMetricsProps { + metrics: StreamMetrics; +} + +function formatMs(ms: number | null): string { + if (ms == null) return "—"; + if (ms < 1000) return `${ms.toFixed(0)}ms`; + return `${(ms / 1000).toFixed(2)}s`; +} + +function formatTps(tps: number | null): string { + if (tps == null) return "—"; + return `${tps.toFixed(1)} t/s`; +} + +function formatCost(usd: number | null): string { + if (usd == null) return "—"; + if (usd < 0.001) return `$${(usd * 1000).toFixed(3)}m`; + return `$${usd.toFixed(4)}`; +} + +/** + * ProviderMetrics — displays TTFT, TPS, token counts, and estimated cost. + * + * All metrics are client-perceived (D12): measured from the first SSE chunk + * to the last. Labeled "(estimated)" as required by D13. + */ +export default function ProviderMetrics({ metrics }: ProviderMetricsProps) { + const { ttftMs, tps, tokensIn, tokensOut, costUsd } = metrics; + + return ( +
+ {/* TTFT */} + + TTFT {formatMs(ttftMs)} + + + {/* TPS */} + + · {formatTps(tps)} + + + {/* Token counts */} + + · {tokensIn}↑ {tokensOut}↓ + + + {/* Cost */} + + · {formatCost(costUsd)} (estimated) + +
+ ); +} diff --git a/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx b/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx new file mode 100644 index 0000000000..6719b7fc40 --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx @@ -0,0 +1,395 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/tabs/CompareTab.tsx + +import { useEffect, useRef, useState } from "react"; +import CompareColumn, { type CompareColumnData, type ColumnStatus } from "../CompareColumn"; +import type { ConfigState } from "../StudioConfigPane"; +import type { StreamMetrics } from "@/shared/schemas/playground"; + +interface CompareTabProps { + configState: ConfigState; +} + +const MAX_COLUMNS = 4; // D10: cap at 4 columns + +interface ColumnState { + id: string; + model: string; + status: ColumnStatus; + metrics: StreamMetrics; + response: string; + errorMessage?: string; +} + +const INITIAL_METRICS: StreamMetrics = { + ttftMs: null, + totalMs: null, + tokensIn: 0, + tokensOut: 0, + tps: null, + costUsd: null, +}; + +/** + * ColumnMetricsTracker — plain (non-React) imperative object for tracking per-column stream metrics. + * Uses simple mutable state (no React hooks) so it can be stored in a ref and called safely + * from async callbacks without violating Rules of Hooks. + */ +class ColumnMetricsTracker { + private startedAt: number | null = null; + private firstChunkAt: number | null = null; + private finishedAt: number | null = null; + private tokensOut = 0; + private tokensIn = 0; + + reset() { + this.startedAt = null; + this.firstChunkAt = null; + this.finishedAt = null; + this.tokensOut = 0; + this.tokensIn = 0; + } + + start() { + this.startedAt = Date.now(); + } + + onFirstChunk() { + if (this.firstChunkAt == null) { + this.firstChunkAt = Date.now(); + } + } + + onChunk(n: number) { + this.tokensOut += n; + } + + finish(usage?: { prompt_tokens?: number; completion_tokens?: number }) { + this.finishedAt = Date.now(); + if (usage?.prompt_tokens != null) this.tokensIn = usage.prompt_tokens; + if (usage?.completion_tokens != null) this.tokensOut = usage.completion_tokens; + } + + getMetrics(): StreamMetrics { + const { startedAt, firstChunkAt, finishedAt, tokensOut, tokensIn } = this; + const ttftMs = startedAt != null && firstChunkAt != null ? firstChunkAt - startedAt : null; + const totalMs = startedAt != null && finishedAt != null ? finishedAt - startedAt : null; + const tps = + totalMs != null && totalMs > 0 && tokensOut > 0 ? (tokensOut / totalMs) * 1000 : null; + return { ttftMs, totalMs, tokensIn, tokensOut, tps, costUsd: null }; + } +} + +/** + * CompareTab — up to 4 parallel streaming columns (D10 + D19 + D22). + * + * Streaming uses native fetch + ReadableStream (not EventSource). + * Each column has its own AbortController. + * Cmd+K (or Ctrl+K) focuses the "add column" model input. + */ +export default function CompareTab({ configState }: CompareTabProps) { + const [columns, setColumns] = useState(() => [ + { + id: crypto.randomUUID(), + model: configState.model, + status: "idle", + metrics: INITIAL_METRICS, + response: "", + }, + ]); + + // AbortControllers per column id + const controllersRef = useRef>(new Map()); + // Metrics trackers per column id (plain class instances, no React hooks) + const metricsTrackersRef = useRef>(new Map()); + + // Input for model name when adding a column + const [newModel, setNewModel] = useState(""); + const addInputRef = useRef(null); + + // Cmd+K / Ctrl+K shortcut to focus model input (D10) + useEffect(() => { + function onKeyDown(e: KeyboardEvent) { + if ((e.metaKey || e.ctrlKey) && e.key === "k") { + e.preventDefault(); + addInputRef.current?.focus(); + } + } + document.addEventListener("keydown", onKeyDown); + return () => document.removeEventListener("keydown", onKeyDown); + }, []); + + function getOrCreateTracker(id: string): ColumnMetricsTracker { + if (!metricsTrackersRef.current.has(id)) { + metricsTrackersRef.current.set(id, new ColumnMetricsTracker()); + } + return metricsTrackersRef.current.get(id)!; + } + + function addColumn() { + if (columns.length >= MAX_COLUMNS) return; + const model = newModel.trim() || configState.model; + const id = crypto.randomUUID(); + setColumns((prev) => [ + ...prev, + { id, model, status: "idle", metrics: INITIAL_METRICS, response: "" }, + ]); + setNewModel(""); + } + + function removeColumn(id: string) { + controllersRef.current.get(id)?.abort(); + controllersRef.current.delete(id); + metricsTrackersRef.current.delete(id); + setColumns((prev) => prev.filter((c) => c.id !== id)); + } + + function cancelColumn(id: string) { + controllersRef.current.get(id)?.abort(); + } + + function cancelAll() { + for (const ctrl of controllersRef.current.values()) { + ctrl.abort(); + } + } + + function updateColumn(id: string, patch: Partial) { + setColumns((prev) => prev.map((c) => (c.id === id ? { ...c, ...patch } : c))); + } + + /** + * Stream a single column's SSE request (D19). + */ + async function streamColumn(col: ColumnState): Promise { + const tracker = getOrCreateTracker(col.id); + tracker.reset(); + + const controller = new AbortController(); + controllersRef.current.set(col.id, controller); + + updateColumn(col.id, { status: "streaming", response: "", metrics: INITIAL_METRICS }); + tracker.start(); + + const body: Record = { + model: col.model, + stream: true, + messages: [ + ...(configState.systemPrompt + ? [{ role: "system", content: configState.systemPrompt }] + : []), + ], + }; + + const { params } = configState; + if (params.temperature != null) body["temperature"] = params.temperature; + if (params.max_tokens != null) body["max_tokens"] = params.max_tokens; + if (params.top_p != null) body["top_p"] = params.top_p; + if (params.presence_penalty != null) body["presence_penalty"] = params.presence_penalty; + if (params.frequency_penalty != null) body["frequency_penalty"] = params.frequency_penalty; + if (params.seed != null) body["seed"] = params.seed; + if (params.stop != null) body["stop"] = params.stop; + + let accumulated = ""; + let firstChunk = true; + + try { + const res = await fetch(`${configState.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal, + }); + + if (!res.ok) { + const text = await res.text().catch(() => "Unknown error"); + throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`); + } + + if (!res.body) throw new Error("No response body"); + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice("data: ".length).trim(); + if (data === "[DONE]") continue; + + let parsed: Record; + try { + parsed = JSON.parse(data) as Record; + } catch { + continue; + } + + const choices = + (parsed["choices"] as Array> | undefined) ?? []; + const delta = choices[0]?.["delta"] as Record | undefined; + const content = delta?.["content"]; + + if (typeof content === "string" && content !== "") { + if (firstChunk) { + tracker.onFirstChunk(); + firstChunk = false; + } + accumulated += content; + tracker.onChunk(1); + + updateColumn(col.id, { + response: accumulated, + metrics: tracker.getMetrics(), + }); + } + + const usage = parsed["usage"] as + | { prompt_tokens?: number; completion_tokens?: number } + | undefined; + if (usage != null) { + tracker.finish(usage); + } + } + } + + tracker.finish(); + updateColumn(col.id, { + status: "done", + metrics: tracker.getMetrics(), + }); + } catch (err) { + if (controller.signal.aborted) { + updateColumn(col.id, { status: "idle" }); + return; + } + const message = err instanceof Error ? err.message : String(err); + updateColumn(col.id, { + status: "error", + errorMessage: message, + metrics: tracker.getMetrics(), + }); + } + } + + /** + * Run all columns in parallel (D10). + */ + async function runAll() { + cancelAll(); + await Promise.allSettled(columns.map((col) => streamColumn(col))); + } + + const isAnyStreaming = columns.some((c) => c.status === "streaming"); + const atColumnLimit = columns.length >= MAX_COLUMNS; + + const displayColumns: CompareColumnData[] = columns.map((c) => ({ + id: c.id, + model: c.model, + status: c.status, + metrics: c.metrics, + response: c.response, + errorMessage: c.errorMessage, + })); + + return ( +
+ {/* Compare toolbar */} +
+ {/* Run / Cancel all */} + {isAnyStreaming ? ( + + ) : ( + + )} + + {/* Add column */} +
+ setNewModel(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") addColumn(); + }} + placeholder="Model (Cmd+K)…" + disabled={atColumnLimit} + className="text-xs bg-surface border border-border rounded px-2 py-1.5 focus:outline-none focus:ring-1 focus:ring-primary text-text-main w-40 disabled:opacity-40" + aria-label="Model name for new column" + /> + +
+ + {/* Column count indicator */} + + {columns.length}/{MAX_COLUMNS} columns + +
+ + {/* Columns area */} +
+ {displayColumns.map((col) => ( + + ))} + + {columns.length === 0 && ( +
+
+ + compare + +

Add a model column to compare

+

+ Up to {MAX_COLUMNS} models simultaneously +

+
+
+ )} +
+
+ ); +} From 331cc68e4508e5e6a138f3f82e2b800d0d0b5f6e Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Thu, 28 May 2026 11:20:10 -0300 Subject: [PATCH 2/7] feat(playground): add BuildTab with tools + structured output --- .../playground/components/tabs/BuildTab.tsx | 399 ++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 src/app/(dashboard)/dashboard/playground/components/tabs/BuildTab.tsx diff --git a/src/app/(dashboard)/dashboard/playground/components/tabs/BuildTab.tsx b/src/app/(dashboard)/dashboard/playground/components/tabs/BuildTab.tsx new file mode 100644 index 0000000000..bf67889bf5 --- /dev/null +++ b/src/app/(dashboard)/dashboard/playground/components/tabs/BuildTab.tsx @@ -0,0 +1,399 @@ +"use client"; + +// src/app/(dashboard)/dashboard/playground/components/tabs/BuildTab.tsx + +import { useRef, useState } from "react"; +import { useToolsBuilder } from "../../hooks/useToolsBuilder"; +import { useStructuredOutput } from "../../hooks/useStructuredOutput"; +import ToolsBuilder from "../ToolsBuilder"; +import StructuredOutputEditor from "../StructuredOutputEditor"; +import MarkdownMessage from "../MarkdownMessage"; +import type { ConfigState } from "../StudioConfigPane"; + +interface BuildTabProps { + configState: ConfigState; +} + +interface ToolCall { + id: string; + function: { + name: string; + arguments: string; + }; +} + +interface Message { + role: "user" | "assistant" | "tool"; + content: string; + toolCallId?: string; +} + +interface ToolResultDraft { + toolCallId: string; + functionName: string; + draft: string; +} + +/** + * BuildTab — tools / function calling UI + structured output (D9). + * + * Runs /v1/chat/completions with: + * - tools[] if any are defined + * - response_format: json_schema if structured output is enabled + * + * When tool_calls appear in the response, shows each tool call with an input + * for the tool_result + "Send result" button to continue the conversation. + */ +export default function BuildTab({ configState }: BuildTabProps) { + const toolsBuilder = useToolsBuilder(); + const structuredOutput = useStructuredOutput(); + + const [prompt, setPrompt] = useState(""); + const [messages, setMessages] = useState([]); + const [running, setRunning] = useState(false); + const [toolCalls, setToolCalls] = useState([]); + const [toolResultDrafts, setToolResultDrafts] = useState([]); + const [validationResult, setValidationResult] = useState<{ + valid: boolean; + error?: string; + } | null>(null); + const abortRef = useRef(null); + + function buildRequestBody(msgs: Message[]) { + const body: Record = { + model: configState.model, + stream: false, + messages: [ + ...(configState.systemPrompt + ? [{ role: "system", content: configState.systemPrompt }] + : []), + ...msgs.map((m) => { + if (m.role === "tool") { + return { + role: "tool", + content: m.content, + tool_call_id: m.toolCallId ?? "", + }; + } + return { role: m.role, content: m.content }; + }), + ], + }; + + // Attach tools if any defined + if (toolsBuilder.tools.length > 0) { + body["tools"] = toolsBuilder.tools; + } + + // Attach response_format if JSON mode enabled and schema set + if (structuredOutput.enabled && structuredOutput.schema != null) { + body["response_format"] = { + type: "json_schema", + json_schema: structuredOutput.schema, + }; + } + + const { params } = configState; + if (params.temperature != null) body["temperature"] = params.temperature; + if (params.max_tokens != null) body["max_tokens"] = params.max_tokens; + if (params.top_p != null) body["top_p"] = params.top_p; + if (params.presence_penalty != null) body["presence_penalty"] = params.presence_penalty; + if (params.frequency_penalty != null) body["frequency_penalty"] = params.frequency_penalty; + if (params.seed != null) body["seed"] = params.seed; + if (params.stop != null) body["stop"] = params.stop; + + return body; + } + + async function runRequest(msgs: Message[]) { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + setRunning(true); + setToolCalls([]); + setToolResultDrafts([]); + setValidationResult(null); + + try { + const res = await fetch(`${configState.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(buildRequestBody(msgs)), + signal: controller.signal, + }); + + if (!res.ok) { + const text = await res.text().catch(() => "Unknown error"); + const errMsg = text.slice(0, 300); + setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errMsg}` }]); + return; + } + + const data = (await res.json()) as { + choices?: Array<{ + message?: { + content?: string | null; + tool_calls?: ToolCall[]; + }; + }>; + }; + + const choice = data.choices?.[0]; + const assistantMsg = choice?.message; + + if (assistantMsg == null) { + setMessages((prev) => [ + ...prev, + { role: "assistant", content: "(empty response)" }, + ]); + return; + } + + if (assistantMsg.tool_calls && assistantMsg.tool_calls.length > 0) { + // Tool call response — show tool call UI + setToolCalls(assistantMsg.tool_calls); + setMessages((prev) => [ + ...prev, + { + role: "assistant", + content: assistantMsg.content ?? "(tool call)", + }, + ]); + // Initialize drafts + setToolResultDrafts( + assistantMsg.tool_calls.map((tc) => ({ + toolCallId: tc.id, + functionName: tc.function.name, + draft: "", + })), + ); + } else { + const content = assistantMsg.content ?? ""; + setMessages((prev) => [ + ...prev, + { role: "assistant", content }, + ]); + + // Validate structured output response if enabled + if (structuredOutput.enabled && structuredOutput.schema != null) { + const validation = structuredOutput.validateResponse(content); + setValidationResult(validation); + } + } + } catch (err) { + if (controller.signal.aborted) return; + const msg = err instanceof Error ? err.message : String(err); + setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${msg}` }]); + } finally { + setRunning(false); + } + } + + async function handleRun() { + if (!prompt.trim() && messages.length === 0) return; + + const newMessages: Message[] = [ + ...messages, + ...(prompt.trim() ? [{ role: "user" as const, content: prompt }] : []), + ]; + + if (prompt.trim()) { + setMessages(newMessages); + setPrompt(""); + } + + await runRequest(newMessages); + } + + async function sendToolResult(toolCallId: string) { + const draft = toolResultDrafts.find((d) => d.toolCallId === toolCallId); + if (draft == null) return; + + const toolResultMsg: Message = { + role: "tool", + content: draft.draft, + toolCallId, + }; + + const newMessages = [...messages, toolResultMsg]; + setMessages(newMessages); + setToolResultDrafts([]); + setToolCalls([]); + + await runRequest(newMessages); + } + + function clearConversation() { + setMessages([]); + setToolCalls([]); + setToolResultDrafts([]); + setValidationResult(null); + setPrompt(""); + } + + return ( +
+ {/* Left panel: conversation + run */} +
+ {/* Toolbar */} +
+ + + {messages.length > 0 && ( + + )} + +
+ {toolsBuilder.tools.length > 0 && ( + + {toolsBuilder.tools.length} tool{toolsBuilder.tools.length !== 1 ? "s" : ""} + + )} + {structuredOutput.enabled && ( + + JSON mode + + )} +
+
+ + {/* Conversation history */} +
+ {messages.map((msg, idx) => ( +
+
+ {msg.role === "user" ? ( + {msg.content} + ) : ( + + )} +
+
+ ))} + + {/* Tool call UI */} + {toolCalls.length > 0 && ( +
+ {toolCalls.map((tc) => { + const draft = toolResultDrafts.find((d) => d.toolCallId === tc.id); + return ( +
+
+ + function + + + {tc.function.name} + +
+ 
+                      {tc.function.arguments}
+
+
+ +