diff --git a/bin/cli/commands/eval.mjs b/bin/cli/commands/eval.mjs new file mode 100644 index 0000000000..2027bed021 --- /dev/null +++ b/bin/cli/commands/eval.mjs @@ -0,0 +1,268 @@ +import { readFileSync } from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; +import { apiFetch } from "../api.mjs"; +import { emit } from "../output.mjs"; +import { t } from "../i18n.mjs"; + +function truncate(v, len = 30) { + if (v == null) return "-"; + const s = String(v); + return s.length > len ? s.slice(0, len - 1) + "…" : s; +} + +function fmtTs(v) { + if (!v) return "-"; + try { + return new Date(v).toLocaleString(); + } catch { + return String(v); + } +} + +const suiteSchema = [ + { key: "id", header: "Suite ID", width: 22 }, + { key: "name", header: "Name", width: 30 }, + { key: "samples", header: "Samples" }, + { key: "rubric", header: "Rubric", width: 16 }, + { key: "updatedAt", header: "Updated", formatter: fmtTs }, +]; + +const runSchema = [ + { key: "id", header: "Run ID", width: 22 }, + { key: "suiteId", header: "Suite", width: 18 }, + { key: "status", header: "Status", width: 12 }, + { key: "model", header: "Model", width: 25 }, + { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(3) : "-") }, + { + key: "duration", + header: "Duration", + formatter: (v) => (v != null ? `${(v / 1000).toFixed(1)}s` : "-"), + }, + { key: "startedAt", header: "Started", formatter: fmtTs }, +]; + +const sampleSchema = [ + { key: "id", header: "Sample", width: 14 }, + { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(2) : "-") }, + { key: "passed", header: "✓", formatter: (v) => (v ? "✓" : "✗") }, + { key: "input", header: "Input", width: 30, formatter: truncate }, + { key: "output", header: "Output", width: 30, formatter: truncate }, +]; + +async function confirm(q) { + return new Promise((resolve) => { + process.stdout.write(`${q} (yes/no) `); + process.stdin.setEncoding("utf8"); + process.stdin.once("data", (c) => resolve(c.toString().trim().toLowerCase().startsWith("y"))); + }); +} + +async function watchRun(runId, globalOpts) { + let lastStatus = ""; + while (true) { + await sleep(3000); + const res = await apiFetch(`/api/evals/${runId}`); + if (!res.ok) continue; + const r = await res.json(); + if (r.status !== lastStatus) { + const done = r.progress?.completed ?? 0; + const total = r.progress?.total ?? "?"; + process.stderr.write(`[${new Date().toISOString()}] ${r.status} — ${done}/${total}\n`); + lastStatus = r.status; + } + if (["completed", "failed", "cancelled"].includes(r.status)) { + emit(r, globalOpts, runSchema); + return; + } + } +} + +function renderScorecard(data) { + const score = data.score ?? data.overallScore ?? null; + const passed = data.passed ?? data.summary?.passed ?? null; + const total = data.total ?? data.summary?.total ?? null; + process.stdout.write("\n=== Scorecard ===\n"); + if (score != null) process.stdout.write(`Overall score: ${(score * 100).toFixed(1)}%\n`); + if (passed != null && total != null) { + process.stdout.write(`Passed: ${passed}/${total}\n`); + const bar = "█".repeat(Math.round((passed / total) * 20)).padEnd(20, "░"); + process.stdout.write(`[${bar}] ${((passed / total) * 100).toFixed(0)}%\n`); + } + const metrics = data.metrics ?? data.breakdown ?? {}; + for (const [k, v] of Object.entries(metrics)) { + process.stdout.write(` ${k}: ${typeof v === "number" ? v.toFixed(3) : v}\n`); + } + process.stdout.write("\n"); +} + +export async function runEvalSuitesList(opts, cmd) { + const res = await apiFetch("/api/evals/suites"); + if (!res.ok) { + process.stderr.write(`Error: ${res.status}\n`); + process.exit(1); + } + const data = await res.json(); + emit(data.items ?? data, cmd.optsWithGlobals(), suiteSchema); +} + +export async function runEvalSuitesGet(id, opts, cmd) { + const res = await apiFetch(`/api/evals/suites/${id}`); + if (!res.ok) { + process.stderr.write(`Not found: ${id}\n`); + process.exit(1); + } + emit(await res.json(), cmd.optsWithGlobals()); +} + +export async function runEvalSuitesCreate(opts, cmd) { + if (!opts.file) { + process.stderr.write("--file required\n"); + process.exit(2); + } + const body = JSON.parse(readFileSync(opts.file, "utf8")); + const res = await apiFetch("/api/evals/suites", { method: "POST", body }); + if (!res.ok) { + process.stderr.write(`Error: ${res.status}\n`); + process.exit(1); + } + emit(await res.json(), cmd.optsWithGlobals()); +} + +export async function runEvalRun(suiteId, opts, cmd) { + const globalOpts = cmd.optsWithGlobals(); + const body = { + suiteId, + model: opts.model ?? "auto", + ...(opts.combo ? { combo: opts.combo } : {}), + concurrency: opts.concurrency ?? 4, + ...(opts.tag ? { tag: opts.tag } : {}), + }; + const res = await apiFetch("/api/evals", { method: "POST", body }); + if (!res.ok) { + process.stderr.write(`Error: ${res.status}\n`); + process.exit(1); + } + const run = await res.json(); + emit(run, globalOpts, runSchema); + if (opts.watch) { + process.stderr.write("\nWatching run... (Ctrl+C to detach)\n"); + await watchRun(run.id, globalOpts); + } +} + +export async function runEvalList(opts, cmd) { + const params = new URLSearchParams({ limit: String(opts.limit ?? 50) }); + if (opts.suite) params.set("suiteId", opts.suite); + if (opts.status) params.set("status", opts.status); + if (opts.since) params.set("since", opts.since); + const res = await apiFetch(`/api/evals?${params}`); + if (!res.ok) { + process.stderr.write(`Error: ${res.status}\n`); + process.exit(1); + } + const data = await res.json(); + emit(data.items ?? data, cmd.optsWithGlobals(), runSchema); +} + +export async function runEvalGet(id, opts, cmd) { + const res = await apiFetch(`/api/evals/${id}`); + if (!res.ok) { + process.stderr.write(`Not found: ${id}\n`); + process.exit(1); + } + emit(await res.json(), cmd.optsWithGlobals()); +} + +export async function runEvalResults(id, opts, cmd) { + const params = new URLSearchParams(); + if (opts.failed) params.set("filter", "failed"); + const res = await apiFetch(`/api/evals/${id}?${params}`); + if (!res.ok) { + process.stderr.write(`Not found: ${id}\n`); + process.exit(1); + } + const data = await res.json(); + emit(data.samples ?? data.results ?? [], cmd.optsWithGlobals(), sampleSchema); +} + +export async function runEvalCancel(id, opts, cmd) { + if (!opts.yes) { + const ok = await confirm(`Cancel run ${id}?`); + if (!ok) return; + } + const res = await apiFetch(`/api/evals/${id}`, { method: "POST", body: { op: "cancel" } }); + if (!res.ok) { + process.stderr.write(`Error: ${res.status}\n`); + process.exit(1); + } + process.stdout.write("Cancelled\n"); +} + +export async function runEvalScorecard(id, opts, cmd) { + const res = await apiFetch(`/api/evals/${id}?scorecard=true`); + if (!res.ok) { + process.stderr.write(`Not found: ${id}\n`); + process.exit(1); + } + const data = await res.json(); + const globalOpts = cmd.optsWithGlobals(); + if (globalOpts.output === "json") { + emit(data, globalOpts); + } else { + renderScorecard(data); + } +} + +export function registerEval(program) { + const evalCmd = program.command("eval").description(t("eval.description")); + + const suites = evalCmd.command("suites").description(t("eval.suites.description")); + suites.command("list").description(t("eval.suites.list.description")).action(runEvalSuitesList); + suites + .command("get ") + .description(t("eval.suites.get.description")) + .action(runEvalSuitesGet); + suites + .command("create") + .description(t("eval.suites.create.description")) + .option("--file ", t("eval.suites.create.file")) + .action(runEvalSuitesCreate); + + evalCmd + .command("run ") + .description(t("eval.run.description")) + .option("-m, --model ", t("eval.run.model"), "auto") + .option("--combo ", t("eval.run.combo")) + .option("--concurrency ", t("eval.run.concurrency"), parseInt, 4) + .option("--tag ", t("eval.run.tag")) + .option("--watch", t("eval.run.watch")) + .action(runEvalRun); + + evalCmd + .command("list") + .description(t("eval.list.description")) + .option("--suite ", t("eval.list.suite")) + .option("--status ", t("eval.list.status")) + .option("--since ", t("eval.list.since")) + .option("--limit ", t("eval.list.limit"), parseInt, 50) + .action(runEvalList); + + evalCmd.command("get ").description(t("eval.get.description")).action(runEvalGet); + + evalCmd + .command("results ") + .description(t("eval.results.description")) + .option("--failed", t("eval.results.failed")) + .action(runEvalResults); + + evalCmd + .command("cancel ") + .description(t("eval.cancel.description")) + .option("--yes", t("eval.cancel.yes")) + .action(runEvalCancel); + + evalCmd + .command("scorecard ") + .description(t("eval.scorecard.description")) + .action(runEvalScorecard); +} diff --git a/tests/unit/cli-eval-commands.test.ts b/tests/unit/cli-eval-commands.test.ts new file mode 100644 index 0000000000..f623bda3c1 --- /dev/null +++ b/tests/unit/cli-eval-commands.test.ts @@ -0,0 +1,219 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const SUITE = { + id: "suite-001", + name: "Chat quality", + samples: 50, + rubric: "accuracy", + updatedAt: "2026-05-14T10:00:00Z", +}; + +const RUN = { + id: "run-001", + suiteId: "suite-001", + status: "completed", + model: "gpt-4o", + score: 0.87, + duration: 42000, + startedAt: "2026-05-14T10:00:00Z", +}; + +const SAMPLES = [ + { id: "s1", score: 0.9, passed: true, input: "Hello", output: "Hi there" }, + { id: "s2", score: 0.4, passed: false, input: "2+2=?", output: "5" }, +]; + +function makeResp(data: unknown, status = 200) { + const obj = { + ok: status < 400, + status, + exitCode: status < 400 ? 0 : 1, + json: () => Promise.resolve(data), + text: () => Promise.resolve(JSON.stringify(data)), + headers: new Headers(), + }; + obj.json = obj.json.bind(obj); + obj.text = obj.text.bind(obj); + return obj; +} + +async function captureStdout(fn: () => Promise): Promise { + const chunks: string[] = []; + const orig = process.stdout.write.bind(process.stdout); + process.stdout.write = (c: string | Uint8Array) => { + chunks.push(typeof c === "string" ? c : c.toString()); + return true; + }; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return chunks.join(""); +} + +function makeCmd(output = "json") { + return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) }; +} + +test("runEvalSuitesList retorna lista de suites", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + assert.ok(url.includes("/api/evals/suites")); + return Promise.resolve(makeResp({ items: [SUITE] })); + }) as any; + + const { runEvalSuitesList } = await import("../../bin/cli/commands/eval.mjs"); + const out = await captureStdout(() => runEvalSuitesList({}, makeCmd() as any)); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed[0].id, "suite-001"); +}); + +test("runEvalSuitesGet busca suite por id", async () => { + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + capturedUrl = url; + return Promise.resolve(makeResp(SUITE)); + }) as any; + + const { runEvalSuitesGet } = await import("../../bin/cli/commands/eval.mjs"); + const out = await captureStdout(() => runEvalSuitesGet("suite-001", {}, makeCmd() as any)); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("/api/evals/suites/suite-001")); + const parsed = JSON.parse(out); + assert.equal(parsed.id, "suite-001"); +}); + +test("runEvalRun envia suiteId e model no body", async () => { + let capturedBody: any = null; + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string, opts: any) => { + capturedUrl = url; + if (opts?.body) capturedBody = JSON.parse(opts.body); + return Promise.resolve(makeResp(RUN)); + }) as any; + + const { runEvalRun } = await import("../../bin/cli/commands/eval.mjs"); + await captureStdout(() => + runEvalRun("suite-001", { model: "gpt-4o", concurrency: 4 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("/api/evals")); + assert.equal(capturedBody.suiteId, "suite-001"); + assert.equal(capturedBody.model, "gpt-4o"); + assert.equal(capturedBody.concurrency, 4); +}); + +test("runEvalList envia filtros na query", async () => { + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + capturedUrl = url; + return Promise.resolve(makeResp({ items: [RUN] })); + }) as any; + + const { runEvalList } = await import("../../bin/cli/commands/eval.mjs"); + await captureStdout(() => + runEvalList({ suite: "suite-001", status: "completed", limit: 25 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("suiteId=suite-001")); + assert.ok(capturedUrl.includes("status=completed")); + assert.ok(capturedUrl.includes("limit=25")); +}); + +test("runEvalGet busca run por id", async () => { + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + capturedUrl = url; + return Promise.resolve(makeResp(RUN)); + }) as any; + + const { runEvalGet } = await import("../../bin/cli/commands/eval.mjs"); + const out = await captureStdout(() => runEvalGet("run-001", {}, makeCmd() as any)); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("/api/evals/run-001")); + const parsed = JSON.parse(out); + assert.equal(parsed.id, "run-001"); +}); + +test("runEvalResults mostra amostras", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = ((_url: string) => { + return Promise.resolve(makeResp({ samples: SAMPLES })); + }) as any; + + const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs"); + const out = await captureStdout(() => runEvalResults("run-001", {}, makeCmd() as any)); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed.length, 2); +}); + +test("runEvalResults com --failed envia filter=failed na query", async () => { + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + capturedUrl = url; + return Promise.resolve(makeResp({ samples: SAMPLES.filter((s) => !s.passed) })); + }) as any; + + const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs"); + await captureStdout(() => runEvalResults("run-001", { failed: true }, makeCmd() as any)); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("filter=failed")); +}); + +test("runEvalCancel com --yes envia op: cancel", async () => { + let capturedBody: any = null; + const origFetch = globalThis.fetch; + globalThis.fetch = ((_url: string, opts: any) => { + if (opts?.body) capturedBody = JSON.parse(opts.body); + return Promise.resolve(makeResp({})); + }) as any; + + const out = await captureStdout(async () => { + const { runEvalCancel } = await import("../../bin/cli/commands/eval.mjs"); + await runEvalCancel("run-001", { yes: true }, makeCmd() as any); + }); + + globalThis.fetch = origFetch; + assert.equal(capturedBody.op, "cancel"); + assert.ok(out.includes("Cancelled")); +}); + +test("runEvalScorecard renderiza scorecard em modo table", async () => { + const scoreData = { + score: 0.87, + passed: 43, + total: 50, + metrics: { accuracy: 0.87, fluency: 0.92 }, + }; + const origFetch = globalThis.fetch; + globalThis.fetch = ((_url: string) => { + return Promise.resolve(makeResp(scoreData)); + }) as any; + + const { runEvalScorecard } = await import("../../bin/cli/commands/eval.mjs"); + const out = await captureStdout(() => + runEvalScorecard("run-001", {}, { optsWithGlobals: () => ({ output: "table" }) } as any) + ); + + globalThis.fetch = origFetch; + assert.ok(out.includes("87.0%") || out.includes("Scorecard")); + assert.ok(out.includes("43/50") || out.includes("43")); +});