feat(cli): fase 4.3 — comando eval com suites, runs e scorecard ASCII

2026-07-31 04:12:10 +03:00 · 2026-05-15 01:52:34 -03:00
parent 76362c4efe
commit b8707dbbb4
2 changed files with 487 additions and 0 deletions
--- a/bin/cli/commands/eval.mjs
+++ b/bin/cli/commands/eval.mjs
@@ -0,0 +1,268 @@
+import { readFileSync } from "node:fs";
+import { setTimeout as sleep } from "node:timers/promises";
+import { apiFetch } from "../api.mjs";
+import { emit } from "../output.mjs";
+import { t } from "../i18n.mjs";
+
+function truncate(v, len = 30) {
+  if (v == null) return "-";
+  const s = String(v);
+  return s.length > len ? s.slice(0, len - 1) + "…" : s;
+}
+
+function fmtTs(v) {
+  if (!v) return "-";
+  try {
+    return new Date(v).toLocaleString();
+  } catch {
+    return String(v);
+  }
+}
+
+const suiteSchema = [
+  { key: "id", header: "Suite ID", width: 22 },
+  { key: "name", header: "Name", width: 30 },
+  { key: "samples", header: "Samples" },
+  { key: "rubric", header: "Rubric", width: 16 },
+  { key: "updatedAt", header: "Updated", formatter: fmtTs },
+];
+
+const runSchema = [
+  { key: "id", header: "Run ID", width: 22 },
+  { key: "suiteId", header: "Suite", width: 18 },
+  { key: "status", header: "Status", width: 12 },
+  { key: "model", header: "Model", width: 25 },
+  { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(3) : "-") },
+  {
+    key: "duration",
+    header: "Duration",
+    formatter: (v) => (v != null ? `${(v / 1000).toFixed(1)}s` : "-"),
+  },
+  { key: "startedAt", header: "Started", formatter: fmtTs },
+];
+
+const sampleSchema = [
+  { key: "id", header: "Sample", width: 14 },
+  { key: "score", header: "Score", formatter: (v) => (v != null ? v.toFixed(2) : "-") },
+  { key: "passed", header: "✓", formatter: (v) => (v ? "✓" : "✗") },
+  { key: "input", header: "Input", width: 30, formatter: truncate },
+  { key: "output", header: "Output", width: 30, formatter: truncate },
+];
+
+async function confirm(q) {
+  return new Promise((resolve) => {
+    process.stdout.write(`${q} (yes/no) `);
+    process.stdin.setEncoding("utf8");
+    process.stdin.once("data", (c) => resolve(c.toString().trim().toLowerCase().startsWith("y")));
+  });
+}
+
+async function watchRun(runId, globalOpts) {
+  let lastStatus = "";
+  while (true) {
+    await sleep(3000);
+    const res = await apiFetch(`/api/evals/${runId}`);
+    if (!res.ok) continue;
+    const r = await res.json();
+    if (r.status !== lastStatus) {
+      const done = r.progress?.completed ?? 0;
+      const total = r.progress?.total ?? "?";
+      process.stderr.write(`[${new Date().toISOString()}] ${r.status} — ${done}/${total}\n`);
+      lastStatus = r.status;
+    }
+    if (["completed", "failed", "cancelled"].includes(r.status)) {
+      emit(r, globalOpts, runSchema);
+      return;
+    }
+  }
+}
+
+function renderScorecard(data) {
+  const score = data.score ?? data.overallScore ?? null;
+  const passed = data.passed ?? data.summary?.passed ?? null;
+  const total = data.total ?? data.summary?.total ?? null;
+  process.stdout.write("\n=== Scorecard ===\n");
+  if (score != null) process.stdout.write(`Overall score: ${(score * 100).toFixed(1)}%\n`);
+  if (passed != null && total != null) {
+    process.stdout.write(`Passed: ${passed}/${total}\n`);
+    const bar = "█".repeat(Math.round((passed / total) * 20)).padEnd(20, "░");
+    process.stdout.write(`[${bar}] ${((passed / total) * 100).toFixed(0)}%\n`);
+  }
+  const metrics = data.metrics ?? data.breakdown ?? {};
+  for (const [k, v] of Object.entries(metrics)) {
+    process.stdout.write(`  ${k}: ${typeof v === "number" ? v.toFixed(3) : v}\n`);
+  }
+  process.stdout.write("\n");
+}
+
+export async function runEvalSuitesList(opts, cmd) {
+  const res = await apiFetch("/api/evals/suites");
+  if (!res.ok) {
+    process.stderr.write(`Error: ${res.status}\n`);
+    process.exit(1);
+  }
+  const data = await res.json();
+  emit(data.items ?? data, cmd.optsWithGlobals(), suiteSchema);
+}
+
+export async function runEvalSuitesGet(id, opts, cmd) {
+  const res = await apiFetch(`/api/evals/suites/${id}`);
+  if (!res.ok) {
+    process.stderr.write(`Not found: ${id}\n`);
+    process.exit(1);
+  }
+  emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalSuitesCreate(opts, cmd) {
+  if (!opts.file) {
+    process.stderr.write("--file required\n");
+    process.exit(2);
+  }
+  const body = JSON.parse(readFileSync(opts.file, "utf8"));
+  const res = await apiFetch("/api/evals/suites", { method: "POST", body });
+  if (!res.ok) {
+    process.stderr.write(`Error: ${res.status}\n`);
+    process.exit(1);
+  }
+  emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalRun(suiteId, opts, cmd) {
+  const globalOpts = cmd.optsWithGlobals();
+  const body = {
+    suiteId,
+    model: opts.model ?? "auto",
+    ...(opts.combo ? { combo: opts.combo } : {}),
+    concurrency: opts.concurrency ?? 4,
+    ...(opts.tag ? { tag: opts.tag } : {}),
+  };
+  const res = await apiFetch("/api/evals", { method: "POST", body });
+  if (!res.ok) {
+    process.stderr.write(`Error: ${res.status}\n`);
+    process.exit(1);
+  }
+  const run = await res.json();
+  emit(run, globalOpts, runSchema);
+  if (opts.watch) {
+    process.stderr.write("\nWatching run... (Ctrl+C to detach)\n");
+    await watchRun(run.id, globalOpts);
+  }
+}
+
+export async function runEvalList(opts, cmd) {
+  const params = new URLSearchParams({ limit: String(opts.limit ?? 50) });
+  if (opts.suite) params.set("suiteId", opts.suite);
+  if (opts.status) params.set("status", opts.status);
+  if (opts.since) params.set("since", opts.since);
+  const res = await apiFetch(`/api/evals?${params}`);
+  if (!res.ok) {
+    process.stderr.write(`Error: ${res.status}\n`);
+    process.exit(1);
+  }
+  const data = await res.json();
+  emit(data.items ?? data, cmd.optsWithGlobals(), runSchema);
+}
+
+export async function runEvalGet(id, opts, cmd) {
+  const res = await apiFetch(`/api/evals/${id}`);
+  if (!res.ok) {
+    process.stderr.write(`Not found: ${id}\n`);
+    process.exit(1);
+  }
+  emit(await res.json(), cmd.optsWithGlobals());
+}
+
+export async function runEvalResults(id, opts, cmd) {
+  const params = new URLSearchParams();
+  if (opts.failed) params.set("filter", "failed");
+  const res = await apiFetch(`/api/evals/${id}?${params}`);
+  if (!res.ok) {
+    process.stderr.write(`Not found: ${id}\n`);
+    process.exit(1);
+  }
+  const data = await res.json();
+  emit(data.samples ?? data.results ?? [], cmd.optsWithGlobals(), sampleSchema);
+}
+
+export async function runEvalCancel(id, opts, cmd) {
+  if (!opts.yes) {
+    const ok = await confirm(`Cancel run ${id}?`);
+    if (!ok) return;
+  }
+  const res = await apiFetch(`/api/evals/${id}`, { method: "POST", body: { op: "cancel" } });
+  if (!res.ok) {
+    process.stderr.write(`Error: ${res.status}\n`);
+    process.exit(1);
+  }
+  process.stdout.write("Cancelled\n");
+}
+
+export async function runEvalScorecard(id, opts, cmd) {
+  const res = await apiFetch(`/api/evals/${id}?scorecard=true`);
+  if (!res.ok) {
+    process.stderr.write(`Not found: ${id}\n`);
+    process.exit(1);
+  }
+  const data = await res.json();
+  const globalOpts = cmd.optsWithGlobals();
+  if (globalOpts.output === "json") {
+    emit(data, globalOpts);
+  } else {
+    renderScorecard(data);
+  }
+}
+
+export function registerEval(program) {
+  const evalCmd = program.command("eval").description(t("eval.description"));
+
+  const suites = evalCmd.command("suites").description(t("eval.suites.description"));
+  suites.command("list").description(t("eval.suites.list.description")).action(runEvalSuitesList);
+  suites
+    .command("get <suiteId>")
+    .description(t("eval.suites.get.description"))
+    .action(runEvalSuitesGet);
+  suites
+    .command("create")
+    .description(t("eval.suites.create.description"))
+    .option("--file <path>", t("eval.suites.create.file"))
+    .action(runEvalSuitesCreate);
+
+  evalCmd
+    .command("run <suiteId>")
+    .description(t("eval.run.description"))
+    .option("-m, --model <id>", t("eval.run.model"), "auto")
+    .option("--combo <name>", t("eval.run.combo"))
+    .option("--concurrency <n>", t("eval.run.concurrency"), parseInt, 4)
+    .option("--tag <tag>", t("eval.run.tag"))
+    .option("--watch", t("eval.run.watch"))
+    .action(runEvalRun);
+
+  evalCmd
+    .command("list")
+    .description(t("eval.list.description"))
+    .option("--suite <id>", t("eval.list.suite"))
+    .option("--status <s>", t("eval.list.status"))
+    .option("--since <ts>", t("eval.list.since"))
+    .option("--limit <n>", t("eval.list.limit"), parseInt, 50)
+    .action(runEvalList);
+
+  evalCmd.command("get <runId>").description(t("eval.get.description")).action(runEvalGet);
+
+  evalCmd
+    .command("results <runId>")
+    .description(t("eval.results.description"))
+    .option("--failed", t("eval.results.failed"))
+    .action(runEvalResults);
+
+  evalCmd
+    .command("cancel <runId>")
+    .description(t("eval.cancel.description"))
+    .option("--yes", t("eval.cancel.yes"))
+    .action(runEvalCancel);
+
+  evalCmd
+    .command("scorecard <runId>")
+    .description(t("eval.scorecard.description"))
+    .action(runEvalScorecard);
+}
--- a/tests/unit/cli-eval-commands.test.ts
+++ b/tests/unit/cli-eval-commands.test.ts
@@ -0,0 +1,219 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const SUITE = {
+  id: "suite-001",
+  name: "Chat quality",
+  samples: 50,
+  rubric: "accuracy",
+  updatedAt: "2026-05-14T10:00:00Z",
+};
+
+const RUN = {
+  id: "run-001",
+  suiteId: "suite-001",
+  status: "completed",
+  model: "gpt-4o",
+  score: 0.87,
+  duration: 42000,
+  startedAt: "2026-05-14T10:00:00Z",
+};
+
+const SAMPLES = [
+  { id: "s1", score: 0.9, passed: true, input: "Hello", output: "Hi there" },
+  { id: "s2", score: 0.4, passed: false, input: "2+2=?", output: "5" },
+];
+
+function makeResp(data: unknown, status = 200) {
+  const obj = {
+    ok: status < 400,
+    status,
+    exitCode: status < 400 ? 0 : 1,
+    json: () => Promise.resolve(data),
+    text: () => Promise.resolve(JSON.stringify(data)),
+    headers: new Headers(),
+  };
+  obj.json = obj.json.bind(obj);
+  obj.text = obj.text.bind(obj);
+  return obj;
+}
+
+async function captureStdout(fn: () => Promise<void>): Promise<string> {
+  const chunks: string[] = [];
+  const orig = process.stdout.write.bind(process.stdout);
+  process.stdout.write = (c: string | Uint8Array) => {
+    chunks.push(typeof c === "string" ? c : c.toString());
+    return true;
+  };
+  try {
+    await fn();
+  } finally {
+    process.stdout.write = orig;
+  }
+  return chunks.join("");
+}
+
+function makeCmd(output = "json") {
+  return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) };
+}
+
+test("runEvalSuitesList retorna lista de suites", async () => {
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string) => {
+    assert.ok(url.includes("/api/evals/suites"));
+    return Promise.resolve(makeResp({ items: [SUITE] }));
+  }) as any;
+
+  const { runEvalSuitesList } = await import("../../bin/cli/commands/eval.mjs");
+  const out = await captureStdout(() => runEvalSuitesList({}, makeCmd() as any));
+
+  globalThis.fetch = origFetch;
+  const parsed = JSON.parse(out);
+  assert.ok(Array.isArray(parsed));
+  assert.equal(parsed[0].id, "suite-001");
+});
+
+test("runEvalSuitesGet busca suite por id", async () => {
+  let capturedUrl = "";
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string) => {
+    capturedUrl = url;
+    return Promise.resolve(makeResp(SUITE));
+  }) as any;
+
+  const { runEvalSuitesGet } = await import("../../bin/cli/commands/eval.mjs");
+  const out = await captureStdout(() => runEvalSuitesGet("suite-001", {}, makeCmd() as any));
+
+  globalThis.fetch = origFetch;
+  assert.ok(capturedUrl.includes("/api/evals/suites/suite-001"));
+  const parsed = JSON.parse(out);
+  assert.equal(parsed.id, "suite-001");
+});
+
+test("runEvalRun envia suiteId e model no body", async () => {
+  let capturedBody: any = null;
+  let capturedUrl = "";
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string, opts: any) => {
+    capturedUrl = url;
+    if (opts?.body) capturedBody = JSON.parse(opts.body);
+    return Promise.resolve(makeResp(RUN));
+  }) as any;
+
+  const { runEvalRun } = await import("../../bin/cli/commands/eval.mjs");
+  await captureStdout(() =>
+    runEvalRun("suite-001", { model: "gpt-4o", concurrency: 4 }, makeCmd() as any)
+  );
+
+  globalThis.fetch = origFetch;
+  assert.ok(capturedUrl.includes("/api/evals"));
+  assert.equal(capturedBody.suiteId, "suite-001");
+  assert.equal(capturedBody.model, "gpt-4o");
+  assert.equal(capturedBody.concurrency, 4);
+});
+
+test("runEvalList envia filtros na query", async () => {
+  let capturedUrl = "";
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string) => {
+    capturedUrl = url;
+    return Promise.resolve(makeResp({ items: [RUN] }));
+  }) as any;
+
+  const { runEvalList } = await import("../../bin/cli/commands/eval.mjs");
+  await captureStdout(() =>
+    runEvalList({ suite: "suite-001", status: "completed", limit: 25 }, makeCmd() as any)
+  );
+
+  globalThis.fetch = origFetch;
+  assert.ok(capturedUrl.includes("suiteId=suite-001"));
+  assert.ok(capturedUrl.includes("status=completed"));
+  assert.ok(capturedUrl.includes("limit=25"));
+});
+
+test("runEvalGet busca run por id", async () => {
+  let capturedUrl = "";
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string) => {
+    capturedUrl = url;
+    return Promise.resolve(makeResp(RUN));
+  }) as any;
+
+  const { runEvalGet } = await import("../../bin/cli/commands/eval.mjs");
+  const out = await captureStdout(() => runEvalGet("run-001", {}, makeCmd() as any));
+
+  globalThis.fetch = origFetch;
+  assert.ok(capturedUrl.includes("/api/evals/run-001"));
+  const parsed = JSON.parse(out);
+  assert.equal(parsed.id, "run-001");
+});
+
+test("runEvalResults mostra amostras", async () => {
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((_url: string) => {
+    return Promise.resolve(makeResp({ samples: SAMPLES }));
+  }) as any;
+
+  const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");
+  const out = await captureStdout(() => runEvalResults("run-001", {}, makeCmd() as any));
+
+  globalThis.fetch = origFetch;
+  const parsed = JSON.parse(out);
+  assert.ok(Array.isArray(parsed));
+  assert.equal(parsed.length, 2);
+});
+
+test("runEvalResults com --failed envia filter=failed na query", async () => {
+  let capturedUrl = "";
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((url: string) => {
+    capturedUrl = url;
+    return Promise.resolve(makeResp({ samples: SAMPLES.filter((s) => !s.passed) }));
+  }) as any;
+
+  const { runEvalResults } = await import("../../bin/cli/commands/eval.mjs");
+  await captureStdout(() => runEvalResults("run-001", { failed: true }, makeCmd() as any));
+
+  globalThis.fetch = origFetch;
+  assert.ok(capturedUrl.includes("filter=failed"));
+});
+
+test("runEvalCancel com --yes envia op: cancel", async () => {
+  let capturedBody: any = null;
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((_url: string, opts: any) => {
+    if (opts?.body) capturedBody = JSON.parse(opts.body);
+    return Promise.resolve(makeResp({}));
+  }) as any;
+
+  const out = await captureStdout(async () => {
+    const { runEvalCancel } = await import("../../bin/cli/commands/eval.mjs");
+    await runEvalCancel("run-001", { yes: true }, makeCmd() as any);
+  });
+
+  globalThis.fetch = origFetch;
+  assert.equal(capturedBody.op, "cancel");
+  assert.ok(out.includes("Cancelled"));
+});
+
+test("runEvalScorecard renderiza scorecard em modo table", async () => {
+  const scoreData = {
+    score: 0.87,
+    passed: 43,
+    total: 50,
+    metrics: { accuracy: 0.87, fluency: 0.92 },
+  };
+  const origFetch = globalThis.fetch;
+  globalThis.fetch = ((_url: string) => {
+    return Promise.resolve(makeResp(scoreData));
+  }) as any;
+
+  const { runEvalScorecard } = await import("../../bin/cli/commands/eval.mjs");
+  const out = await captureStdout(() =>
+    runEvalScorecard("run-001", {}, { optsWithGlobals: () => ({ output: "table" }) } as any)
+  );
+
+  globalThis.fetch = origFetch;
+  assert.ok(out.includes("87.0%") || out.includes("Scorecard"));
+  assert.ok(out.includes("43/50") || out.includes("43"));
+});