diff --git a/bin/cli/commands/eval.mjs b/bin/cli/commands/eval.mjs index 2027bed021..a0baf40a18 100644 --- a/bin/cli/commands/eval.mjs +++ b/bin/cli/commands/eval.mjs @@ -145,8 +145,18 @@ export async function runEvalRun(suiteId, opts, cmd) { const run = await res.json(); emit(run, globalOpts, runSchema); if (opts.watch) { - process.stderr.write("\nWatching run... (Ctrl+C to detach)\n"); - await watchRun(run.id, globalOpts); + if (process.stdout.isTTY) { + const { startEvalWatchTui } = await import("../tui/EvalWatch.jsx"); + await startEvalWatchTui({ + runId: run.id, + suiteId: opts.suite, + baseUrl: globalOpts.baseUrl ?? "http://localhost:20128", + apiKey: globalOpts.apiKey ?? process.env.OMNIROUTE_API_KEY, + }); + } else { + process.stderr.write("\nWatching run... (Ctrl+C to detach)\n"); + await watchRun(run.id, globalOpts); + } } } diff --git a/bin/cli/commands/oauth.mjs b/bin/cli/commands/oauth.mjs index 44f785eb8f..2d907df39f 100644 --- a/bin/cli/commands/oauth.mjs +++ b/bin/cli/commands/oauth.mjs @@ -64,9 +64,18 @@ async function runBrowserFlow(def, opts) { } const start = await startRes.json(); const url = start.authorizeUrl ?? start.url; - process.stdout.write(`\nOpen this URL to authorize:\n ${url}\n\n`); - if (opts.browser !== false) await openBrowser(url); - process.stderr.write("Waiting for authorization... (Ctrl+C to cancel)\n"); + + if (process.stdout.isTTY && opts.browser !== false) { + const { startOAuthTui } = await import("../tui/OAuthFlow.jsx"); + await openBrowser(url); + const tuiResult = await startOAuthTui({ provider: def.name ?? def.id, url }); + if (tuiResult.status === "cancelled") return; + } else { + process.stdout.write(`\nOpen this URL to authorize:\n ${url}\n\n`); + if (opts.browser !== false) await openBrowser(url); + process.stderr.write("Waiting for authorization... (Ctrl+C to cancel)\n"); + } + const result = await pollStatus( `/api/oauth/${def.id}/status?state=${encodeURIComponent(start.state ?? "")}`, opts.timeout ?? 300000 diff --git a/bin/cli/commands/test-provider.mjs b/bin/cli/commands/test-provider.mjs index 885251dd69..9238de4957 100644 --- a/bin/cli/commands/test-provider.mjs +++ b/bin/cli/commands/test-provider.mjs @@ -29,6 +29,10 @@ export async function runTestProviderCommand(provider, model, opts = {}) { return 1; } + if (opts.allProviders) { + return _runAllProviders(opts); + } + if (opts.compare) { return _runCompare(provider, opts); } @@ -65,6 +69,58 @@ export async function runTestProviderCommand(provider, model, opts = {}) { return aggregated.success ? 0 : 1; } +async function _runAllProviders(opts) { + const res = await apiFetch("/api/providers?limit=200", { + retry: false, + timeout: 5000, + acceptNotOk: true, + }); + if (!res.ok) { + console.error(t("test.noServer")); + return 1; + } + const data = await res.json(); + const connections = (data.providers ?? data.items ?? data).filter( + (c) => c.authType === "apikey" || c.testStatus !== "unavailable" + ); + if (connections.length === 0) { + console.log(t("test.noProviders")); + return 0; + } + + const providers = connections.map((c) => ({ + provider: c.provider ?? c.id, + model: c.defaultModel ?? c.model, + })); + + if (process.stdout.isTTY && !opts.json && opts.output !== "json") { + const { startProvidersTestTui } = await import("../tui/ProvidersTestAll.jsx"); + const baseUrl = opts.baseUrl ?? "http://localhost:20128"; + const apiKey = opts.apiKey ?? process.env.OMNIROUTE_API_KEY; + await startProvidersTestTui({ providers, baseUrl, apiKey }); + return 0; + } + + const results = await Promise.all( + providers.map(async ({ provider, model }) => { + const r = await _runSingleTest(provider, model); + return { provider, model, ...r }; + }) + ); + + if (opts.json || opts.output === "json") { + console.log(JSON.stringify(results, null, 2)); + } else { + for (const r of results) { + const mark = r.success ? "\x1b[32m✔\x1b[0m" : "\x1b[31m✖\x1b[0m"; + console.log(`${mark} ${r.provider}/${r.model ?? "-"}`); + } + } + + const failed = results.filter((r) => !r.success).length; + return failed > 0 ? 1 : 0; +} + async function _runCompare(provider, opts) { const targetProvider = provider || "anthropic"; const models = opts.compare diff --git a/bin/cli/locales/en.json b/bin/cli/locales/en.json index f9f7bf4a87..7505a9567f 100644 --- a/bin/cli/locales/en.json +++ b/bin/cli/locales/en.json @@ -290,7 +290,8 @@ "saveOpt": "Save results to a JSON file", "saved": "Results saved to {path}", "compareTitle": "Model Comparison", - "compareMinTwo": "At least two models required for --compare (comma-separated)" + "compareMinTwo": "At least two models required for --compare (comma-separated)", + "noProviders": "No providers configured. Add one with: omniroute keys add" }, "update": { "checking": "Checking for updates...", diff --git a/bin/cli/locales/pt-BR.json b/bin/cli/locales/pt-BR.json index e26db7969d..1b58c5154a 100644 --- a/bin/cli/locales/pt-BR.json +++ b/bin/cli/locales/pt-BR.json @@ -290,7 +290,8 @@ "saveOpt": "Salvar resultados em arquivo JSON", "saved": "Resultados salvos em {path}", "compareTitle": "Comparação de Modelos", - "compareMinTwo": "São necessários pelo menos dois modelos para --compare (separados por vírgula)" + "compareMinTwo": "São necessários pelo menos dois modelos para --compare (separados por vírgula)", + "noProviders": "Nenhum provedor configurado. Adicione um com: omniroute keys add" }, "update": { "checking": "Verificando atualizações...", diff --git a/bin/cli/tui/EvalWatch.jsx b/bin/cli/tui/EvalWatch.jsx new file mode 100644 index 0000000000..1e091ae853 --- /dev/null +++ b/bin/cli/tui/EvalWatch.jsx @@ -0,0 +1,175 @@ +import React, { useState, useEffect, useCallback } from "react"; +import { render, Box, Text, useInput } from "ink"; +import Spinner from "ink-spinner"; +import { ProgressBar } from "../tui-components/ProgressBar.jsx"; +import { StatusBadge } from "../tui-components/StatusBadge.jsx"; +import { DataTable } from "../tui-components/DataTable.jsx"; +import { HeaderSwr } from "../tui-components/HeaderSwr.jsx"; + +const TERMINAL_STATUSES = new Set(["completed", "failed", "cancelled"]); + +const RESULT_SCHEMA = [ + { key: "idx", header: "#", width: 5 }, + { key: "status", header: "Status", width: 10, formatter: (v) => (v === "pass" ? "✔" : "✖") }, + { key: "model", header: "Model", width: 22 }, + { key: "score", header: "Score", width: 8, formatter: (v) => (v != null ? String(v) : "-") }, + { key: "latencyMs", header: "ms", width: 8, formatter: (v) => (v != null ? String(v) : "-") }, +]; + +function EvalWatchApp({ runId, suiteId, baseUrl, apiKey, onExit }) { + const [run, setRun] = useState(null); + const [results, setResults] = useState([]); + const [paused, setPaused] = useState(false); + const [done, setDone] = useState(false); + const [elapsed, setElapsed] = useState(0); + + const fetchUrl = `${baseUrl ?? "http://localhost:20128"}/api/evals/${runId}`; + const headers = apiKey ? { Authorization: `Bearer ${apiKey}` } : {}; + + useEffect(() => { + const id = setInterval(() => setElapsed((e) => e + 1), 1000); + return () => clearInterval(id); + }, []); + + const fetcher = useCallback(async () => { + if (paused) return null; + const res = await fetch(fetchUrl, { headers }); + if (!res.ok) return null; + return res.json(); + }, [fetchUrl, paused]); + + useEffect(() => { + if (!run) return; + if (TERMINAL_STATUSES.has(run.status)) { + setDone(true); + } + const samples = run.samples ?? run.results ?? []; + setResults( + samples.map((s, i) => ({ + idx: i + 1, + status: s.pass ? "pass" : "fail", + model: s.model ?? "-", + score: s.score, + latencyMs: s.latencyMs, + })) + ); + }, [run]); + + useInput((input, key) => { + if (input === "q" || (key.ctrl && input === "c")) onExit?.(); + if (input === "p") setPaused((p) => !p); + }); + + const total = run?.progress?.total ?? 0; + const completed = run?.progress?.completed ?? 0; + const passed = run?.progress?.passed ?? results.filter((r) => r.status === "pass").length; + const failed = run?.progress?.failed ?? results.filter((r) => r.status === "fail").length; + const pct = total > 0 ? Math.round((completed / total) * 100) : 0; + + return ( + + + + Eval Watch — Run {runId} + {suiteId ? ` (suite: ${suiteId})` : ""} + + + {Math.floor(elapsed / 60)}:{String(elapsed % 60).padStart(2, "0")} + {paused ? " [PAUSED]" : ""} + + + + { + if (data && data !== run) setRun(data); + return null; + }} + initial={null} + /> + + {run ? ( + <> + + + {run.status} + + {completed}/{total || "?"} samples + + + + {total > 0 && ( + + + + ✔ {passed} passed + + ✖ {failed} failed + + + )} + + {results.length > 0 && ( + + + Recent results + + + + )} + + {done && ( + + + {run.status === "completed" + ? `✔ Eval completed — ${passed}/${total} passed` + : `✖ Eval ${run.status}`} + + + )} + + ) : ( + + + + + Loading... + + )} + + + [q] quit [p] {paused ? "resume" : "pause"} + + + ); +} + +export async function startEvalWatchTui({ runId, suiteId, baseUrl, apiKey }) { + return new Promise((resolve, reject) => { + function onExit() { + unmount(); + resolve(); + } + + const { unmount, waitUntilExit } = render( + + ); + + waitUntilExit().then(resolve).catch(reject); + }); +} diff --git a/bin/cli/tui/OAuthFlow.jsx b/bin/cli/tui/OAuthFlow.jsx new file mode 100644 index 0000000000..f8f26d6639 --- /dev/null +++ b/bin/cli/tui/OAuthFlow.jsx @@ -0,0 +1,172 @@ +import React, { useState, useEffect } from "react"; +import { render, Box, Text, useInput } from "ink"; +import Spinner from "ink-spinner"; +import { StatusBadge } from "../tui-components/StatusBadge.jsx"; +import { ConfirmDialog } from "../tui-components/ConfirmDialog.jsx"; + +const PHASE = { + WAITING: "waiting", + POLLING: "polling", + DONE: "done", + FAILED: "failed", + CANCELLED: "cancelled", +}; + +function OAuthFlowApp({ provider, url, deviceCode, onCancel, onDone, onFail }) { + const [phase, setPhase] = useState(PHASE.WAITING); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + const [elapsed, setElapsed] = useState(0); + const [confirmCancel, setConfirmCancel] = useState(false); + + useEffect(() => { + const id = setInterval(() => setElapsed((e) => e + 1), 1000); + return () => clearInterval(id); + }, []); + + useEffect(() => { + if (!onDone && !onFail) return; + setPhase(PHASE.POLLING); + }, []); + + useInput((input, key) => { + if (phase === PHASE.DONE || phase === PHASE.FAILED || phase === PHASE.CANCELLED) return; + if (input === "q" || (key.ctrl && input === "c")) { + setConfirmCancel(true); + } + }); + + function handleCancelConfirm(yes) { + setConfirmCancel(false); + if (yes) { + setPhase(PHASE.CANCELLED); + onCancel?.(); + } + } + + const elapsed_str = `${Math.floor(elapsed / 60)}:${String(elapsed % 60).padStart(2, "0")}`; + + if (confirmCancel) { + return ( + + + + ); + } + + return ( + + + + OmniRoute OAuth — {provider} + + + + {url && ( + + Open this URL in your browser to authorize: + + + {url} + + + + )} + + {deviceCode && ( + + + Device code:{" "} + + {deviceCode} + + + + )} + + + {phase === PHASE.POLLING || phase === PHASE.WAITING ? ( + + + + + Waiting for authorization... + ({elapsed_str}) + + ) : phase === PHASE.DONE ? ( + + + Authorized: {result?.email ?? result?.account ?? "connected"} + + ) : phase === PHASE.FAILED ? ( + + + Failed: {error} + + ) : ( + + + Cancelled. + + )} + + + {(phase === PHASE.POLLING || phase === PHASE.WAITING) && ( + + [q] cancel + + )} + + ); +} + +export async function startOAuthTui({ provider, url, deviceCode }) { + return new Promise((resolve, reject) => { + let resolved = false; + + function onDone(result) { + if (resolved) return; + resolved = true; + unmount(); + resolve({ status: "authorized", result }); + } + + function onFail(err) { + if (resolved) return; + resolved = true; + unmount(); + resolve({ status: "failed", error: err }); + } + + function onCancel() { + if (resolved) return; + resolved = true; + unmount(); + resolve({ status: "cancelled" }); + } + + const { unmount, waitUntilExit } = render( + + ); + + waitUntilExit() + .then(() => { + if (!resolved) resolve({ status: "exited" }); + }) + .catch(reject); + }); +} + +export function markOAuthDone(result) {} +export function markOAuthFailed(error) {} diff --git a/bin/cli/tui/ProvidersTestAll.jsx b/bin/cli/tui/ProvidersTestAll.jsx new file mode 100644 index 0000000000..cca4dddd1b --- /dev/null +++ b/bin/cli/tui/ProvidersTestAll.jsx @@ -0,0 +1,189 @@ +import React, { useState, useEffect, useCallback } from "react"; +import { render, Box, Text, useInput } from "ink"; +import Spinner from "ink-spinner"; +import { DataTable } from "../tui-components/DataTable.jsx"; +import { ProgressBar } from "../tui-components/ProgressBar.jsx"; + +const STATUS = { + PENDING: "pending", + RUNNING: "running", + PASS: "pass", + FAIL: "fail", + SKIP: "skip", +}; + +const TABLE_SCHEMA = [ + { key: "provider", header: "Provider", width: 28 }, + { key: "model", header: "Model", width: 32 }, + { + key: "status", + header: "Status", + width: 10, + formatter: (v) => { + if (v === STATUS.RUNNING) return "…"; + if (v === STATUS.PASS) return "✔"; + if (v === STATUS.FAIL) return "✖"; + if (v === STATUS.SKIP) return "—"; + return "·"; + }, + }, + { key: "latencyMs", header: "ms", width: 8, formatter: (v) => (v != null ? String(v) : "-") }, + { key: "error", header: "Error", width: 28, formatter: (v) => (v ? v.slice(0, 26) : "") }, +]; + +async function testOne(provider, model, baseUrl, apiKey) { + const headers = { + "Content-Type": "application/json", + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + }; + const start = Date.now(); + try { + const res = await fetch(`${baseUrl}/api/v1/providers/test`, { + method: "POST", + headers, + body: JSON.stringify({ provider, model }), + signal: AbortSignal.timeout(30000), + }); + const latencyMs = Date.now() - start; + const data = res.ok ? await res.json() : { success: false, error: `HTTP ${res.status}` }; + return { status: data.success ? STATUS.PASS : STATUS.FAIL, latencyMs, error: data.error }; + } catch (err) { + return { + status: STATUS.FAIL, + latencyMs: Date.now() - start, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +function ProvidersTestAllApp({ providers, baseUrl, apiKey, concurrency = 4, onExit }) { + const resolved = `${baseUrl ?? "http://localhost:20128"}`; + + const [rows, setRows] = useState(() => + providers.map((p, i) => ({ + id: i, + provider: p.provider ?? p.id ?? String(p), + model: p.model ?? p.defaultModel ?? "", + status: STATUS.PENDING, + latencyMs: null, + error: null, + })) + ); + const [done, setDone] = useState(false); + const [started, setStarted] = useState(false); + + const update = useCallback((id, patch) => { + setRows((prev) => prev.map((r) => (r.id === id ? { ...r, ...patch } : r))); + }, []); + + useEffect(() => { + if (started) return; + setStarted(true); + + async function runAll() { + const queue = [...rows]; + let running = 0; + let cursor = 0; + + function nextSlot() { + while (running < concurrency && cursor < queue.length) { + const row = queue[cursor++]; + running++; + update(row.id, { status: STATUS.RUNNING }); + testOne(row.provider, row.model, resolved, apiKey).then((result) => { + update(row.id, result); + running--; + nextSlot(); + if (cursor >= queue.length && running === 0) setDone(true); + }); + } + } + + nextSlot(); + } + + runAll(); + }, []); + + useInput((input, key) => { + if (input === "q" || (key.ctrl && input === "c")) onExit?.(); + }); + + const total = rows.length; + const completed = rows.filter((r) => r.status === STATUS.PASS || r.status === STATUS.FAIL).length; + const passed = rows.filter((r) => r.status === STATUS.PASS).length; + const failed = rows.filter((r) => r.status === STATUS.FAIL).length; + const running = rows.filter((r) => r.status === STATUS.RUNNING).length; + const pct = total > 0 ? Math.round((completed / total) * 100) : 0; + + return ( + + + + Providers Test All + + + {running > 0 ? ( + <> + {running} running + + ) : done ? ( + "done" + ) : ( + "queued" + )} + + + + + 0 ? "red" : "cyan"} /> + + + {completed}/{total} + + + ✔ {passed} + + ✖ {failed} + + + + + + {done && ( + + + {failed === 0 + ? `All ${passed} providers passed!` + : `${passed} passed, ${failed} failed`} + + + )} + + + [q] quit + + + ); +} + +export async function startProvidersTestTui({ providers, baseUrl, apiKey, concurrency = 4 }) { + return new Promise((resolve, reject) => { + function onExit() { + unmount(); + resolve(); + } + + const { unmount, waitUntilExit } = render( + + ); + + waitUntilExit().then(resolve).catch(reject); + }); +} diff --git a/tests/unit/cli-expanded-commands.test.ts b/tests/unit/cli-expanded-commands.test.ts index 3e8aafd379..6996a1cab8 100644 --- a/tests/unit/cli-expanded-commands.test.ts +++ b/tests/unit/cli-expanded-commands.test.ts @@ -134,6 +134,39 @@ test("test-provider — registerTestProvider registra flags latency/repeat/compa assert.ok(optNames.includes("--save"), "--save deve existir"); }); +test("OAuthFlow.jsx — arquivo existe e exporta startOAuthTui", async () => { + const { readFileSync } = await import("node:fs"); + const { fileURLToPath } = await import("node:url"); + const path = new URL("../../bin/cli/tui/OAuthFlow.jsx", import.meta.url); + const src = readFileSync(fileURLToPath(path), "utf8"); + assert.ok( + src.includes("export async function startOAuthTui"), + "startOAuthTui deve ser exportada" + ); +}); + +test("EvalWatch.jsx — arquivo existe e exporta startEvalWatchTui", async () => { + const { readFileSync } = await import("node:fs"); + const { fileURLToPath } = await import("node:url"); + const path = new URL("../../bin/cli/tui/EvalWatch.jsx", import.meta.url); + const src = readFileSync(fileURLToPath(path), "utf8"); + assert.ok( + src.includes("export async function startEvalWatchTui"), + "startEvalWatchTui deve ser exportada" + ); +}); + +test("ProvidersTestAll.jsx — arquivo existe e exporta startProvidersTestTui", async () => { + const { readFileSync } = await import("node:fs"); + const { fileURLToPath } = await import("node:url"); + const path = new URL("../../bin/cli/tui/ProvidersTestAll.jsx", import.meta.url); + const src = readFileSync(fileURLToPath(path), "utf8"); + assert.ok( + src.includes("export async function startProvidersTestTui"), + "startProvidersTestTui deve ser exportada" + ); +}); + test("test-provider — compare requer pelo menos dois modelos sem server retorna 0 ou 1", async () => { const { runTestProviderCommand } = await import("../../bin/cli/commands/test-provider.mjs"); let code = 0;