feat(cli): R7 — OAuthFlow/EvalWatch/ProvidersTestAll TUI + integração (spec 8.10)

Adiciona 3 componentes TUI Ink faltantes da spec 8.10:
- OAuthFlow.jsx: spinner + URL interativo para fluxo browser OAuth
- EvalWatch.jsx: monitor live de eval run com ProgressBar e DataTable
- ProvidersTestAll.jsx: teste paralelo com concorrência configurável

Integração nos comandos:
- oauth start browser flow → OAuthFlow quando TTY
- eval run --watch → EvalWatch quando TTY (fallback texto sem TTY)
- test --all-providers → ProvidersTestAll quando TTY (fallback tabela sem TTY)
This commit is contained in:
diegosouzapw
2026-05-15 09:13:56 -03:00
parent 2faf3608b2
commit 09db1129a1
9 changed files with 653 additions and 7 deletions

View File

@@ -145,8 +145,18 @@ export async function runEvalRun(suiteId, opts, cmd) {
const run = await res.json();
emit(run, globalOpts, runSchema);
if (opts.watch) {
process.stderr.write("\nWatching run... (Ctrl+C to detach)\n");
await watchRun(run.id, globalOpts);
if (process.stdout.isTTY) {
const { startEvalWatchTui } = await import("../tui/EvalWatch.jsx");
await startEvalWatchTui({
runId: run.id,
suiteId: opts.suite,
baseUrl: globalOpts.baseUrl ?? "http://localhost:20128",
apiKey: globalOpts.apiKey ?? process.env.OMNIROUTE_API_KEY,
});
} else {
process.stderr.write("\nWatching run... (Ctrl+C to detach)\n");
await watchRun(run.id, globalOpts);
}
}
}

View File

@@ -64,9 +64,18 @@ async function runBrowserFlow(def, opts) {
}
const start = await startRes.json();
const url = start.authorizeUrl ?? start.url;
process.stdout.write(`\nOpen this URL to authorize:\n ${url}\n\n`);
if (opts.browser !== false) await openBrowser(url);
process.stderr.write("Waiting for authorization... (Ctrl+C to cancel)\n");
if (process.stdout.isTTY && opts.browser !== false) {
const { startOAuthTui } = await import("../tui/OAuthFlow.jsx");
await openBrowser(url);
const tuiResult = await startOAuthTui({ provider: def.name ?? def.id, url });
if (tuiResult.status === "cancelled") return;
} else {
process.stdout.write(`\nOpen this URL to authorize:\n ${url}\n\n`);
if (opts.browser !== false) await openBrowser(url);
process.stderr.write("Waiting for authorization... (Ctrl+C to cancel)\n");
}
const result = await pollStatus(
`/api/oauth/${def.id}/status?state=${encodeURIComponent(start.state ?? "")}`,
opts.timeout ?? 300000

View File

@@ -29,6 +29,10 @@ export async function runTestProviderCommand(provider, model, opts = {}) {
return 1;
}
if (opts.allProviders) {
return _runAllProviders(opts);
}
if (opts.compare) {
return _runCompare(provider, opts);
}
@@ -65,6 +69,58 @@ export async function runTestProviderCommand(provider, model, opts = {}) {
return aggregated.success ? 0 : 1;
}
async function _runAllProviders(opts) {
const res = await apiFetch("/api/providers?limit=200", {
retry: false,
timeout: 5000,
acceptNotOk: true,
});
if (!res.ok) {
console.error(t("test.noServer"));
return 1;
}
const data = await res.json();
const connections = (data.providers ?? data.items ?? data).filter(
(c) => c.authType === "apikey" || c.testStatus !== "unavailable"
);
if (connections.length === 0) {
console.log(t("test.noProviders"));
return 0;
}
const providers = connections.map((c) => ({
provider: c.provider ?? c.id,
model: c.defaultModel ?? c.model,
}));
if (process.stdout.isTTY && !opts.json && opts.output !== "json") {
const { startProvidersTestTui } = await import("../tui/ProvidersTestAll.jsx");
const baseUrl = opts.baseUrl ?? "http://localhost:20128";
const apiKey = opts.apiKey ?? process.env.OMNIROUTE_API_KEY;
await startProvidersTestTui({ providers, baseUrl, apiKey });
return 0;
}
const results = await Promise.all(
providers.map(async ({ provider, model }) => {
const r = await _runSingleTest(provider, model);
return { provider, model, ...r };
})
);
if (opts.json || opts.output === "json") {
console.log(JSON.stringify(results, null, 2));
} else {
for (const r of results) {
const mark = r.success ? "\x1b[32m✔\x1b[0m" : "\x1b[31m✖\x1b[0m";
console.log(`${mark} ${r.provider}/${r.model ?? "-"}`);
}
}
const failed = results.filter((r) => !r.success).length;
return failed > 0 ? 1 : 0;
}
async function _runCompare(provider, opts) {
const targetProvider = provider || "anthropic";
const models = opts.compare

View File

@@ -290,7 +290,8 @@
"saveOpt": "Save results to a JSON file",
"saved": "Results saved to {path}",
"compareTitle": "Model Comparison",
"compareMinTwo": "At least two models required for --compare (comma-separated)"
"compareMinTwo": "At least two models required for --compare (comma-separated)",
"noProviders": "No providers configured. Add one with: omniroute keys add"
},
"update": {
"checking": "Checking for updates...",

View File

@@ -290,7 +290,8 @@
"saveOpt": "Salvar resultados em arquivo JSON",
"saved": "Resultados salvos em {path}",
"compareTitle": "Comparação de Modelos",
"compareMinTwo": "São necessários pelo menos dois modelos para --compare (separados por vírgula)"
"compareMinTwo": "São necessários pelo menos dois modelos para --compare (separados por vírgula)",
"noProviders": "Nenhum provedor configurado. Adicione um com: omniroute keys add"
},
"update": {
"checking": "Verificando atualizações...",

175
bin/cli/tui/EvalWatch.jsx Normal file
View File

@@ -0,0 +1,175 @@
import React, { useState, useEffect, useCallback } from "react";
import { render, Box, Text, useInput } from "ink";
import Spinner from "ink-spinner";
import { ProgressBar } from "../tui-components/ProgressBar.jsx";
import { StatusBadge } from "../tui-components/StatusBadge.jsx";
import { DataTable } from "../tui-components/DataTable.jsx";
import { HeaderSwr } from "../tui-components/HeaderSwr.jsx";
const TERMINAL_STATUSES = new Set(["completed", "failed", "cancelled"]);
const RESULT_SCHEMA = [
{ key: "idx", header: "#", width: 5 },
{ key: "status", header: "Status", width: 10, formatter: (v) => (v === "pass" ? "✔" : "✖") },
{ key: "model", header: "Model", width: 22 },
{ key: "score", header: "Score", width: 8, formatter: (v) => (v != null ? String(v) : "-") },
{ key: "latencyMs", header: "ms", width: 8, formatter: (v) => (v != null ? String(v) : "-") },
];
function EvalWatchApp({ runId, suiteId, baseUrl, apiKey, onExit }) {
const [run, setRun] = useState(null);
const [results, setResults] = useState([]);
const [paused, setPaused] = useState(false);
const [done, setDone] = useState(false);
const [elapsed, setElapsed] = useState(0);
const fetchUrl = `${baseUrl ?? "http://localhost:20128"}/api/evals/${runId}`;
const headers = apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
useEffect(() => {
const id = setInterval(() => setElapsed((e) => e + 1), 1000);
return () => clearInterval(id);
}, []);
const fetcher = useCallback(async () => {
if (paused) return null;
const res = await fetch(fetchUrl, { headers });
if (!res.ok) return null;
return res.json();
}, [fetchUrl, paused]);
useEffect(() => {
if (!run) return;
if (TERMINAL_STATUSES.has(run.status)) {
setDone(true);
}
const samples = run.samples ?? run.results ?? [];
setResults(
samples.map((s, i) => ({
idx: i + 1,
status: s.pass ? "pass" : "fail",
model: s.model ?? "-",
score: s.score,
latencyMs: s.latencyMs,
}))
);
}, [run]);
useInput((input, key) => {
if (input === "q" || (key.ctrl && input === "c")) onExit?.();
if (input === "p") setPaused((p) => !p);
});
const total = run?.progress?.total ?? 0;
const completed = run?.progress?.completed ?? 0;
const passed = run?.progress?.passed ?? results.filter((r) => r.status === "pass").length;
const failed = run?.progress?.failed ?? results.filter((r) => r.status === "fail").length;
const pct = total > 0 ? Math.round((completed / total) * 100) : 0;
return (
<Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={1} paddingY={1}>
<Box marginBottom={1} justifyContent="space-between">
<Text bold color="cyan">
Eval Watch Run {runId}
{suiteId ? ` (suite: ${suiteId})` : ""}
</Text>
<Text dimColor>
{Math.floor(elapsed / 60)}:{String(elapsed % 60).padStart(2, "0")}
{paused ? " [PAUSED]" : ""}
</Text>
</Box>
<HeaderSwr
fetcher={fetcher}
interval={done ? 0 : 3000}
render={(data) => {
if (data && data !== run) setRun(data);
return null;
}}
initial={null}
/>
{run ? (
<>
<Box marginBottom={1}>
<StatusBadge
status={
run.status === "running"
? "running"
: run.status === "completed"
? "running"
: "error"
}
/>
<Text> {run.status} </Text>
<Text dimColor>
{completed}/{total || "?"} samples
</Text>
</Box>
{total > 0 && (
<Box marginBottom={1} flexDirection="column">
<ProgressBar value={pct} total={100} color="cyan" />
<Box marginTop={0}>
<Text color="green"> {passed} passed</Text>
<Text> </Text>
<Text color="red"> {failed} failed</Text>
</Box>
</Box>
)}
{results.length > 0 && (
<Box flexDirection="column" marginTop={1}>
<Text bold dimColor>
Recent results
</Text>
<DataTable rows={results.slice(-10)} schema={RESULT_SCHEMA} />
</Box>
)}
{done && (
<Box marginTop={1}>
<Text bold color={run.status === "completed" ? "green" : "red"}>
{run.status === "completed"
? `✔ Eval completed — ${passed}/${total} passed`
: `✖ Eval ${run.status}`}
</Text>
</Box>
)}
</>
) : (
<Box>
<Text color="green">
<Spinner type="dots" />
</Text>
<Text> Loading...</Text>
</Box>
)}
<Box marginTop={1}>
<Text dimColor>[q] quit [p] {paused ? "resume" : "pause"}</Text>
</Box>
</Box>
);
}
export async function startEvalWatchTui({ runId, suiteId, baseUrl, apiKey }) {
return new Promise((resolve, reject) => {
function onExit() {
unmount();
resolve();
}
const { unmount, waitUntilExit } = render(
<EvalWatchApp
runId={runId}
suiteId={suiteId}
baseUrl={baseUrl}
apiKey={apiKey}
onExit={onExit}
/>
);
waitUntilExit().then(resolve).catch(reject);
});
}

172
bin/cli/tui/OAuthFlow.jsx Normal file
View File

@@ -0,0 +1,172 @@
import React, { useState, useEffect } from "react";
import { render, Box, Text, useInput } from "ink";
import Spinner from "ink-spinner";
import { StatusBadge } from "../tui-components/StatusBadge.jsx";
import { ConfirmDialog } from "../tui-components/ConfirmDialog.jsx";
const PHASE = {
WAITING: "waiting",
POLLING: "polling",
DONE: "done",
FAILED: "failed",
CANCELLED: "cancelled",
};
function OAuthFlowApp({ provider, url, deviceCode, onCancel, onDone, onFail }) {
const [phase, setPhase] = useState(PHASE.WAITING);
const [result, setResult] = useState(null);
const [error, setError] = useState(null);
const [elapsed, setElapsed] = useState(0);
const [confirmCancel, setConfirmCancel] = useState(false);
useEffect(() => {
const id = setInterval(() => setElapsed((e) => e + 1), 1000);
return () => clearInterval(id);
}, []);
useEffect(() => {
if (!onDone && !onFail) return;
setPhase(PHASE.POLLING);
}, []);
useInput((input, key) => {
if (phase === PHASE.DONE || phase === PHASE.FAILED || phase === PHASE.CANCELLED) return;
if (input === "q" || (key.ctrl && input === "c")) {
setConfirmCancel(true);
}
});
function handleCancelConfirm(yes) {
setConfirmCancel(false);
if (yes) {
setPhase(PHASE.CANCELLED);
onCancel?.();
}
}
const elapsed_str = `${Math.floor(elapsed / 60)}:${String(elapsed % 60).padStart(2, "0")}`;
if (confirmCancel) {
return (
<Box flexDirection="column" padding={1}>
<ConfirmDialog
message="Cancel OAuth authorization?"
onConfirm={handleCancelConfirm}
defaultNo
/>
</Box>
);
}
return (
<Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={2} paddingY={1}>
<Box marginBottom={1}>
<Text bold color="cyan">
OmniRoute OAuth {provider}
</Text>
</Box>
{url && (
<Box flexDirection="column" marginBottom={1}>
<Text>Open this URL in your browser to authorize:</Text>
<Box marginTop={0}>
<Text bold color="yellow">
{url}
</Text>
</Box>
</Box>
)}
{deviceCode && (
<Box flexDirection="column" marginBottom={1}>
<Text>
Device code:{" "}
<Text bold color="yellow">
{deviceCode}
</Text>
</Text>
</Box>
)}
<Box marginTop={1}>
{phase === PHASE.POLLING || phase === PHASE.WAITING ? (
<Box>
<Text color="green">
<Spinner type="dots" />
</Text>
<Text> Waiting for authorization... </Text>
<Text dimColor>({elapsed_str})</Text>
</Box>
) : phase === PHASE.DONE ? (
<Box>
<StatusBadge status="running" />
<Text> Authorized: {result?.email ?? result?.account ?? "connected"}</Text>
</Box>
) : phase === PHASE.FAILED ? (
<Box>
<StatusBadge status="error" />
<Text> Failed: {error}</Text>
</Box>
) : (
<Box>
<StatusBadge status="warn" />
<Text> Cancelled.</Text>
</Box>
)}
</Box>
{(phase === PHASE.POLLING || phase === PHASE.WAITING) && (
<Box marginTop={1}>
<Text dimColor>[q] cancel</Text>
</Box>
)}
</Box>
);
}
export async function startOAuthTui({ provider, url, deviceCode }) {
return new Promise((resolve, reject) => {
let resolved = false;
function onDone(result) {
if (resolved) return;
resolved = true;
unmount();
resolve({ status: "authorized", result });
}
function onFail(err) {
if (resolved) return;
resolved = true;
unmount();
resolve({ status: "failed", error: err });
}
function onCancel() {
if (resolved) return;
resolved = true;
unmount();
resolve({ status: "cancelled" });
}
const { unmount, waitUntilExit } = render(
<OAuthFlowApp
provider={provider}
url={url}
deviceCode={deviceCode}
onDone={onDone}
onFail={onFail}
onCancel={onCancel}
/>
);
waitUntilExit()
.then(() => {
if (!resolved) resolve({ status: "exited" });
})
.catch(reject);
});
}
export function markOAuthDone(result) {}
export function markOAuthFailed(error) {}

View File

@@ -0,0 +1,189 @@
import React, { useState, useEffect, useCallback } from "react";
import { render, Box, Text, useInput } from "ink";
import Spinner from "ink-spinner";
import { DataTable } from "../tui-components/DataTable.jsx";
import { ProgressBar } from "../tui-components/ProgressBar.jsx";
const STATUS = {
PENDING: "pending",
RUNNING: "running",
PASS: "pass",
FAIL: "fail",
SKIP: "skip",
};
const TABLE_SCHEMA = [
{ key: "provider", header: "Provider", width: 28 },
{ key: "model", header: "Model", width: 32 },
{
key: "status",
header: "Status",
width: 10,
formatter: (v) => {
if (v === STATUS.RUNNING) return "…";
if (v === STATUS.PASS) return "✔";
if (v === STATUS.FAIL) return "✖";
if (v === STATUS.SKIP) return "—";
return "·";
},
},
{ key: "latencyMs", header: "ms", width: 8, formatter: (v) => (v != null ? String(v) : "-") },
{ key: "error", header: "Error", width: 28, formatter: (v) => (v ? v.slice(0, 26) : "") },
];
async function testOne(provider, model, baseUrl, apiKey) {
const headers = {
"Content-Type": "application/json",
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
};
const start = Date.now();
try {
const res = await fetch(`${baseUrl}/api/v1/providers/test`, {
method: "POST",
headers,
body: JSON.stringify({ provider, model }),
signal: AbortSignal.timeout(30000),
});
const latencyMs = Date.now() - start;
const data = res.ok ? await res.json() : { success: false, error: `HTTP ${res.status}` };
return { status: data.success ? STATUS.PASS : STATUS.FAIL, latencyMs, error: data.error };
} catch (err) {
return {
status: STATUS.FAIL,
latencyMs: Date.now() - start,
error: err instanceof Error ? err.message : String(err),
};
}
}
function ProvidersTestAllApp({ providers, baseUrl, apiKey, concurrency = 4, onExit }) {
const resolved = `${baseUrl ?? "http://localhost:20128"}`;
const [rows, setRows] = useState(() =>
providers.map((p, i) => ({
id: i,
provider: p.provider ?? p.id ?? String(p),
model: p.model ?? p.defaultModel ?? "",
status: STATUS.PENDING,
latencyMs: null,
error: null,
}))
);
const [done, setDone] = useState(false);
const [started, setStarted] = useState(false);
const update = useCallback((id, patch) => {
setRows((prev) => prev.map((r) => (r.id === id ? { ...r, ...patch } : r)));
}, []);
useEffect(() => {
if (started) return;
setStarted(true);
async function runAll() {
const queue = [...rows];
let running = 0;
let cursor = 0;
function nextSlot() {
while (running < concurrency && cursor < queue.length) {
const row = queue[cursor++];
running++;
update(row.id, { status: STATUS.RUNNING });
testOne(row.provider, row.model, resolved, apiKey).then((result) => {
update(row.id, result);
running--;
nextSlot();
if (cursor >= queue.length && running === 0) setDone(true);
});
}
}
nextSlot();
}
runAll();
}, []);
useInput((input, key) => {
if (input === "q" || (key.ctrl && input === "c")) onExit?.();
});
const total = rows.length;
const completed = rows.filter((r) => r.status === STATUS.PASS || r.status === STATUS.FAIL).length;
const passed = rows.filter((r) => r.status === STATUS.PASS).length;
const failed = rows.filter((r) => r.status === STATUS.FAIL).length;
const running = rows.filter((r) => r.status === STATUS.RUNNING).length;
const pct = total > 0 ? Math.round((completed / total) * 100) : 0;
return (
<Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={1} paddingY={1}>
<Box marginBottom={1} justifyContent="space-between">
<Text bold color="cyan">
Providers Test All
</Text>
<Text dimColor>
{running > 0 ? (
<>
<Spinner type="dots" /> {running} running
</>
) : done ? (
"done"
) : (
"queued"
)}
</Text>
</Box>
<Box marginBottom={1} flexDirection="column">
<ProgressBar value={pct} total={100} color={done && failed > 0 ? "red" : "cyan"} />
<Box marginTop={0}>
<Text>
{completed}/{total}
</Text>
<Text> </Text>
<Text color="green"> {passed}</Text>
<Text> </Text>
<Text color="red"> {failed}</Text>
</Box>
</Box>
<DataTable rows={rows} schema={TABLE_SCHEMA} />
{done && (
<Box marginTop={1}>
<Text bold color={failed === 0 ? "green" : "yellow"}>
{failed === 0
? `All ${passed} providers passed!`
: `${passed} passed, ${failed} failed`}
</Text>
</Box>
)}
<Box marginTop={1}>
<Text dimColor>[q] quit</Text>
</Box>
</Box>
);
}
export async function startProvidersTestTui({ providers, baseUrl, apiKey, concurrency = 4 }) {
return new Promise((resolve, reject) => {
function onExit() {
unmount();
resolve();
}
const { unmount, waitUntilExit } = render(
<ProvidersTestAllApp
providers={providers}
baseUrl={baseUrl}
apiKey={apiKey}
concurrency={concurrency}
onExit={onExit}
/>
);
waitUntilExit().then(resolve).catch(reject);
});
}

View File

@@ -134,6 +134,39 @@ test("test-provider — registerTestProvider registra flags latency/repeat/compa
assert.ok(optNames.includes("--save"), "--save deve existir");
});
test("OAuthFlow.jsx — arquivo existe e exporta startOAuthTui", async () => {
const { readFileSync } = await import("node:fs");
const { fileURLToPath } = await import("node:url");
const path = new URL("../../bin/cli/tui/OAuthFlow.jsx", import.meta.url);
const src = readFileSync(fileURLToPath(path), "utf8");
assert.ok(
src.includes("export async function startOAuthTui"),
"startOAuthTui deve ser exportada"
);
});
test("EvalWatch.jsx — arquivo existe e exporta startEvalWatchTui", async () => {
const { readFileSync } = await import("node:fs");
const { fileURLToPath } = await import("node:url");
const path = new URL("../../bin/cli/tui/EvalWatch.jsx", import.meta.url);
const src = readFileSync(fileURLToPath(path), "utf8");
assert.ok(
src.includes("export async function startEvalWatchTui"),
"startEvalWatchTui deve ser exportada"
);
});
test("ProvidersTestAll.jsx — arquivo existe e exporta startProvidersTestTui", async () => {
const { readFileSync } = await import("node:fs");
const { fileURLToPath } = await import("node:url");
const path = new URL("../../bin/cli/tui/ProvidersTestAll.jsx", import.meta.url);
const src = readFileSync(fileURLToPath(path), "utf8");
assert.ok(
src.includes("export async function startProvidersTestTui"),
"startProvidersTestTui deve ser exportada"
);
});
test("test-provider — compare requer pelo menos dois modelos sem server retorna 0 ou 1", async () => {
const { runTestProviderCommand } = await import("../../bin/cli/commands/test-provider.mjs");
let code = 0;