diff --git a/bin/cli/commands/providers.mjs b/bin/cli/commands/providers.mjs index b9affdeb8c..f197bbf598 100644 --- a/bin/cli/commands/providers.mjs +++ b/bin/cli/commands/providers.mjs @@ -1,3 +1,5 @@ +import { apiFetch } from "../api.mjs"; +import { emit } from "../output.mjs"; import { printHeading } from "../io.mjs"; import { getAvailableProviderCategories, loadAvailableProviders } from "../provider-catalog.mjs"; import { testProviderApiKey } from "../provider-test.mjs"; @@ -354,4 +356,128 @@ export function registerProviders(program) { const exitCode = await runValidateCommand({ ...opts, output: globalOpts.output }); if (exitCode !== 0) process.exit(exitCode); }); + + extendProvidersMetrics(providers); +} + +const providerMetricsSchema = [ + { key: "provider", header: "Provider", width: 20 }, + { key: "requests", header: "Reqs", formatter: (v) => (v != null ? v.toLocaleString() : "0") }, + { + key: "successRate", + header: "Success %", + formatter: (v) => (v != null ? `${(v * 100).toFixed(1)}%` : "-"), + }, + { + key: "avgLatencyMs", + header: "Avg Latency", + formatter: (v) => (v ? `${Math.round(v)}ms` : "-"), + }, + { key: "latencyP95Ms", header: "P95", formatter: (v) => (v ? `${v}ms` : "-") }, + { + key: "costUsd", + header: "Cost (USD)", + formatter: (v) => (v != null ? `$${Number(v).toFixed(4)}` : "-"), + }, + { key: "errors", header: "Errors", formatter: (v) => (v != null ? String(v) : "0") }, + { key: "breakerState", header: "Breaker" }, +]; + +function sortRows(rows, sortBy) { + if (!sortBy) return rows; + return [...rows].sort((a, b) => { + const av = a[sortBy] ?? 0; + const bv = b[sortBy] ?? 0; + return bv > av ? 1 : bv < av ? -1 : 0; + }); +} + +export async function runProvidersMetrics(opts, cmd) { + const globalOpts = cmd.optsWithGlobals(); + + const fetchMetrics = async () => { + const params = new URLSearchParams({ period: opts.period ?? "24h" }); + if (opts.provider) params.set("provider", opts.provider); + if (opts.connectionId) params.set("connectionId", opts.connectionId); + if (opts.compare) params.set("providers", opts.compare); + const res = await apiFetch(`/api/provider-metrics?${params}`, { + timeout: globalOpts.timeout, + acceptNotOk: true, + }); + if (!res.ok) return []; + const data = await res.json(); + const raw = data.metrics ?? data.items ?? data.providers ?? data; + if (Array.isArray(raw)) return raw; + return Object.entries(raw).map(([provider, m]) => ({ + provider, + ...(typeof m === "object" && m !== null ? m : {}), + })); + }; + + const renderOnce = async () => { + const rows = await fetchMetrics(); + const sorted = sortRows(rows, opts.sortBy); + emit(sorted.slice(0, opts.limit ?? 50), globalOpts, providerMetricsSchema); + }; + + if (opts.watch) { + process.stderr.write("[watching — Ctrl+C to exit]\n"); + await renderOnce(); + const interval = setInterval(async () => { + process.stdout.write("\x1b[2J\x1b[H"); + await renderOnce(); + }, 5000); + process.on("SIGINT", () => { + clearInterval(interval); + process.exit(0); + }); + return; + } + + await renderOnce(); +} + +export async function runProviderMetricSingle(id, metric, opts, cmd) { + const globalOpts = cmd.optsWithGlobals(); + const params = new URLSearchParams({ connectionId: id, period: opts.period ?? "24h" }); + const res = await apiFetch(`/api/provider-metrics?${params}`, { + timeout: globalOpts.timeout, + acceptNotOk: true, + }); + if (!res.ok) { + process.stderr.write(t("common.serverOffline") + "\n"); + process.exit(res.exitCode ?? 1); + } + const data = await res.json(); + const raw = data.metrics ?? data; + const connData = + raw[id] ?? + (Array.isArray(raw) ? raw.find((r) => r.connectionId === id || r.provider === id) : null); + const value = connData?.[metric] ?? connData; + if (globalOpts.output === "json") { + process.stdout.write(JSON.stringify(value, null, 2) + "\n"); + } else { + process.stdout.write(`${metric}: ${JSON.stringify(value)}\n`); + } +} + +export function extendProvidersMetrics(providers) { + providers + .command("metrics") + .description(t("providers.metrics.description")) + .option("--provider ", t("providers.metrics.provider")) + .option("--connection-id ", t("providers.metrics.connection_id")) + .option("--period

", t("providers.metrics.period"), "24h") + .option("--metric ", t("providers.metrics.metric")) + .option("--sort-by ", t("providers.metrics.sort")) + .option("--limit ", t("providers.metrics.limit"), parseInt, 50) + .option("--watch", t("providers.metrics.watch")) + .option("--compare ", t("providers.metrics.compare")) + .action(runProvidersMetrics); + + providers + .command("metric ") + .description(t("providers.metric_single.description")) + .option("--period

", t("providers.metrics.period"), "24h") + .action(runProviderMetricSingle); } diff --git a/bin/cli/locales/en.json b/bin/cli/locales/en.json index f090385c3e..af144fc735 100644 --- a/bin/cli/locales/en.json +++ b/bin/cli/locales/en.json @@ -44,7 +44,21 @@ "available": "{count} provider(s) available", "connected": "Connected", "disconnected": "Disconnected", - "validationFailed": "Validation failed: {error}" + "validationFailed": "Validation failed: {error}", + "metrics": { + "description": "Show provider performance metrics (latency, success rate, cost)", + "provider": "Filter by provider ID", + "connection_id": "Filter by connection ID", + "period": "Time period: 1h|6h|24h|7d|30d (default: 24h)", + "metric": "Focus on a specific metric field", + "sort": "Sort by field (descending)", + "limit": "Maximum rows (default: 50)", + "watch": "Refresh every 5s (live mode)", + "compare": "Comma-separated provider IDs to compare side-by-side" + }, + "metric_single": { + "description": "Get a single metric value for a specific connection" + } }, "keys": { "title": "API Keys", diff --git a/bin/cli/locales/pt-BR.json b/bin/cli/locales/pt-BR.json index 9bd15feeb1..2328cc6f3e 100644 --- a/bin/cli/locales/pt-BR.json +++ b/bin/cli/locales/pt-BR.json @@ -44,7 +44,21 @@ "available": "{count} provedor(es) disponível(is)", "connected": "Conectado", "disconnected": "Desconectado", - "validationFailed": "Validação falhou: {error}" + "validationFailed": "Validação falhou: {error}", + "metrics": { + "description": "Exibir métricas de performance dos provedores (latência, taxa de sucesso, custo)", + "provider": "Filtrar por ID de provedor", + "connection_id": "Filtrar por ID de conexão", + "period": "Período: 1h|6h|24h|7d|30d (padrão: 24h)", + "metric": "Focar em um campo de métrica específico", + "sort": "Ordenar por campo (decrescente)", + "limit": "Máximo de linhas (padrão: 50)", + "watch": "Atualizar a cada 5s (modo live)", + "compare": "IDs de provedores separados por vírgula para comparar" + }, + "metric_single": { + "description": "Obter um único valor de métrica para uma conexão específica" + } }, "keys": { "title": "Chaves de API", diff --git a/tests/unit/cli-providers-metrics.test.ts b/tests/unit/cli-providers-metrics.test.ts new file mode 100644 index 0000000000..d41575ec91 --- /dev/null +++ b/tests/unit/cli-providers-metrics.test.ts @@ -0,0 +1,144 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +const METRICS_OBJECT = { + metrics: { + openai: { totalRequests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 }, + anthropic: { totalRequests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 }, + gemini: { totalRequests: 80, successRate: 0.95, avgLatencyMs: 600, errors: 4 }, + }, +}; + +const METRICS_ARRAY = { + providers: [ + { provider: "openai", requests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 }, + { provider: "anthropic", requests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 }, + ], +}; + +function makeResp(data: unknown, status = 200) { + const obj = { + ok: status < 400, + status, + exitCode: status < 400 ? 0 : 1, + json: () => Promise.resolve(data), + text: () => Promise.resolve(JSON.stringify(data)), + headers: new Headers(), + }; + obj.json = obj.json.bind(obj); + obj.text = obj.text.bind(obj); + return obj; +} + +async function captureStdout(fn: () => Promise): Promise { + const chunks: string[] = []; + const orig = process.stdout.write.bind(process.stdout); + process.stdout.write = (c: string | Uint8Array) => { + chunks.push(typeof c === "string" ? c : c.toString()); + return true; + }; + try { + await fn(); + } finally { + process.stdout.write = orig; + } + return chunks.join(""); +} + +function makeCmd(output = "json") { + return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) }; +} + +test("runProvidersMetrics --output json normaliza objeto de metrics", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + const out = await captureStdout(() => + runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed.length, 3); + assert.ok(parsed.some((r: any) => r.provider === "openai")); + assert.ok(parsed.some((r: any) => r.provider === "anthropic")); +}); + +test("runProvidersMetrics --output json aceita array de providers", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_ARRAY))) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + const out = await captureStdout(() => + runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.ok(Array.isArray(parsed)); + assert.ok(parsed.length >= 1); + assert.ok(parsed.some((r: any) => r.provider === "openai")); +}); + +test("runProvidersMetrics --limit trunca resultado", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + const out = await captureStdout(() => + runProvidersMetrics({ period: "24h", limit: 2 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.equal(parsed.length, 2); +}); + +test("runProvidersMetrics envia --provider na query", async () => { + let capturedUrl = ""; + const origFetch = globalThis.fetch; + globalThis.fetch = ((url: string) => { + capturedUrl = url; + return Promise.resolve(makeResp(METRICS_OBJECT)); + }) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + await captureStdout(() => + runProvidersMetrics({ period: "7d", provider: "openai", limit: 50 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + assert.ok(capturedUrl.includes("provider=openai")); + assert.ok(capturedUrl.includes("period=7d")); +}); + +test("runProvidersMetrics exibe tabela com success rate formatada", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + const out = await captureStdout(() => + runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd("table") as any) + ); + + globalThis.fetch = origFetch; + assert.ok(out.includes("openai") || out.includes("Provider")); + assert.ok(out.includes("%") || out.includes("Success")); +}); + +test("runProvidersMetrics retorna vazio quando endpoint retorna 404", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (() => Promise.resolve(makeResp({}, 404))) as any; + + const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs"); + const out = await captureStdout(() => + runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(out); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed.length, 0); +});