feat(cli): adicionar omniroute providers metrics (Fase 2.6)

Extende o grupo `providers` com dois subcomandos:
- `providers metrics`: lista métricas de performance de todos os provedores
  (latência avg/P95, success rate, custo, breaker state, erros).
  Suporta --provider, --connection-id, --period, --sort-by, --limit,
  --watch (refresh 5s) e --compare (filtro multi-provider).
- `providers metric <id> <metric>`: retorna valor escalar de uma métrica
  específica para uma conexão.

Fonte: GET /api/provider-metrics — normaliza tanto formato objeto
{metrics: {[provider]: {}}} quanto array de providers.
This commit is contained in:
diegosouzapw
2026-05-15 01:10:10 -03:00
parent 85489a0295
commit ba14064ea4
4 changed files with 300 additions and 2 deletions

View File

@@ -1,3 +1,5 @@
import { apiFetch } from "../api.mjs";
import { emit } from "../output.mjs";
import { printHeading } from "../io.mjs";
import { getAvailableProviderCategories, loadAvailableProviders } from "../provider-catalog.mjs";
import { testProviderApiKey } from "../provider-test.mjs";
@@ -354,4 +356,128 @@ export function registerProviders(program) {
const exitCode = await runValidateCommand({ ...opts, output: globalOpts.output });
if (exitCode !== 0) process.exit(exitCode);
});
extendProvidersMetrics(providers);
}
const providerMetricsSchema = [
{ key: "provider", header: "Provider", width: 20 },
{ key: "requests", header: "Reqs", formatter: (v) => (v != null ? v.toLocaleString() : "0") },
{
key: "successRate",
header: "Success %",
formatter: (v) => (v != null ? `${(v * 100).toFixed(1)}%` : "-"),
},
{
key: "avgLatencyMs",
header: "Avg Latency",
formatter: (v) => (v ? `${Math.round(v)}ms` : "-"),
},
{ key: "latencyP95Ms", header: "P95", formatter: (v) => (v ? `${v}ms` : "-") },
{
key: "costUsd",
header: "Cost (USD)",
formatter: (v) => (v != null ? `$${Number(v).toFixed(4)}` : "-"),
},
{ key: "errors", header: "Errors", formatter: (v) => (v != null ? String(v) : "0") },
{ key: "breakerState", header: "Breaker" },
];
function sortRows(rows, sortBy) {
if (!sortBy) return rows;
return [...rows].sort((a, b) => {
const av = a[sortBy] ?? 0;
const bv = b[sortBy] ?? 0;
return bv > av ? 1 : bv < av ? -1 : 0;
});
}
export async function runProvidersMetrics(opts, cmd) {
const globalOpts = cmd.optsWithGlobals();
const fetchMetrics = async () => {
const params = new URLSearchParams({ period: opts.period ?? "24h" });
if (opts.provider) params.set("provider", opts.provider);
if (opts.connectionId) params.set("connectionId", opts.connectionId);
if (opts.compare) params.set("providers", opts.compare);
const res = await apiFetch(`/api/provider-metrics?${params}`, {
timeout: globalOpts.timeout,
acceptNotOk: true,
});
if (!res.ok) return [];
const data = await res.json();
const raw = data.metrics ?? data.items ?? data.providers ?? data;
if (Array.isArray(raw)) return raw;
return Object.entries(raw).map(([provider, m]) => ({
provider,
...(typeof m === "object" && m !== null ? m : {}),
}));
};
const renderOnce = async () => {
const rows = await fetchMetrics();
const sorted = sortRows(rows, opts.sortBy);
emit(sorted.slice(0, opts.limit ?? 50), globalOpts, providerMetricsSchema);
};
if (opts.watch) {
process.stderr.write("[watching — Ctrl+C to exit]\n");
await renderOnce();
const interval = setInterval(async () => {
process.stdout.write("\x1b[2J\x1b[H");
await renderOnce();
}, 5000);
process.on("SIGINT", () => {
clearInterval(interval);
process.exit(0);
});
return;
}
await renderOnce();
}
export async function runProviderMetricSingle(id, metric, opts, cmd) {
const globalOpts = cmd.optsWithGlobals();
const params = new URLSearchParams({ connectionId: id, period: opts.period ?? "24h" });
const res = await apiFetch(`/api/provider-metrics?${params}`, {
timeout: globalOpts.timeout,
acceptNotOk: true,
});
if (!res.ok) {
process.stderr.write(t("common.serverOffline") + "\n");
process.exit(res.exitCode ?? 1);
}
const data = await res.json();
const raw = data.metrics ?? data;
const connData =
raw[id] ??
(Array.isArray(raw) ? raw.find((r) => r.connectionId === id || r.provider === id) : null);
const value = connData?.[metric] ?? connData;
if (globalOpts.output === "json") {
process.stdout.write(JSON.stringify(value, null, 2) + "\n");
} else {
process.stdout.write(`${metric}: ${JSON.stringify(value)}\n`);
}
}
export function extendProvidersMetrics(providers) {
providers
.command("metrics")
.description(t("providers.metrics.description"))
.option("--provider <id>", t("providers.metrics.provider"))
.option("--connection-id <id>", t("providers.metrics.connection_id"))
.option("--period <p>", t("providers.metrics.period"), "24h")
.option("--metric <m>", t("providers.metrics.metric"))
.option("--sort-by <field>", t("providers.metrics.sort"))
.option("--limit <n>", t("providers.metrics.limit"), parseInt, 50)
.option("--watch", t("providers.metrics.watch"))
.option("--compare <list>", t("providers.metrics.compare"))
.action(runProvidersMetrics);
providers
.command("metric <connectionId> <metric>")
.description(t("providers.metric_single.description"))
.option("--period <p>", t("providers.metrics.period"), "24h")
.action(runProviderMetricSingle);
}

View File

@@ -44,7 +44,21 @@
"available": "{count} provider(s) available",
"connected": "Connected",
"disconnected": "Disconnected",
"validationFailed": "Validation failed: {error}"
"validationFailed": "Validation failed: {error}",
"metrics": {
"description": "Show provider performance metrics (latency, success rate, cost)",
"provider": "Filter by provider ID",
"connection_id": "Filter by connection ID",
"period": "Time period: 1h|6h|24h|7d|30d (default: 24h)",
"metric": "Focus on a specific metric field",
"sort": "Sort by field (descending)",
"limit": "Maximum rows (default: 50)",
"watch": "Refresh every 5s (live mode)",
"compare": "Comma-separated provider IDs to compare side-by-side"
},
"metric_single": {
"description": "Get a single metric value for a specific connection"
}
},
"keys": {
"title": "API Keys",

View File

@@ -44,7 +44,21 @@
"available": "{count} provedor(es) disponível(is)",
"connected": "Conectado",
"disconnected": "Desconectado",
"validationFailed": "Validação falhou: {error}"
"validationFailed": "Validação falhou: {error}",
"metrics": {
"description": "Exibir métricas de performance dos provedores (latência, taxa de sucesso, custo)",
"provider": "Filtrar por ID de provedor",
"connection_id": "Filtrar por ID de conexão",
"period": "Período: 1h|6h|24h|7d|30d (padrão: 24h)",
"metric": "Focar em um campo de métrica específico",
"sort": "Ordenar por campo (decrescente)",
"limit": "Máximo de linhas (padrão: 50)",
"watch": "Atualizar a cada 5s (modo live)",
"compare": "IDs de provedores separados por vírgula para comparar"
},
"metric_single": {
"description": "Obter um único valor de métrica para uma conexão específica"
}
},
"keys": {
"title": "Chaves de API",

View File

@@ -0,0 +1,144 @@
import test from "node:test";
import assert from "node:assert/strict";
const METRICS_OBJECT = {
metrics: {
openai: { totalRequests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 },
anthropic: { totalRequests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 },
gemini: { totalRequests: 80, successRate: 0.95, avgLatencyMs: 600, errors: 4 },
},
};
const METRICS_ARRAY = {
providers: [
{ provider: "openai", requests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 },
{ provider: "anthropic", requests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 },
],
};
function makeResp(data: unknown, status = 200) {
const obj = {
ok: status < 400,
status,
exitCode: status < 400 ? 0 : 1,
json: () => Promise.resolve(data),
text: () => Promise.resolve(JSON.stringify(data)),
headers: new Headers(),
};
obj.json = obj.json.bind(obj);
obj.text = obj.text.bind(obj);
return obj;
}
async function captureStdout(fn: () => Promise<void>): Promise<string> {
const chunks: string[] = [];
const orig = process.stdout.write.bind(process.stdout);
process.stdout.write = (c: string | Uint8Array) => {
chunks.push(typeof c === "string" ? c : c.toString());
return true;
};
try {
await fn();
} finally {
process.stdout.write = orig;
}
return chunks.join("");
}
function makeCmd(output = "json") {
return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) };
}
test("runProvidersMetrics --output json normaliza objeto de metrics", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
const out = await captureStdout(() =>
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed.length, 3);
assert.ok(parsed.some((r: any) => r.provider === "openai"));
assert.ok(parsed.some((r: any) => r.provider === "anthropic"));
});
test("runProvidersMetrics --output json aceita array de providers", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_ARRAY))) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
const out = await captureStdout(() =>
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.ok(parsed.length >= 1);
assert.ok(parsed.some((r: any) => r.provider === "openai"));
});
test("runProvidersMetrics --limit trunca resultado", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
const out = await captureStdout(() =>
runProvidersMetrics({ period: "24h", limit: 2 }, makeCmd() as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.equal(parsed.length, 2);
});
test("runProvidersMetrics envia --provider na query", async () => {
let capturedUrl = "";
const origFetch = globalThis.fetch;
globalThis.fetch = ((url: string) => {
capturedUrl = url;
return Promise.resolve(makeResp(METRICS_OBJECT));
}) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
await captureStdout(() =>
runProvidersMetrics({ period: "7d", provider: "openai", limit: 50 }, makeCmd() as any)
);
globalThis.fetch = origFetch;
assert.ok(capturedUrl.includes("provider=openai"));
assert.ok(capturedUrl.includes("period=7d"));
});
test("runProvidersMetrics exibe tabela com success rate formatada", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
const out = await captureStdout(() =>
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd("table") as any)
);
globalThis.fetch = origFetch;
assert.ok(out.includes("openai") || out.includes("Provider"));
assert.ok(out.includes("%") || out.includes("Success"));
});
test("runProvidersMetrics retorna vazio quando endpoint retorna 404", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = (() => Promise.resolve(makeResp({}, 404))) as any;
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
const out = await captureStdout(() =>
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(out);
assert.ok(Array.isArray(parsed));
assert.equal(parsed.length, 0);
});