mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
feat(cli): adicionar omniroute providers metrics (Fase 2.6)
Extende o grupo `providers` com dois subcomandos:
- `providers metrics`: lista métricas de performance de todos os provedores
(latência avg/P95, success rate, custo, breaker state, erros).
Suporta --provider, --connection-id, --period, --sort-by, --limit,
--watch (refresh 5s) e --compare (filtro multi-provider).
- `providers metric <id> <metric>`: retorna valor escalar de uma métrica
específica para uma conexão.
Fonte: GET /api/provider-metrics — normaliza tanto formato objeto
{metrics: {[provider]: {}}} quanto array de providers.
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
import { apiFetch } from "../api.mjs";
|
||||
import { emit } from "../output.mjs";
|
||||
import { printHeading } from "../io.mjs";
|
||||
import { getAvailableProviderCategories, loadAvailableProviders } from "../provider-catalog.mjs";
|
||||
import { testProviderApiKey } from "../provider-test.mjs";
|
||||
@@ -354,4 +356,128 @@ export function registerProviders(program) {
|
||||
const exitCode = await runValidateCommand({ ...opts, output: globalOpts.output });
|
||||
if (exitCode !== 0) process.exit(exitCode);
|
||||
});
|
||||
|
||||
extendProvidersMetrics(providers);
|
||||
}
|
||||
|
||||
const providerMetricsSchema = [
|
||||
{ key: "provider", header: "Provider", width: 20 },
|
||||
{ key: "requests", header: "Reqs", formatter: (v) => (v != null ? v.toLocaleString() : "0") },
|
||||
{
|
||||
key: "successRate",
|
||||
header: "Success %",
|
||||
formatter: (v) => (v != null ? `${(v * 100).toFixed(1)}%` : "-"),
|
||||
},
|
||||
{
|
||||
key: "avgLatencyMs",
|
||||
header: "Avg Latency",
|
||||
formatter: (v) => (v ? `${Math.round(v)}ms` : "-"),
|
||||
},
|
||||
{ key: "latencyP95Ms", header: "P95", formatter: (v) => (v ? `${v}ms` : "-") },
|
||||
{
|
||||
key: "costUsd",
|
||||
header: "Cost (USD)",
|
||||
formatter: (v) => (v != null ? `$${Number(v).toFixed(4)}` : "-"),
|
||||
},
|
||||
{ key: "errors", header: "Errors", formatter: (v) => (v != null ? String(v) : "0") },
|
||||
{ key: "breakerState", header: "Breaker" },
|
||||
];
|
||||
|
||||
function sortRows(rows, sortBy) {
|
||||
if (!sortBy) return rows;
|
||||
return [...rows].sort((a, b) => {
|
||||
const av = a[sortBy] ?? 0;
|
||||
const bv = b[sortBy] ?? 0;
|
||||
return bv > av ? 1 : bv < av ? -1 : 0;
|
||||
});
|
||||
}
|
||||
|
||||
export async function runProvidersMetrics(opts, cmd) {
|
||||
const globalOpts = cmd.optsWithGlobals();
|
||||
|
||||
const fetchMetrics = async () => {
|
||||
const params = new URLSearchParams({ period: opts.period ?? "24h" });
|
||||
if (opts.provider) params.set("provider", opts.provider);
|
||||
if (opts.connectionId) params.set("connectionId", opts.connectionId);
|
||||
if (opts.compare) params.set("providers", opts.compare);
|
||||
const res = await apiFetch(`/api/provider-metrics?${params}`, {
|
||||
timeout: globalOpts.timeout,
|
||||
acceptNotOk: true,
|
||||
});
|
||||
if (!res.ok) return [];
|
||||
const data = await res.json();
|
||||
const raw = data.metrics ?? data.items ?? data.providers ?? data;
|
||||
if (Array.isArray(raw)) return raw;
|
||||
return Object.entries(raw).map(([provider, m]) => ({
|
||||
provider,
|
||||
...(typeof m === "object" && m !== null ? m : {}),
|
||||
}));
|
||||
};
|
||||
|
||||
const renderOnce = async () => {
|
||||
const rows = await fetchMetrics();
|
||||
const sorted = sortRows(rows, opts.sortBy);
|
||||
emit(sorted.slice(0, opts.limit ?? 50), globalOpts, providerMetricsSchema);
|
||||
};
|
||||
|
||||
if (opts.watch) {
|
||||
process.stderr.write("[watching — Ctrl+C to exit]\n");
|
||||
await renderOnce();
|
||||
const interval = setInterval(async () => {
|
||||
process.stdout.write("\x1b[2J\x1b[H");
|
||||
await renderOnce();
|
||||
}, 5000);
|
||||
process.on("SIGINT", () => {
|
||||
clearInterval(interval);
|
||||
process.exit(0);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await renderOnce();
|
||||
}
|
||||
|
||||
export async function runProviderMetricSingle(id, metric, opts, cmd) {
|
||||
const globalOpts = cmd.optsWithGlobals();
|
||||
const params = new URLSearchParams({ connectionId: id, period: opts.period ?? "24h" });
|
||||
const res = await apiFetch(`/api/provider-metrics?${params}`, {
|
||||
timeout: globalOpts.timeout,
|
||||
acceptNotOk: true,
|
||||
});
|
||||
if (!res.ok) {
|
||||
process.stderr.write(t("common.serverOffline") + "\n");
|
||||
process.exit(res.exitCode ?? 1);
|
||||
}
|
||||
const data = await res.json();
|
||||
const raw = data.metrics ?? data;
|
||||
const connData =
|
||||
raw[id] ??
|
||||
(Array.isArray(raw) ? raw.find((r) => r.connectionId === id || r.provider === id) : null);
|
||||
const value = connData?.[metric] ?? connData;
|
||||
if (globalOpts.output === "json") {
|
||||
process.stdout.write(JSON.stringify(value, null, 2) + "\n");
|
||||
} else {
|
||||
process.stdout.write(`${metric}: ${JSON.stringify(value)}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
export function extendProvidersMetrics(providers) {
|
||||
providers
|
||||
.command("metrics")
|
||||
.description(t("providers.metrics.description"))
|
||||
.option("--provider <id>", t("providers.metrics.provider"))
|
||||
.option("--connection-id <id>", t("providers.metrics.connection_id"))
|
||||
.option("--period <p>", t("providers.metrics.period"), "24h")
|
||||
.option("--metric <m>", t("providers.metrics.metric"))
|
||||
.option("--sort-by <field>", t("providers.metrics.sort"))
|
||||
.option("--limit <n>", t("providers.metrics.limit"), parseInt, 50)
|
||||
.option("--watch", t("providers.metrics.watch"))
|
||||
.option("--compare <list>", t("providers.metrics.compare"))
|
||||
.action(runProvidersMetrics);
|
||||
|
||||
providers
|
||||
.command("metric <connectionId> <metric>")
|
||||
.description(t("providers.metric_single.description"))
|
||||
.option("--period <p>", t("providers.metrics.period"), "24h")
|
||||
.action(runProviderMetricSingle);
|
||||
}
|
||||
|
||||
@@ -44,7 +44,21 @@
|
||||
"available": "{count} provider(s) available",
|
||||
"connected": "Connected",
|
||||
"disconnected": "Disconnected",
|
||||
"validationFailed": "Validation failed: {error}"
|
||||
"validationFailed": "Validation failed: {error}",
|
||||
"metrics": {
|
||||
"description": "Show provider performance metrics (latency, success rate, cost)",
|
||||
"provider": "Filter by provider ID",
|
||||
"connection_id": "Filter by connection ID",
|
||||
"period": "Time period: 1h|6h|24h|7d|30d (default: 24h)",
|
||||
"metric": "Focus on a specific metric field",
|
||||
"sort": "Sort by field (descending)",
|
||||
"limit": "Maximum rows (default: 50)",
|
||||
"watch": "Refresh every 5s (live mode)",
|
||||
"compare": "Comma-separated provider IDs to compare side-by-side"
|
||||
},
|
||||
"metric_single": {
|
||||
"description": "Get a single metric value for a specific connection"
|
||||
}
|
||||
},
|
||||
"keys": {
|
||||
"title": "API Keys",
|
||||
|
||||
@@ -44,7 +44,21 @@
|
||||
"available": "{count} provedor(es) disponível(is)",
|
||||
"connected": "Conectado",
|
||||
"disconnected": "Desconectado",
|
||||
"validationFailed": "Validação falhou: {error}"
|
||||
"validationFailed": "Validação falhou: {error}",
|
||||
"metrics": {
|
||||
"description": "Exibir métricas de performance dos provedores (latência, taxa de sucesso, custo)",
|
||||
"provider": "Filtrar por ID de provedor",
|
||||
"connection_id": "Filtrar por ID de conexão",
|
||||
"period": "Período: 1h|6h|24h|7d|30d (padrão: 24h)",
|
||||
"metric": "Focar em um campo de métrica específico",
|
||||
"sort": "Ordenar por campo (decrescente)",
|
||||
"limit": "Máximo de linhas (padrão: 50)",
|
||||
"watch": "Atualizar a cada 5s (modo live)",
|
||||
"compare": "IDs de provedores separados por vírgula para comparar"
|
||||
},
|
||||
"metric_single": {
|
||||
"description": "Obter um único valor de métrica para uma conexão específica"
|
||||
}
|
||||
},
|
||||
"keys": {
|
||||
"title": "Chaves de API",
|
||||
|
||||
144
tests/unit/cli-providers-metrics.test.ts
Normal file
144
tests/unit/cli-providers-metrics.test.ts
Normal file
@@ -0,0 +1,144 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const METRICS_OBJECT = {
|
||||
metrics: {
|
||||
openai: { totalRequests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 },
|
||||
anthropic: { totalRequests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 },
|
||||
gemini: { totalRequests: 80, successRate: 0.95, avgLatencyMs: 600, errors: 4 },
|
||||
},
|
||||
};
|
||||
|
||||
const METRICS_ARRAY = {
|
||||
providers: [
|
||||
{ provider: "openai", requests: 200, successRate: 0.97, avgLatencyMs: 450, errors: 6 },
|
||||
{ provider: "anthropic", requests: 120, successRate: 0.99, avgLatencyMs: 320, errors: 1 },
|
||||
],
|
||||
};
|
||||
|
||||
function makeResp(data: unknown, status = 200) {
|
||||
const obj = {
|
||||
ok: status < 400,
|
||||
status,
|
||||
exitCode: status < 400 ? 0 : 1,
|
||||
json: () => Promise.resolve(data),
|
||||
text: () => Promise.resolve(JSON.stringify(data)),
|
||||
headers: new Headers(),
|
||||
};
|
||||
obj.json = obj.json.bind(obj);
|
||||
obj.text = obj.text.bind(obj);
|
||||
return obj;
|
||||
}
|
||||
|
||||
async function captureStdout(fn: () => Promise<void>): Promise<string> {
|
||||
const chunks: string[] = [];
|
||||
const orig = process.stdout.write.bind(process.stdout);
|
||||
process.stdout.write = (c: string | Uint8Array) => {
|
||||
chunks.push(typeof c === "string" ? c : c.toString());
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
process.stdout.write = orig;
|
||||
}
|
||||
return chunks.join("");
|
||||
}
|
||||
|
||||
function makeCmd(output = "json") {
|
||||
return { optsWithGlobals: () => ({ output, quiet: output !== "table" }) };
|
||||
}
|
||||
|
||||
test("runProvidersMetrics --output json normaliza objeto de metrics", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
const out = await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(out);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.equal(parsed.length, 3);
|
||||
assert.ok(parsed.some((r: any) => r.provider === "openai"));
|
||||
assert.ok(parsed.some((r: any) => r.provider === "anthropic"));
|
||||
});
|
||||
|
||||
test("runProvidersMetrics --output json aceita array de providers", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_ARRAY))) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
const out = await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(out);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.ok(parsed.length >= 1);
|
||||
assert.ok(parsed.some((r: any) => r.provider === "openai"));
|
||||
});
|
||||
|
||||
test("runProvidersMetrics --limit trunca resultado", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
const out = await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "24h", limit: 2 }, makeCmd() as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(out);
|
||||
assert.equal(parsed.length, 2);
|
||||
});
|
||||
|
||||
test("runProvidersMetrics envia --provider na query", async () => {
|
||||
let capturedUrl = "";
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = ((url: string) => {
|
||||
capturedUrl = url;
|
||||
return Promise.resolve(makeResp(METRICS_OBJECT));
|
||||
}) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "7d", provider: "openai", limit: 50 }, makeCmd() as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
assert.ok(capturedUrl.includes("provider=openai"));
|
||||
assert.ok(capturedUrl.includes("period=7d"));
|
||||
});
|
||||
|
||||
test("runProvidersMetrics exibe tabela com success rate formatada", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => Promise.resolve(makeResp(METRICS_OBJECT))) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
const out = await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd("table") as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
assert.ok(out.includes("openai") || out.includes("Provider"));
|
||||
assert.ok(out.includes("%") || out.includes("Success"));
|
||||
});
|
||||
|
||||
test("runProvidersMetrics retorna vazio quando endpoint retorna 404", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => Promise.resolve(makeResp({}, 404))) as any;
|
||||
|
||||
const { runProvidersMetrics } = await import("../../bin/cli/commands/providers.mjs");
|
||||
const out = await captureStdout(() =>
|
||||
runProvidersMetrics({ period: "24h", limit: 50 }, makeCmd() as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(out);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.equal(parsed.length, 0);
|
||||
});
|
||||
Reference in New Issue
Block a user