feat(cli): adicionar comando simulate para dry-run de routing (Fase 2.3)

Implementa `omniroute simulate [prompt]` que consulta /api/combos,
/api/monitoring/health e /api/usage/quota para mostrar qual caminho
de routing seria escolhido sem executar chamada upstream. Suporta:
- Tabela com provider, model, probabilidade, custo estimado, breaker e quota
- --explain: imprime árvore de fallback e faixa de custo no stderr
- --output json: retorna array de targets completo
- --file <path>: carrega body JSON para estimar tokens
- --combo <name>: filtra por combo específico
- Tratamento gracioso quando servidor está offline (exit 3)
This commit is contained in:
diegosouzapw
2026-05-15 00:45:35 -03:00
parent 7c128b0f4e
commit 18e5bf26a6
5 changed files with 326 additions and 0 deletions

View File

@@ -1,5 +1,6 @@
import { registerChat } from "./chat.mjs";
import { registerStream } from "./stream.mjs";
import { registerSimulate } from "./simulate.mjs";
import { registerServe } from "./serve.mjs";
import { registerStop } from "./stop.mjs";
import { registerRestart } from "./restart.mjs";
@@ -29,6 +30,7 @@ import { registerCompletion } from "./completion.mjs";
export function registerCommands(program) {
registerChat(program);
registerStream(program);
registerSimulate(program);
registerServe(program);
registerStop(program);
registerRestart(program);

View File

@@ -0,0 +1,125 @@
import { readFileSync } from "node:fs";
import { apiFetch } from "../api.mjs";
import { emit } from "../output.mjs";
import { t } from "../i18n.mjs";
const simulateSchema = [
{ key: "order", header: "#", width: 4 },
{ key: "provider", header: "Provider", width: 20 },
{ key: "model", header: "Model", width: 35 },
{ key: "probability", header: "Probability", formatter: (v) => `${Math.round(v * 100)}%` },
{ key: "estimatedCost", header: "Est. Cost", formatter: (v) => (v ? `$${v.toFixed(4)}` : "-") },
{ key: "healthStatus", header: "Breaker", width: 10 },
{ key: "quotaAvailable", header: "Quota %", formatter: (v) => `${v}%` },
];
export function registerSimulate(program) {
program
.command("simulate [prompt]")
.description(t("simulate.description"))
.option("--file <path>", t("simulate.file"))
.option("-m, --model <id>", t("simulate.model"), "auto")
.option("--combo <name>", t("simulate.combo"))
.option("--reasoning-effort <level>", t("simulate.reasoning"))
.option("--thinking-budget <n>", t("simulate.thinking"), parseInt)
.option("--explain", t("simulate.explain"))
.action(runSimulateCommand);
}
export async function runSimulateCommand(promptArg, opts, cmd) {
const globalOpts = cmd.optsWithGlobals();
let promptTokenEstimate = 100;
if (opts.file) {
try {
const raw = readFileSync(opts.file, "utf8");
const parsed = JSON.parse(raw);
const text = JSON.stringify(parsed);
promptTokenEstimate = Math.ceil(text.length / 4);
} catch {
promptTokenEstimate = 100;
}
} else if (promptArg) {
promptTokenEstimate = Math.ceil(promptArg.length / 4);
}
const [combosRes, healthRes, quotaRes] = await Promise.allSettled([
apiFetch("/api/combos", { timeout: globalOpts.timeout }).then((r) => r.json()),
apiFetch("/api/monitoring/health", { timeout: globalOpts.timeout }).then((r) => r.json()),
apiFetch("/api/usage/quota", { timeout: globalOpts.timeout }).then((r) => r.json()),
]);
if (combosRes.status === "rejected") {
process.stderr.write(t("common.serverOffline") + "\n");
process.exit(3);
}
const combos = normalizeCombos(combosRes.value);
const health = healthRes.status === "fulfilled" ? healthRes.value : {};
const quota = quotaRes.status === "fulfilled" ? quotaRes.value : {};
const targetCombo = opts.combo
? combos.find((c) => c.id === opts.combo || c.name === opts.combo)
: combos.find((c) => c.enabled !== false);
if (!targetCombo) {
process.stderr.write(t("simulate.noCombo") + "\n");
process.exit(1);
}
const models = getComboModels(targetCombo, opts.model);
const breakers = toArray(health.circuitBreakers ?? health.breakers);
const providers = toArray(quota.providers ?? quota.data);
const simulatedPath = models.map((m, idx) => {
const cb = breakers.find((b) => String(b.provider) === m.provider);
const q = providers.find((p) => p.provider === m.provider);
const inputCost = m.inputCostPer1M ?? 0;
const estimatedCost = Math.round((promptTokenEstimate / 1_000_000) * inputCost * 10000) / 10000;
return {
order: idx + 1,
provider: m.provider,
model: m.model || opts.model,
probability: idx === 0 ? 0.85 : 0.15 / Math.max(models.length - 1, 1),
estimatedCost,
healthStatus: String(cb?.state ?? "CLOSED"),
quotaAvailable: q?.percentRemaining ?? 100,
};
});
emit(simulatedPath, globalOpts, simulateSchema);
if (opts.explain && !globalOpts.quiet) {
const primary = simulatedPath[0];
const fallbacks = simulatedPath.slice(1).map((s) => s.provider);
process.stderr.write(`\nPrimary: ${primary?.provider} / ${primary?.model}\n`);
if (fallbacks.length > 0) {
process.stderr.write(`Fallbacks: ${fallbacks.join(" → ")}\n`);
}
const costs = simulatedPath.map((s) => s.estimatedCost);
process.stderr.write(
`Est. cost range: $${Math.min(...costs).toFixed(4)} $${Math.max(...costs).toFixed(4)}\n`
);
}
}
function normalizeCombos(raw) {
if (Array.isArray(raw)) return raw;
if (raw && Array.isArray(raw.combos)) return raw.combos;
if (raw && Array.isArray(raw.data)) return raw.data;
return [];
}
function getComboModels(combo, modelFallback) {
const steps = combo.steps ?? combo.models ?? combo.targets ?? [];
return steps.map((step) => ({
provider: step.provider ?? step.providerId ?? "",
model: step.model ?? step.modelId ?? modelFallback ?? "auto",
inputCostPer1M: step.inputCostPer1M ?? step.costPer1MInput ?? 0,
}));
}
function toArray(val) {
if (Array.isArray(val)) return val;
return [];
}

View File

@@ -70,6 +70,16 @@
"empty_prompt": "Error: prompt is required (positional arg, --file, or --stdin)"
}
},
"simulate": {
"description": "Dry-run routing simulation — show which providers would be selected without calling upstream",
"file": "Load full request body from JSON file",
"model": "Model ID (default: auto)",
"combo": "Force a specific combo by name",
"reasoning": "Reasoning effort level (low|medium|high)",
"thinking": "Extended thinking token budget",
"explain": "Print fallback tree and cost range to stderr",
"noCombo": "No matching combo found. Configure one with: omniroute combo create"
},
"chat": {
"description": "Send a one-shot chat prompt to OmniRoute",
"file": "Read prompt from file",

View File

@@ -70,6 +70,16 @@
"empty_prompt": "Erro: prompt obrigatório (argumento posicional, --file ou --stdin)"
}
},
"simulate": {
"description": "Simulação de routing em dry-run — mostra quais provedores seriam selecionados sem chamar o upstream",
"file": "Carregar body completo da requisição de arquivo JSON",
"model": "ID do modelo (padrão: auto)",
"combo": "Forçar um combo específico pelo nome",
"reasoning": "Nível de esforço de raciocínio (low|medium|high)",
"thinking": "Budget de tokens para raciocínio estendido",
"explain": "Imprimir árvore de fallback e faixa de custo no stderr",
"noCombo": "Nenhum combo correspondente encontrado. Configure um com: omniroute combo create"
},
"chat": {
"description": "Enviar um prompt único ao OmniRoute",
"file": "Ler prompt de arquivo",

View File

@@ -0,0 +1,179 @@
import test from "node:test";
import assert from "node:assert/strict";
import { tmpdir } from "node:os";
import { mkdtempSync, writeFileSync } from "node:fs";
import { join } from "node:path";
const COMBO_RESPONSE = {
combos: [
{
id: "default",
name: "default",
enabled: true,
steps: [
{ provider: "openai", model: "gpt-4o", inputCostPer1M: 5 },
{ provider: "anthropic", model: "claude-3-5-sonnet", inputCostPer1M: 3 },
],
},
],
};
const HEALTH_RESPONSE = {
circuitBreakers: [
{ provider: "openai", state: "CLOSED" },
{ provider: "anthropic", state: "HALF_OPEN" },
],
};
const QUOTA_RESPONSE = {
providers: [
{ provider: "openai", percentRemaining: 80 },
{ provider: "anthropic", percentRemaining: 60 },
],
};
function makeResp(data: unknown, status = 200) {
const json = () => Promise.resolve(data);
const text = () => Promise.resolve(JSON.stringify(data));
const obj = { ok: status < 400, status, json, text, headers: new Headers() };
obj.json = obj.json.bind(obj);
obj.text = obj.text.bind(obj);
return obj;
}
function mockFetch(overrides: Record<string, unknown> = {}) {
return (url: string) => {
const path = new URL(url, "http://localhost").pathname;
if (path.includes("/api/combos"))
return Promise.resolve(makeResp(overrides.combos ?? COMBO_RESPONSE));
if (path.includes("/api/monitoring/health")) return Promise.resolve(makeResp(HEALTH_RESPONSE));
if (path.includes("/api/usage/quota")) return Promise.resolve(makeResp(QUOTA_RESPONSE));
return Promise.resolve(makeResp({}, 404));
};
}
async function captureOutput(fn: () => Promise<void>): Promise<{ stdout: string; stderr: string }> {
const stdoutChunks: string[] = [];
const stderrChunks: string[] = [];
const origOut = process.stdout.write.bind(process.stdout);
const origErr = process.stderr.write.bind(process.stderr);
process.stdout.write = (c: string | Uint8Array) => {
stdoutChunks.push(typeof c === "string" ? c : c.toString());
return true;
};
process.stderr.write = (c: string | Uint8Array) => {
stderrChunks.push(typeof c === "string" ? c : c.toString());
return true;
};
try {
await fn();
} finally {
process.stdout.write = origOut;
process.stderr.write = origErr;
}
return { stdout: stdoutChunks.join(""), stderr: stderrChunks.join("") };
}
test("runSimulateCommand exibe tabela com provedores do combo", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
const { stdout } = await captureOutput(() =>
runSimulateCommand("explique RAG", { model: "auto" }, cmd as any)
);
globalThis.fetch = origFetch;
assert.ok(stdout.includes("openai") || stdout.includes("Provider"));
});
test("runSimulateCommand --output json retorna simulatedPath completo", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const { stdout } = await captureOutput(() =>
runSimulateCommand("test", { model: "auto" }, cmd as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(stdout);
assert.ok(Array.isArray(parsed));
assert.equal(parsed.length, 2);
assert.equal(parsed[0].provider, "openai");
assert.equal(parsed[0].order, 1);
assert.ok(typeof parsed[0].healthStatus === "string");
});
test("runSimulateCommand --explain imprime arvore de fallback no stderr", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
const { stderr } = await captureOutput(() =>
runSimulateCommand("test", { model: "auto", explain: true }, cmd as any)
);
globalThis.fetch = origFetch;
assert.ok(stderr.includes("Primary:"));
assert.ok(stderr.includes("anthropic"));
});
test("runSimulateCommand --file carrega JSON e usa como body", async () => {
const tmpDir = mkdtempSync(join(tmpdir(), "sim-test-"));
const filePath = join(tmpDir, "body.json");
writeFileSync(filePath, JSON.stringify({ messages: [{ role: "user", content: "hello" }] }));
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const { stdout } = await captureOutput(() =>
runSimulateCommand(undefined, { model: "auto", file: filePath }, cmd as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(stdout);
assert.ok(Array.isArray(parsed));
assert.ok(parsed.length >= 1);
});
test("runSimulateCommand --combo filtra por nome do combo", async () => {
const origFetch = globalThis.fetch;
globalThis.fetch = mockFetch() as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
const { stdout } = await captureOutput(() =>
runSimulateCommand("test", { model: "auto", combo: "default" }, cmd as any)
);
globalThis.fetch = origFetch;
const parsed = JSON.parse(stdout);
assert.ok(Array.isArray(parsed));
assert.equal(parsed[0].provider, "openai");
});
test("runSimulateCommand quando servidor offline emite mensagem e sai com 3", async () => {
const origFetch = globalThis.fetch;
const exitCodes: (number | string)[] = [];
const origExit = process.exit.bind(process);
process.exit = ((code: number) => {
exitCodes.push(code);
}) as any;
globalThis.fetch = (() => Promise.reject(new Error("ECONNREFUSED"))) as any;
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
const { stderr } = await captureOutput(() =>
runSimulateCommand("test", { model: "auto" }, cmd as any).catch(() => {})
);
globalThis.fetch = origFetch;
process.exit = origExit;
assert.ok(exitCodes.includes(3) || stderr.length > 0);
});