From 18e5bf26a657b0d654ff3f1f7eecfddc6a3be7dd Mon Sep 17 00:00:00 2001 From: diegosouzapw Date: Fri, 15 May 2026 00:45:35 -0300 Subject: [PATCH] feat(cli): adicionar comando simulate para dry-run de routing (Fase 2.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementa `omniroute simulate [prompt]` que consulta /api/combos, /api/monitoring/health e /api/usage/quota para mostrar qual caminho de routing seria escolhido sem executar chamada upstream. Suporta: - Tabela com provider, model, probabilidade, custo estimado, breaker e quota - --explain: imprime árvore de fallback e faixa de custo no stderr - --output json: retorna array de targets completo - --file : carrega body JSON para estimar tokens - --combo : filtra por combo específico - Tratamento gracioso quando servidor está offline (exit 3) --- bin/cli/commands/registry.mjs | 2 + bin/cli/commands/simulate.mjs | 125 ++++++++++++++++++++++ bin/cli/locales/en.json | 10 ++ bin/cli/locales/pt-BR.json | 10 ++ tests/unit/cli-simulate.test.ts | 179 ++++++++++++++++++++++++++++++++ 5 files changed, 326 insertions(+) create mode 100644 bin/cli/commands/simulate.mjs create mode 100644 tests/unit/cli-simulate.test.ts diff --git a/bin/cli/commands/registry.mjs b/bin/cli/commands/registry.mjs index 7b65dbf06e..4fd5a762fc 100644 --- a/bin/cli/commands/registry.mjs +++ b/bin/cli/commands/registry.mjs @@ -1,5 +1,6 @@ import { registerChat } from "./chat.mjs"; import { registerStream } from "./stream.mjs"; +import { registerSimulate } from "./simulate.mjs"; import { registerServe } from "./serve.mjs"; import { registerStop } from "./stop.mjs"; import { registerRestart } from "./restart.mjs"; @@ -29,6 +30,7 @@ import { registerCompletion } from "./completion.mjs"; export function registerCommands(program) { registerChat(program); registerStream(program); + registerSimulate(program); registerServe(program); registerStop(program); registerRestart(program); diff --git a/bin/cli/commands/simulate.mjs b/bin/cli/commands/simulate.mjs new file mode 100644 index 0000000000..cf00e79080 --- /dev/null +++ b/bin/cli/commands/simulate.mjs @@ -0,0 +1,125 @@ +import { readFileSync } from "node:fs"; +import { apiFetch } from "../api.mjs"; +import { emit } from "../output.mjs"; +import { t } from "../i18n.mjs"; + +const simulateSchema = [ + { key: "order", header: "#", width: 4 }, + { key: "provider", header: "Provider", width: 20 }, + { key: "model", header: "Model", width: 35 }, + { key: "probability", header: "Probability", formatter: (v) => `${Math.round(v * 100)}%` }, + { key: "estimatedCost", header: "Est. Cost", formatter: (v) => (v ? `$${v.toFixed(4)}` : "-") }, + { key: "healthStatus", header: "Breaker", width: 10 }, + { key: "quotaAvailable", header: "Quota %", formatter: (v) => `${v}%` }, +]; + +export function registerSimulate(program) { + program + .command("simulate [prompt]") + .description(t("simulate.description")) + .option("--file ", t("simulate.file")) + .option("-m, --model ", t("simulate.model"), "auto") + .option("--combo ", t("simulate.combo")) + .option("--reasoning-effort ", t("simulate.reasoning")) + .option("--thinking-budget ", t("simulate.thinking"), parseInt) + .option("--explain", t("simulate.explain")) + .action(runSimulateCommand); +} + +export async function runSimulateCommand(promptArg, opts, cmd) { + const globalOpts = cmd.optsWithGlobals(); + + let promptTokenEstimate = 100; + if (opts.file) { + try { + const raw = readFileSync(opts.file, "utf8"); + const parsed = JSON.parse(raw); + const text = JSON.stringify(parsed); + promptTokenEstimate = Math.ceil(text.length / 4); + } catch { + promptTokenEstimate = 100; + } + } else if (promptArg) { + promptTokenEstimate = Math.ceil(promptArg.length / 4); + } + + const [combosRes, healthRes, quotaRes] = await Promise.allSettled([ + apiFetch("/api/combos", { timeout: globalOpts.timeout }).then((r) => r.json()), + apiFetch("/api/monitoring/health", { timeout: globalOpts.timeout }).then((r) => r.json()), + apiFetch("/api/usage/quota", { timeout: globalOpts.timeout }).then((r) => r.json()), + ]); + + if (combosRes.status === "rejected") { + process.stderr.write(t("common.serverOffline") + "\n"); + process.exit(3); + } + + const combos = normalizeCombos(combosRes.value); + const health = healthRes.status === "fulfilled" ? healthRes.value : {}; + const quota = quotaRes.status === "fulfilled" ? quotaRes.value : {}; + + const targetCombo = opts.combo + ? combos.find((c) => c.id === opts.combo || c.name === opts.combo) + : combos.find((c) => c.enabled !== false); + + if (!targetCombo) { + process.stderr.write(t("simulate.noCombo") + "\n"); + process.exit(1); + } + + const models = getComboModels(targetCombo, opts.model); + const breakers = toArray(health.circuitBreakers ?? health.breakers); + const providers = toArray(quota.providers ?? quota.data); + + const simulatedPath = models.map((m, idx) => { + const cb = breakers.find((b) => String(b.provider) === m.provider); + const q = providers.find((p) => p.provider === m.provider); + const inputCost = m.inputCostPer1M ?? 0; + const estimatedCost = Math.round((promptTokenEstimate / 1_000_000) * inputCost * 10000) / 10000; + return { + order: idx + 1, + provider: m.provider, + model: m.model || opts.model, + probability: idx === 0 ? 0.85 : 0.15 / Math.max(models.length - 1, 1), + estimatedCost, + healthStatus: String(cb?.state ?? "CLOSED"), + quotaAvailable: q?.percentRemaining ?? 100, + }; + }); + + emit(simulatedPath, globalOpts, simulateSchema); + + if (opts.explain && !globalOpts.quiet) { + const primary = simulatedPath[0]; + const fallbacks = simulatedPath.slice(1).map((s) => s.provider); + process.stderr.write(`\nPrimary: ${primary?.provider} / ${primary?.model}\n`); + if (fallbacks.length > 0) { + process.stderr.write(`Fallbacks: ${fallbacks.join(" → ")}\n`); + } + const costs = simulatedPath.map((s) => s.estimatedCost); + process.stderr.write( + `Est. cost range: $${Math.min(...costs).toFixed(4)} – $${Math.max(...costs).toFixed(4)}\n` + ); + } +} + +function normalizeCombos(raw) { + if (Array.isArray(raw)) return raw; + if (raw && Array.isArray(raw.combos)) return raw.combos; + if (raw && Array.isArray(raw.data)) return raw.data; + return []; +} + +function getComboModels(combo, modelFallback) { + const steps = combo.steps ?? combo.models ?? combo.targets ?? []; + return steps.map((step) => ({ + provider: step.provider ?? step.providerId ?? "", + model: step.model ?? step.modelId ?? modelFallback ?? "auto", + inputCostPer1M: step.inputCostPer1M ?? step.costPer1MInput ?? 0, + })); +} + +function toArray(val) { + if (Array.isArray(val)) return val; + return []; +} diff --git a/bin/cli/locales/en.json b/bin/cli/locales/en.json index e113f688a6..fa1a728d2b 100644 --- a/bin/cli/locales/en.json +++ b/bin/cli/locales/en.json @@ -70,6 +70,16 @@ "empty_prompt": "Error: prompt is required (positional arg, --file, or --stdin)" } }, + "simulate": { + "description": "Dry-run routing simulation — show which providers would be selected without calling upstream", + "file": "Load full request body from JSON file", + "model": "Model ID (default: auto)", + "combo": "Force a specific combo by name", + "reasoning": "Reasoning effort level (low|medium|high)", + "thinking": "Extended thinking token budget", + "explain": "Print fallback tree and cost range to stderr", + "noCombo": "No matching combo found. Configure one with: omniroute combo create" + }, "chat": { "description": "Send a one-shot chat prompt to OmniRoute", "file": "Read prompt from file", diff --git a/bin/cli/locales/pt-BR.json b/bin/cli/locales/pt-BR.json index 4332974ee1..799a399f8b 100644 --- a/bin/cli/locales/pt-BR.json +++ b/bin/cli/locales/pt-BR.json @@ -70,6 +70,16 @@ "empty_prompt": "Erro: prompt obrigatório (argumento posicional, --file ou --stdin)" } }, + "simulate": { + "description": "Simulação de routing em dry-run — mostra quais provedores seriam selecionados sem chamar o upstream", + "file": "Carregar body completo da requisição de arquivo JSON", + "model": "ID do modelo (padrão: auto)", + "combo": "Forçar um combo específico pelo nome", + "reasoning": "Nível de esforço de raciocínio (low|medium|high)", + "thinking": "Budget de tokens para raciocínio estendido", + "explain": "Imprimir árvore de fallback e faixa de custo no stderr", + "noCombo": "Nenhum combo correspondente encontrado. Configure um com: omniroute combo create" + }, "chat": { "description": "Enviar um prompt único ao OmniRoute", "file": "Ler prompt de arquivo", diff --git a/tests/unit/cli-simulate.test.ts b/tests/unit/cli-simulate.test.ts new file mode 100644 index 0000000000..2aa6d61c7b --- /dev/null +++ b/tests/unit/cli-simulate.test.ts @@ -0,0 +1,179 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const COMBO_RESPONSE = { + combos: [ + { + id: "default", + name: "default", + enabled: true, + steps: [ + { provider: "openai", model: "gpt-4o", inputCostPer1M: 5 }, + { provider: "anthropic", model: "claude-3-5-sonnet", inputCostPer1M: 3 }, + ], + }, + ], +}; + +const HEALTH_RESPONSE = { + circuitBreakers: [ + { provider: "openai", state: "CLOSED" }, + { provider: "anthropic", state: "HALF_OPEN" }, + ], +}; + +const QUOTA_RESPONSE = { + providers: [ + { provider: "openai", percentRemaining: 80 }, + { provider: "anthropic", percentRemaining: 60 }, + ], +}; + +function makeResp(data: unknown, status = 200) { + const json = () => Promise.resolve(data); + const text = () => Promise.resolve(JSON.stringify(data)); + const obj = { ok: status < 400, status, json, text, headers: new Headers() }; + obj.json = obj.json.bind(obj); + obj.text = obj.text.bind(obj); + return obj; +} + +function mockFetch(overrides: Record = {}) { + return (url: string) => { + const path = new URL(url, "http://localhost").pathname; + if (path.includes("/api/combos")) + return Promise.resolve(makeResp(overrides.combos ?? COMBO_RESPONSE)); + if (path.includes("/api/monitoring/health")) return Promise.resolve(makeResp(HEALTH_RESPONSE)); + if (path.includes("/api/usage/quota")) return Promise.resolve(makeResp(QUOTA_RESPONSE)); + return Promise.resolve(makeResp({}, 404)); + }; +} + +async function captureOutput(fn: () => Promise): Promise<{ stdout: string; stderr: string }> { + const stdoutChunks: string[] = []; + const stderrChunks: string[] = []; + const origOut = process.stdout.write.bind(process.stdout); + const origErr = process.stderr.write.bind(process.stderr); + process.stdout.write = (c: string | Uint8Array) => { + stdoutChunks.push(typeof c === "string" ? c : c.toString()); + return true; + }; + process.stderr.write = (c: string | Uint8Array) => { + stderrChunks.push(typeof c === "string" ? c : c.toString()); + return true; + }; + try { + await fn(); + } finally { + process.stdout.write = origOut; + process.stderr.write = origErr; + } + return { stdout: stdoutChunks.join(""), stderr: stderrChunks.join("") }; +} + +test("runSimulateCommand exibe tabela com provedores do combo", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = mockFetch() as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) }; + const { stdout } = await captureOutput(() => + runSimulateCommand("explique RAG", { model: "auto" }, cmd as any) + ); + + globalThis.fetch = origFetch; + assert.ok(stdout.includes("openai") || stdout.includes("Provider")); +}); + +test("runSimulateCommand --output json retorna simulatedPath completo", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = mockFetch() as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) }; + const { stdout } = await captureOutput(() => + runSimulateCommand("test", { model: "auto" }, cmd as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(stdout); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed.length, 2); + assert.equal(parsed[0].provider, "openai"); + assert.equal(parsed[0].order, 1); + assert.ok(typeof parsed[0].healthStatus === "string"); +}); + +test("runSimulateCommand --explain imprime arvore de fallback no stderr", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = mockFetch() as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) }; + const { stderr } = await captureOutput(() => + runSimulateCommand("test", { model: "auto", explain: true }, cmd as any) + ); + + globalThis.fetch = origFetch; + assert.ok(stderr.includes("Primary:")); + assert.ok(stderr.includes("anthropic")); +}); + +test("runSimulateCommand --file carrega JSON e usa como body", async () => { + const tmpDir = mkdtempSync(join(tmpdir(), "sim-test-")); + const filePath = join(tmpDir, "body.json"); + writeFileSync(filePath, JSON.stringify({ messages: [{ role: "user", content: "hello" }] })); + + const origFetch = globalThis.fetch; + globalThis.fetch = mockFetch() as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) }; + const { stdout } = await captureOutput(() => + runSimulateCommand(undefined, { model: "auto", file: filePath }, cmd as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(stdout); + assert.ok(Array.isArray(parsed)); + assert.ok(parsed.length >= 1); +}); + +test("runSimulateCommand --combo filtra por nome do combo", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = mockFetch() as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) }; + const { stdout } = await captureOutput(() => + runSimulateCommand("test", { model: "auto", combo: "default" }, cmd as any) + ); + + globalThis.fetch = origFetch; + const parsed = JSON.parse(stdout); + assert.ok(Array.isArray(parsed)); + assert.equal(parsed[0].provider, "openai"); +}); + +test("runSimulateCommand quando servidor offline emite mensagem e sai com 3", async () => { + const origFetch = globalThis.fetch; + const exitCodes: (number | string)[] = []; + const origExit = process.exit.bind(process); + process.exit = ((code: number) => { + exitCodes.push(code); + }) as any; + globalThis.fetch = (() => Promise.reject(new Error("ECONNREFUSED"))) as any; + + const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs"); + const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) }; + const { stderr } = await captureOutput(() => + runSimulateCommand("test", { model: "auto" }, cmd as any).catch(() => {}) + ); + + globalThis.fetch = origFetch; + process.exit = origExit; + assert.ok(exitCodes.includes(3) || stderr.length > 0); +});