mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-01 12:52:11 +03:00
feat(cli): adicionar comando simulate para dry-run de routing (Fase 2.3)
Implementa `omniroute simulate [prompt]` que consulta /api/combos, /api/monitoring/health e /api/usage/quota para mostrar qual caminho de routing seria escolhido sem executar chamada upstream. Suporta: - Tabela com provider, model, probabilidade, custo estimado, breaker e quota - --explain: imprime árvore de fallback e faixa de custo no stderr - --output json: retorna array de targets completo - --file <path>: carrega body JSON para estimar tokens - --combo <name>: filtra por combo específico - Tratamento gracioso quando servidor está offline (exit 3)
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { registerChat } from "./chat.mjs";
|
||||
import { registerStream } from "./stream.mjs";
|
||||
import { registerSimulate } from "./simulate.mjs";
|
||||
import { registerServe } from "./serve.mjs";
|
||||
import { registerStop } from "./stop.mjs";
|
||||
import { registerRestart } from "./restart.mjs";
|
||||
@@ -29,6 +30,7 @@ import { registerCompletion } from "./completion.mjs";
|
||||
export function registerCommands(program) {
|
||||
registerChat(program);
|
||||
registerStream(program);
|
||||
registerSimulate(program);
|
||||
registerServe(program);
|
||||
registerStop(program);
|
||||
registerRestart(program);
|
||||
|
||||
125
bin/cli/commands/simulate.mjs
Normal file
125
bin/cli/commands/simulate.mjs
Normal file
@@ -0,0 +1,125 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { apiFetch } from "../api.mjs";
|
||||
import { emit } from "../output.mjs";
|
||||
import { t } from "../i18n.mjs";
|
||||
|
||||
const simulateSchema = [
|
||||
{ key: "order", header: "#", width: 4 },
|
||||
{ key: "provider", header: "Provider", width: 20 },
|
||||
{ key: "model", header: "Model", width: 35 },
|
||||
{ key: "probability", header: "Probability", formatter: (v) => `${Math.round(v * 100)}%` },
|
||||
{ key: "estimatedCost", header: "Est. Cost", formatter: (v) => (v ? `$${v.toFixed(4)}` : "-") },
|
||||
{ key: "healthStatus", header: "Breaker", width: 10 },
|
||||
{ key: "quotaAvailable", header: "Quota %", formatter: (v) => `${v}%` },
|
||||
];
|
||||
|
||||
export function registerSimulate(program) {
|
||||
program
|
||||
.command("simulate [prompt]")
|
||||
.description(t("simulate.description"))
|
||||
.option("--file <path>", t("simulate.file"))
|
||||
.option("-m, --model <id>", t("simulate.model"), "auto")
|
||||
.option("--combo <name>", t("simulate.combo"))
|
||||
.option("--reasoning-effort <level>", t("simulate.reasoning"))
|
||||
.option("--thinking-budget <n>", t("simulate.thinking"), parseInt)
|
||||
.option("--explain", t("simulate.explain"))
|
||||
.action(runSimulateCommand);
|
||||
}
|
||||
|
||||
export async function runSimulateCommand(promptArg, opts, cmd) {
|
||||
const globalOpts = cmd.optsWithGlobals();
|
||||
|
||||
let promptTokenEstimate = 100;
|
||||
if (opts.file) {
|
||||
try {
|
||||
const raw = readFileSync(opts.file, "utf8");
|
||||
const parsed = JSON.parse(raw);
|
||||
const text = JSON.stringify(parsed);
|
||||
promptTokenEstimate = Math.ceil(text.length / 4);
|
||||
} catch {
|
||||
promptTokenEstimate = 100;
|
||||
}
|
||||
} else if (promptArg) {
|
||||
promptTokenEstimate = Math.ceil(promptArg.length / 4);
|
||||
}
|
||||
|
||||
const [combosRes, healthRes, quotaRes] = await Promise.allSettled([
|
||||
apiFetch("/api/combos", { timeout: globalOpts.timeout }).then((r) => r.json()),
|
||||
apiFetch("/api/monitoring/health", { timeout: globalOpts.timeout }).then((r) => r.json()),
|
||||
apiFetch("/api/usage/quota", { timeout: globalOpts.timeout }).then((r) => r.json()),
|
||||
]);
|
||||
|
||||
if (combosRes.status === "rejected") {
|
||||
process.stderr.write(t("common.serverOffline") + "\n");
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
const combos = normalizeCombos(combosRes.value);
|
||||
const health = healthRes.status === "fulfilled" ? healthRes.value : {};
|
||||
const quota = quotaRes.status === "fulfilled" ? quotaRes.value : {};
|
||||
|
||||
const targetCombo = opts.combo
|
||||
? combos.find((c) => c.id === opts.combo || c.name === opts.combo)
|
||||
: combos.find((c) => c.enabled !== false);
|
||||
|
||||
if (!targetCombo) {
|
||||
process.stderr.write(t("simulate.noCombo") + "\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const models = getComboModels(targetCombo, opts.model);
|
||||
const breakers = toArray(health.circuitBreakers ?? health.breakers);
|
||||
const providers = toArray(quota.providers ?? quota.data);
|
||||
|
||||
const simulatedPath = models.map((m, idx) => {
|
||||
const cb = breakers.find((b) => String(b.provider) === m.provider);
|
||||
const q = providers.find((p) => p.provider === m.provider);
|
||||
const inputCost = m.inputCostPer1M ?? 0;
|
||||
const estimatedCost = Math.round((promptTokenEstimate / 1_000_000) * inputCost * 10000) / 10000;
|
||||
return {
|
||||
order: idx + 1,
|
||||
provider: m.provider,
|
||||
model: m.model || opts.model,
|
||||
probability: idx === 0 ? 0.85 : 0.15 / Math.max(models.length - 1, 1),
|
||||
estimatedCost,
|
||||
healthStatus: String(cb?.state ?? "CLOSED"),
|
||||
quotaAvailable: q?.percentRemaining ?? 100,
|
||||
};
|
||||
});
|
||||
|
||||
emit(simulatedPath, globalOpts, simulateSchema);
|
||||
|
||||
if (opts.explain && !globalOpts.quiet) {
|
||||
const primary = simulatedPath[0];
|
||||
const fallbacks = simulatedPath.slice(1).map((s) => s.provider);
|
||||
process.stderr.write(`\nPrimary: ${primary?.provider} / ${primary?.model}\n`);
|
||||
if (fallbacks.length > 0) {
|
||||
process.stderr.write(`Fallbacks: ${fallbacks.join(" → ")}\n`);
|
||||
}
|
||||
const costs = simulatedPath.map((s) => s.estimatedCost);
|
||||
process.stderr.write(
|
||||
`Est. cost range: $${Math.min(...costs).toFixed(4)} – $${Math.max(...costs).toFixed(4)}\n`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCombos(raw) {
|
||||
if (Array.isArray(raw)) return raw;
|
||||
if (raw && Array.isArray(raw.combos)) return raw.combos;
|
||||
if (raw && Array.isArray(raw.data)) return raw.data;
|
||||
return [];
|
||||
}
|
||||
|
||||
function getComboModels(combo, modelFallback) {
|
||||
const steps = combo.steps ?? combo.models ?? combo.targets ?? [];
|
||||
return steps.map((step) => ({
|
||||
provider: step.provider ?? step.providerId ?? "",
|
||||
model: step.model ?? step.modelId ?? modelFallback ?? "auto",
|
||||
inputCostPer1M: step.inputCostPer1M ?? step.costPer1MInput ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
function toArray(val) {
|
||||
if (Array.isArray(val)) return val;
|
||||
return [];
|
||||
}
|
||||
@@ -70,6 +70,16 @@
|
||||
"empty_prompt": "Error: prompt is required (positional arg, --file, or --stdin)"
|
||||
}
|
||||
},
|
||||
"simulate": {
|
||||
"description": "Dry-run routing simulation — show which providers would be selected without calling upstream",
|
||||
"file": "Load full request body from JSON file",
|
||||
"model": "Model ID (default: auto)",
|
||||
"combo": "Force a specific combo by name",
|
||||
"reasoning": "Reasoning effort level (low|medium|high)",
|
||||
"thinking": "Extended thinking token budget",
|
||||
"explain": "Print fallback tree and cost range to stderr",
|
||||
"noCombo": "No matching combo found. Configure one with: omniroute combo create"
|
||||
},
|
||||
"chat": {
|
||||
"description": "Send a one-shot chat prompt to OmniRoute",
|
||||
"file": "Read prompt from file",
|
||||
|
||||
@@ -70,6 +70,16 @@
|
||||
"empty_prompt": "Erro: prompt obrigatório (argumento posicional, --file ou --stdin)"
|
||||
}
|
||||
},
|
||||
"simulate": {
|
||||
"description": "Simulação de routing em dry-run — mostra quais provedores seriam selecionados sem chamar o upstream",
|
||||
"file": "Carregar body completo da requisição de arquivo JSON",
|
||||
"model": "ID do modelo (padrão: auto)",
|
||||
"combo": "Forçar um combo específico pelo nome",
|
||||
"reasoning": "Nível de esforço de raciocínio (low|medium|high)",
|
||||
"thinking": "Budget de tokens para raciocínio estendido",
|
||||
"explain": "Imprimir árvore de fallback e faixa de custo no stderr",
|
||||
"noCombo": "Nenhum combo correspondente encontrado. Configure um com: omniroute combo create"
|
||||
},
|
||||
"chat": {
|
||||
"description": "Enviar um prompt único ao OmniRoute",
|
||||
"file": "Ler prompt de arquivo",
|
||||
|
||||
179
tests/unit/cli-simulate.test.ts
Normal file
179
tests/unit/cli-simulate.test.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { tmpdir } from "node:os";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
const COMBO_RESPONSE = {
|
||||
combos: [
|
||||
{
|
||||
id: "default",
|
||||
name: "default",
|
||||
enabled: true,
|
||||
steps: [
|
||||
{ provider: "openai", model: "gpt-4o", inputCostPer1M: 5 },
|
||||
{ provider: "anthropic", model: "claude-3-5-sonnet", inputCostPer1M: 3 },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const HEALTH_RESPONSE = {
|
||||
circuitBreakers: [
|
||||
{ provider: "openai", state: "CLOSED" },
|
||||
{ provider: "anthropic", state: "HALF_OPEN" },
|
||||
],
|
||||
};
|
||||
|
||||
const QUOTA_RESPONSE = {
|
||||
providers: [
|
||||
{ provider: "openai", percentRemaining: 80 },
|
||||
{ provider: "anthropic", percentRemaining: 60 },
|
||||
],
|
||||
};
|
||||
|
||||
function makeResp(data: unknown, status = 200) {
|
||||
const json = () => Promise.resolve(data);
|
||||
const text = () => Promise.resolve(JSON.stringify(data));
|
||||
const obj = { ok: status < 400, status, json, text, headers: new Headers() };
|
||||
obj.json = obj.json.bind(obj);
|
||||
obj.text = obj.text.bind(obj);
|
||||
return obj;
|
||||
}
|
||||
|
||||
function mockFetch(overrides: Record<string, unknown> = {}) {
|
||||
return (url: string) => {
|
||||
const path = new URL(url, "http://localhost").pathname;
|
||||
if (path.includes("/api/combos"))
|
||||
return Promise.resolve(makeResp(overrides.combos ?? COMBO_RESPONSE));
|
||||
if (path.includes("/api/monitoring/health")) return Promise.resolve(makeResp(HEALTH_RESPONSE));
|
||||
if (path.includes("/api/usage/quota")) return Promise.resolve(makeResp(QUOTA_RESPONSE));
|
||||
return Promise.resolve(makeResp({}, 404));
|
||||
};
|
||||
}
|
||||
|
||||
async function captureOutput(fn: () => Promise<void>): Promise<{ stdout: string; stderr: string }> {
|
||||
const stdoutChunks: string[] = [];
|
||||
const stderrChunks: string[] = [];
|
||||
const origOut = process.stdout.write.bind(process.stdout);
|
||||
const origErr = process.stderr.write.bind(process.stderr);
|
||||
process.stdout.write = (c: string | Uint8Array) => {
|
||||
stdoutChunks.push(typeof c === "string" ? c : c.toString());
|
||||
return true;
|
||||
};
|
||||
process.stderr.write = (c: string | Uint8Array) => {
|
||||
stderrChunks.push(typeof c === "string" ? c : c.toString());
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
process.stdout.write = origOut;
|
||||
process.stderr.write = origErr;
|
||||
}
|
||||
return { stdout: stdoutChunks.join(""), stderr: stderrChunks.join("") };
|
||||
}
|
||||
|
||||
test("runSimulateCommand exibe tabela com provedores do combo", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = mockFetch() as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
|
||||
const { stdout } = await captureOutput(() =>
|
||||
runSimulateCommand("explique RAG", { model: "auto" }, cmd as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
assert.ok(stdout.includes("openai") || stdout.includes("Provider"));
|
||||
});
|
||||
|
||||
test("runSimulateCommand --output json retorna simulatedPath completo", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = mockFetch() as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
|
||||
const { stdout } = await captureOutput(() =>
|
||||
runSimulateCommand("test", { model: "auto" }, cmd as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(stdout);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.equal(parsed.length, 2);
|
||||
assert.equal(parsed[0].provider, "openai");
|
||||
assert.equal(parsed[0].order, 1);
|
||||
assert.ok(typeof parsed[0].healthStatus === "string");
|
||||
});
|
||||
|
||||
test("runSimulateCommand --explain imprime arvore de fallback no stderr", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = mockFetch() as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
|
||||
const { stderr } = await captureOutput(() =>
|
||||
runSimulateCommand("test", { model: "auto", explain: true }, cmd as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
assert.ok(stderr.includes("Primary:"));
|
||||
assert.ok(stderr.includes("anthropic"));
|
||||
});
|
||||
|
||||
test("runSimulateCommand --file carrega JSON e usa como body", async () => {
|
||||
const tmpDir = mkdtempSync(join(tmpdir(), "sim-test-"));
|
||||
const filePath = join(tmpDir, "body.json");
|
||||
writeFileSync(filePath, JSON.stringify({ messages: [{ role: "user", content: "hello" }] }));
|
||||
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = mockFetch() as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
|
||||
const { stdout } = await captureOutput(() =>
|
||||
runSimulateCommand(undefined, { model: "auto", file: filePath }, cmd as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(stdout);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.ok(parsed.length >= 1);
|
||||
});
|
||||
|
||||
test("runSimulateCommand --combo filtra por nome do combo", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
globalThis.fetch = mockFetch() as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "json", quiet: true }) };
|
||||
const { stdout } = await captureOutput(() =>
|
||||
runSimulateCommand("test", { model: "auto", combo: "default" }, cmd as any)
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
const parsed = JSON.parse(stdout);
|
||||
assert.ok(Array.isArray(parsed));
|
||||
assert.equal(parsed[0].provider, "openai");
|
||||
});
|
||||
|
||||
test("runSimulateCommand quando servidor offline emite mensagem e sai com 3", async () => {
|
||||
const origFetch = globalThis.fetch;
|
||||
const exitCodes: (number | string)[] = [];
|
||||
const origExit = process.exit.bind(process);
|
||||
process.exit = ((code: number) => {
|
||||
exitCodes.push(code);
|
||||
}) as any;
|
||||
globalThis.fetch = (() => Promise.reject(new Error("ECONNREFUSED"))) as any;
|
||||
|
||||
const { runSimulateCommand } = await import("../../bin/cli/commands/simulate.mjs");
|
||||
const cmd = { optsWithGlobals: () => ({ output: "table", quiet: false }) };
|
||||
const { stderr } = await captureOutput(() =>
|
||||
runSimulateCommand("test", { model: "auto" }, cmd as any).catch(() => {})
|
||||
);
|
||||
|
||||
globalThis.fetch = origFetch;
|
||||
process.exit = origExit;
|
||||
assert.ok(exitCodes.includes(3) || stderr.length > 0);
|
||||
});
|
||||
Reference in New Issue
Block a user