mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
f8e56e4615
* feat(eval): add router-eval harness (AIQ scoring, regression gate, Pareto search)
Extracts a standalone router-eval evaluation tool that replays routing
decisions (from NDJSON corpora or the usage_history/call_logs SQLite
tables) into an AIQ (success/latency/cost) score, compares baseline vs.
candidate router configs with a retained-run regression gate, and ranks
Pareto-optimal candidates across a search space — a sibling to the
existing eval:compression harness.
New scripts: scripts/router-eval/{index,compare,patch-compare,search,
trends}.ts, scripts/check/check-router-eval-regression.ts, and
src/lib/routerEval/index.ts, wired via 6 new package.json entries
(eval:router, eval:router:compare, eval:router:patch-compare,
eval:router:search, eval:router:trends, check:router-eval).
Reconstructed onto current release/v3.8.49 from the original ~142-commit
stale PR branch: only the genuinely new router-eval payload (17 files)
was extracted — the other ~560 changed files in the original diff were
base-drift already present on release in newer form. The new package.json
scripts now invoke `node --import tsx` instead of `bun`, matching the
`eval:compression` precedent (Bun is reserved for a closed 5-script
allowlist). The runtime-detection shim (`"Bun" in globalThis`) already
present in the harness gracefully falls back to better-sqlite3 under
Node, so no logic changes were needed there; the retained-run manifest's
previously-hardcoded `runtime: "bun"` field and matching CLI help text
were corrected to reflect the actual invocation.
All 27 existing router-eval unit tests pass unchanged.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
* refactor(router-eval): decompose toRouterObservation below complexity gate
toRouterObservation had cyclomatic complexity 18 (gate max is 15). Extract
the per-field parsing/normalization into pure helpers (sampleId, model
fields, latency, cost derivation, success) so the entry point is a plain
sequential assembly of a RouterObservation. Behavior is unchanged — same
tests pass, same fallbacks, same precedence between input aliases.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
---------
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
84 lines
2.2 KiB
TypeScript
84 lines
2.2 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { spawnSync } from "node:child_process";
|
|
|
|
const compareScript = "scripts/router-eval/compare.ts";
|
|
|
|
test("router eval compare retains named comparison artifacts", () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "router-eval-compare-"));
|
|
const baseline = join(dir, "baseline.ndjson");
|
|
const candidate = join(dir, "candidate.ndjson");
|
|
const artifactDir = join(dir, "artifacts");
|
|
const runId = "policy-a-vs-policy-b";
|
|
|
|
writeFileSync(
|
|
baseline,
|
|
JSON.stringify({
|
|
sampleId: "b1",
|
|
configId: "policy-a",
|
|
selectedModel: "gpt-4.1",
|
|
expectedModel: "gpt-4.1",
|
|
latencyMs: 140,
|
|
costUsd: 0.004,
|
|
success: true,
|
|
})
|
|
);
|
|
writeFileSync(
|
|
candidate,
|
|
JSON.stringify({
|
|
sampleId: "c1",
|
|
configId: "policy-b",
|
|
selectedModel: "gpt-4.1",
|
|
expectedModel: "gpt-4.1",
|
|
latencyMs: 130,
|
|
costUsd: 0.004,
|
|
success: true,
|
|
})
|
|
);
|
|
|
|
const result = spawnSync(
|
|
process.execPath,
|
|
[
|
|
"--import",
|
|
"tsx",
|
|
compareScript,
|
|
"--baseline",
|
|
baseline,
|
|
"--candidate",
|
|
candidate,
|
|
"--baseline-name",
|
|
"policy-a",
|
|
"--candidate-name",
|
|
"policy-b",
|
|
"--artifact-dir",
|
|
artifactDir,
|
|
"--run-id",
|
|
runId,
|
|
],
|
|
{ encoding: "utf8" }
|
|
);
|
|
|
|
try {
|
|
assert.equal(result.status, 0);
|
|
assert.ok((result.stdout ?? "").includes("Router Eval Comparison"));
|
|
const runDir = join(artifactDir, runId);
|
|
assert.ok(
|
|
readFileSync(join(runDir, "router-eval.md"), "utf8").includes("Router Eval Comparison")
|
|
);
|
|
assert.ok(
|
|
readFileSync(join(runDir, "router-eval.json"), "utf8").includes("router-eval-comparison")
|
|
);
|
|
const comparison = JSON.parse(readFileSync(join(runDir, "comparison.json"), "utf8")) as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
assert.equal(comparison.baselineName, "policy-a");
|
|
assert.equal(comparison.candidateName, "policy-b");
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|