mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
f8e56e4615
* feat(eval): add router-eval harness (AIQ scoring, regression gate, Pareto search)
Extracts a standalone router-eval evaluation tool that replays routing
decisions (from NDJSON corpora or the usage_history/call_logs SQLite
tables) into an AIQ (success/latency/cost) score, compares baseline vs.
candidate router configs with a retained-run regression gate, and ranks
Pareto-optimal candidates across a search space — a sibling to the
existing eval:compression harness.
New scripts: scripts/router-eval/{index,compare,patch-compare,search,
trends}.ts, scripts/check/check-router-eval-regression.ts, and
src/lib/routerEval/index.ts, wired via 6 new package.json entries
(eval:router, eval:router:compare, eval:router:patch-compare,
eval:router:search, eval:router:trends, check:router-eval).
Reconstructed onto current release/v3.8.49 from the original ~142-commit
stale PR branch: only the genuinely new router-eval payload (17 files)
was extracted — the other ~560 changed files in the original diff were
base-drift already present on release in newer form. The new package.json
scripts now invoke `node --import tsx` instead of `bun`, matching the
`eval:compression` precedent (Bun is reserved for a closed 5-script
allowlist). The runtime-detection shim (`"Bun" in globalThis`) already
present in the harness gracefully falls back to better-sqlite3 under
Node, so no logic changes were needed there; the retained-run manifest's
previously-hardcoded `runtime: "bun"` field and matching CLI help text
were corrected to reflect the actual invocation.
All 27 existing router-eval unit tests pass unchanged.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
* refactor(router-eval): decompose toRouterObservation below complexity gate
toRouterObservation had cyclomatic complexity 18 (gate max is 15). Extract
the per-field parsing/normalization into pure helpers (sampleId, model
fields, latency, cost derivation, success) so the entry point is a plain
sequential assembly of a RouterObservation. Behavior is unchanged — same
tests pass, same fallbacks, same precedence between input aliases.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
---------
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
93 lines
3.1 KiB
TypeScript
93 lines
3.1 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import {
|
|
compareRouterEvalRuns,
|
|
createRouterEvalArtifact,
|
|
runRouterEval,
|
|
toRouterObservation,
|
|
} from "@/lib/routerEval/index.ts";
|
|
|
|
function obs(overrides: Record<string, unknown>) {
|
|
return toRouterObservation({
|
|
sampleId: "sample-1",
|
|
configId: "config",
|
|
selectedModel: "gpt-4.1",
|
|
expectedModel: "gpt-4.1",
|
|
latencyMs: 120,
|
|
costUsd: 0.05,
|
|
success: true,
|
|
timestamp: "2026-01-01T00:00:00.000Z",
|
|
...overrides,
|
|
})!;
|
|
}
|
|
|
|
test("toRouterObservation normalizes token cost and booleans", () => {
|
|
const observation = toRouterObservation({
|
|
sampleId: "s-1",
|
|
configId: "combo-a",
|
|
selectedModel: "gpt-4.1",
|
|
requestedModel: "gpt-4.1",
|
|
durationMs: 240,
|
|
tokens: { input: 100, output: 50 },
|
|
status: 200,
|
|
});
|
|
|
|
assert.ok(observation);
|
|
assert.equal(observation!.sampleId, "s-1");
|
|
assert.equal(observation!.configId, "combo-a");
|
|
assert.equal(observation!.latencyMs, 240);
|
|
assert.equal(observation!.costUsd, 0.00015);
|
|
assert.equal(observation!.success, true);
|
|
assert.equal(observation!.expectedModel, "gpt-4.1");
|
|
});
|
|
|
|
test("runRouterEval computes summary and frontiers", () => {
|
|
const report = runRouterEval([
|
|
obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
|
obs({ sampleId: "s2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
|
obs({ sampleId: "s3", configId: "b", latencyMs: 80, costUsd: 0.02, success: true }),
|
|
obs({ sampleId: "s4", configId: "b", latencyMs: 120, costUsd: 0.02, success: false }),
|
|
]);
|
|
|
|
assert.equal(report.summary.totalSamples, 4);
|
|
assert.equal(report.summary.validSamples, 4);
|
|
assert.equal(report.summary.uniqueConfigs, 2);
|
|
assert.equal(report.configurations.length, 2);
|
|
assert.equal(report.top[0]?.configId, "a");
|
|
assert.ok(report.frontier.length >= 1);
|
|
});
|
|
|
|
test("compareRouterEvalRuns captures AIQ and cost regressions", () => {
|
|
const baseline = runRouterEval([
|
|
obs({ sampleId: "b1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
|
obs({ sampleId: "b2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
|
]);
|
|
const candidate = runRouterEval([
|
|
obs({ sampleId: "c1", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }),
|
|
obs({ sampleId: "c2", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }),
|
|
]);
|
|
|
|
const comparison = compareRouterEvalRuns(baseline, candidate, {
|
|
aiqDrop: 5,
|
|
relativeCostIncrease: 1.2,
|
|
});
|
|
|
|
assert.equal(comparison.regressions.length > 0, true);
|
|
assert.equal(comparison.delta.aiq <= 0, true);
|
|
assert.equal(comparison.delta.costUsd > 0, true);
|
|
});
|
|
|
|
test("createRouterEvalArtifact wraps reports with a stable schema", () => {
|
|
const report = runRouterEval([
|
|
obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
|
]);
|
|
|
|
const artifact = createRouterEvalArtifact(report);
|
|
|
|
assert.equal(artifact.schemaVersion, 1);
|
|
assert.equal(artifact.kind, "router-eval-report");
|
|
assert.equal(artifact.generatedAt, report.evaluatedAt);
|
|
assert.equal(artifact.report?.summary.totalSamples, 1);
|
|
});
|