mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-07-26 09:52:11 +03:00
fix(router-eval): retained-optimization gate cleanup (#7318)
* feat(eval): add router-eval harness (AIQ scoring, regression gate, Pareto search)
Extracts a standalone router-eval evaluation tool that replays routing
decisions (from NDJSON corpora or the usage_history/call_logs SQLite
tables) into an AIQ (success/latency/cost) score, compares baseline vs.
candidate router configs with a retained-run regression gate, and ranks
Pareto-optimal candidates across a search space — a sibling to the
existing eval:compression harness.
New scripts: scripts/router-eval/{index,compare,patch-compare,search,
trends}.ts, scripts/check/check-router-eval-regression.ts, and
src/lib/routerEval/index.ts, wired via 6 new package.json entries
(eval:router, eval:router:compare, eval:router:patch-compare,
eval:router:search, eval:router:trends, check:router-eval).
Reconstructed onto current release/v3.8.49 from the original ~142-commit
stale PR branch: only the genuinely new router-eval payload (17 files)
was extracted — the other ~560 changed files in the original diff were
base-drift already present on release in newer form. The new package.json
scripts now invoke `node --import tsx` instead of `bun`, matching the
`eval:compression` precedent (Bun is reserved for a closed 5-script
allowlist). The runtime-detection shim (`"Bun" in globalThis`) already
present in the harness gracefully falls back to better-sqlite3 under
Node, so no logic changes were needed there; the retained-run manifest's
previously-hardcoded `runtime: "bun"` field and matching CLI help text
were corrected to reflect the actual invocation.
All 27 existing router-eval unit tests pass unchanged.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
* refactor(router-eval): decompose toRouterObservation below complexity gate
toRouterObservation had cyclomatic complexity 18 (gate max is 15). Extract
the per-field parsing/normalization into pure helpers (sampleId, model
fields, latency, cost derivation, success) so the entry point is a plain
sequential assembly of a RouterObservation. Behavior is unchanged — same
tests pass, same fallbacks, same precedence between input aliases.
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
---------
Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com>
This commit is contained in:
KooshaPari
1
changelog.d/features/7318-router-eval-harness.md
Normal file
1
changelog.d/features/7318-router-eval-harness.md
Normal file
@@ -0,0 +1 @@
|
||||
- **feat(eval):** added a router-eval harness (`npm run eval:router`, `eval:router:compare`, `eval:router:patch-compare`, `eval:router:search`, `eval:router:trends`, `check:router-eval`) that replays routing decisions — from NDJSON corpora or the `usage_history`/`call_logs` SQLite tables — into an AIQ (success/latency/cost) score, compares baseline vs. candidate router configs with a retained-run regression gate, and ranks Pareto-optimal candidates; a sibling tool to the existing `eval:compression` harness (#7318 — thanks @KooshaPari).
|
||||
@@ -79,6 +79,11 @@
|
||||
"gen:provider-reference": "bun scripts/docs/gen-provider-reference.ts",
|
||||
"bench:compression": "bun scripts/compression/benchmark.ts",
|
||||
"eval:compression": "node --import tsx scripts/compression-eval/index.ts",
|
||||
"eval:router": "node --import tsx scripts/router-eval/index.ts",
|
||||
"eval:router:compare": "node --import tsx scripts/router-eval/compare.ts",
|
||||
"eval:router:patch-compare": "node --import tsx scripts/router-eval/patch-compare.ts",
|
||||
"eval:router:search": "node --import tsx scripts/router-eval/search.ts",
|
||||
"eval:router:trends": "node --import tsx scripts/router-eval/trends.ts",
|
||||
"release:sync-changelog-i18n": "node scripts/release/sync-changelog-i18n.mjs",
|
||||
"build": "node scripts/build/build-next-isolated.mjs",
|
||||
"build:secure": "OMNIROUTE_BUILD_PROFILE=minimal node scripts/build/build-next-isolated.mjs",
|
||||
@@ -118,6 +123,7 @@
|
||||
"check:docs-counts": "node scripts/check/check-docs-counts-sync.mjs",
|
||||
"check:deprecated-versions": "node scripts/check/check-deprecated-versions.mjs",
|
||||
"check:compression-budget": "bun scripts/check/check-compression-budget.ts",
|
||||
"check:router-eval": "node --import tsx scripts/check/check-router-eval-regression.ts",
|
||||
"check:doc-links": "node scripts/check/check-doc-links.mjs",
|
||||
"check:fabricated-docs": "node scripts/check/check-fabricated-docs.mjs --strict",
|
||||
"check:docs-all": "npm run check:docs-sync && npm run check:docs-counts && npm run check:env-doc-sync && npm run check:deprecated-versions && npm run check:doc-links && npm run check:fabricated-docs",
|
||||
|
||||
287
scripts/check/check-router-eval-regression.ts
Normal file
287
scripts/check/check-router-eval-regression.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type Args = {
|
||||
baseline: string;
|
||||
candidate: string;
|
||||
baselinePatch?: string;
|
||||
candidatePatch?: string;
|
||||
output: string;
|
||||
jsonOutput: string;
|
||||
patchOutput: string;
|
||||
patchJsonOutput: string;
|
||||
artifactDir?: string;
|
||||
runId: string;
|
||||
maxAiqDrop: string;
|
||||
maxCostIncrease: string;
|
||||
maxPatchAiqDrop: string;
|
||||
maxPatchCostIncrease: string;
|
||||
maxPatchLatencyIncrease: string;
|
||||
maxPatchRegressionIncrease: string;
|
||||
};
|
||||
|
||||
type GateManifest = {
|
||||
schemaVersion: 1;
|
||||
kind: "router-eval-gate-run";
|
||||
runId: string;
|
||||
generatedAt: string;
|
||||
command: string[];
|
||||
thresholds: {
|
||||
maxAiqDrop: number;
|
||||
maxCostIncrease: number;
|
||||
patch?: {
|
||||
maxAiqDrop: number;
|
||||
maxCostIncrease: number;
|
||||
maxLatencyIncrease: number;
|
||||
maxRegressionIncrease: number;
|
||||
};
|
||||
};
|
||||
inputs: {
|
||||
baseline: string;
|
||||
candidate: string;
|
||||
baselinePatch?: string;
|
||||
candidatePatch?: string;
|
||||
};
|
||||
outputs: {
|
||||
markdown: string;
|
||||
json: string;
|
||||
patchMarkdown?: string;
|
||||
patchJson?: string;
|
||||
};
|
||||
environment: {
|
||||
runtime: "bun" | "node";
|
||||
platform: NodeJS.Platform;
|
||||
};
|
||||
result: {
|
||||
status: number;
|
||||
};
|
||||
};
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const isBunRuntime = "Bun" in globalThis;
|
||||
|
||||
function runTypeScriptScript(args: string[]) {
|
||||
return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
const defaultFixtureDir = path.join(repoRoot, "tests/fixtures/router-eval");
|
||||
const defaultArtifactDir = path.join(os.tmpdir(), "omniroute-router-eval");
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
return value && !value.startsWith("--") ? value : undefined;
|
||||
}
|
||||
|
||||
function readArgs(): Args {
|
||||
const artifactDir = getArgValue("artifact-dir");
|
||||
const runId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const retainedDir = artifactDir ? path.join(path.resolve(artifactDir), runId) : undefined;
|
||||
return {
|
||||
baseline: getArgValue("baseline") ?? path.join(defaultFixtureDir, "baseline.ndjson"),
|
||||
candidate: getArgValue("candidate") ?? path.join(defaultFixtureDir, "candidate.ndjson"),
|
||||
baselinePatch: getArgValue("baseline-patch"),
|
||||
candidatePatch: getArgValue("candidate-patch"),
|
||||
output: getArgValue("output") ?? path.join(retainedDir ?? defaultArtifactDir, "router-eval.md"),
|
||||
jsonOutput:
|
||||
getArgValue("json-output") ??
|
||||
path.join(retainedDir ?? defaultArtifactDir, "router-eval.json"),
|
||||
patchOutput:
|
||||
getArgValue("patch-output") ??
|
||||
path.join(retainedDir ?? defaultArtifactDir, "patch-comparison.md"),
|
||||
patchJsonOutput:
|
||||
getArgValue("patch-json-output") ??
|
||||
path.join(retainedDir ?? defaultArtifactDir, "patch-comparison.json"),
|
||||
artifactDir,
|
||||
runId,
|
||||
maxAiqDrop: getArgValue("max-aiq-drop") ?? "1",
|
||||
maxCostIncrease: getArgValue("max-cost-increase") ?? "0.05",
|
||||
maxPatchAiqDrop: getArgValue("max-patch-aiq-drop") ?? "1",
|
||||
maxPatchCostIncrease: getArgValue("max-patch-cost-increase") ?? "0.05",
|
||||
maxPatchLatencyIncrease: getArgValue("max-patch-latency-increase") ?? "0.05",
|
||||
maxPatchRegressionIncrease: getArgValue("max-patch-regression-increase") ?? "0",
|
||||
};
|
||||
}
|
||||
|
||||
function ensureReadable(filePath: string, label: string): void {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`[router-eval] ${label} missing: ${filePath}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
function writeRetainedRun(args: Args, status: number): void {
|
||||
if (!args.artifactDir) return;
|
||||
|
||||
const runDir = path.join(path.resolve(args.artifactDir), args.runId);
|
||||
const inputDir = path.join(runDir, "inputs");
|
||||
fs.mkdirSync(inputDir, { recursive: true });
|
||||
|
||||
const baselineCopy = path.join(inputDir, "baseline.ndjson");
|
||||
const candidateCopy = path.join(inputDir, "candidate.ndjson");
|
||||
fs.copyFileSync(args.baseline, baselineCopy);
|
||||
fs.copyFileSync(args.candidate, candidateCopy);
|
||||
const baselinePatchCopy = args.baselinePatch
|
||||
? path.join(inputDir, "baseline.patch.json")
|
||||
: undefined;
|
||||
const candidatePatchCopy = args.candidatePatch
|
||||
? path.join(inputDir, "candidate.patch.json")
|
||||
: undefined;
|
||||
if (args.baselinePatch && baselinePatchCopy)
|
||||
fs.copyFileSync(args.baselinePatch, baselinePatchCopy);
|
||||
if (args.candidatePatch && candidatePatchCopy)
|
||||
fs.copyFileSync(args.candidatePatch, candidatePatchCopy);
|
||||
|
||||
const manifest: GateManifest = {
|
||||
schemaVersion: 1,
|
||||
kind: "router-eval-gate-run",
|
||||
runId: args.runId,
|
||||
generatedAt: new Date().toISOString(),
|
||||
command: process.argv.slice(1),
|
||||
thresholds: {
|
||||
maxAiqDrop: Number.parseFloat(args.maxAiqDrop),
|
||||
maxCostIncrease: Number.parseFloat(args.maxCostIncrease),
|
||||
...(args.baselinePatch && args.candidatePatch
|
||||
? {
|
||||
patch: {
|
||||
maxAiqDrop: Number.parseFloat(args.maxPatchAiqDrop),
|
||||
maxCostIncrease: Number.parseFloat(args.maxPatchCostIncrease),
|
||||
maxLatencyIncrease: Number.parseFloat(args.maxPatchLatencyIncrease),
|
||||
maxRegressionIncrease: Number.parseFloat(args.maxPatchRegressionIncrease),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
inputs: {
|
||||
baseline: path.relative(runDir, baselineCopy),
|
||||
candidate: path.relative(runDir, candidateCopy),
|
||||
...(baselinePatchCopy && candidatePatchCopy
|
||||
? {
|
||||
baselinePatch: path.relative(runDir, baselinePatchCopy),
|
||||
candidatePatch: path.relative(runDir, candidatePatchCopy),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
outputs: {
|
||||
markdown: path.relative(runDir, args.output),
|
||||
json: path.relative(runDir, args.jsonOutput),
|
||||
...(args.baselinePatch && args.candidatePatch
|
||||
? {
|
||||
patchMarkdown: path.relative(runDir, args.patchOutput),
|
||||
patchJson: path.relative(runDir, args.patchJsonOutput),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
environment: {
|
||||
runtime: isBunRuntime ? "bun" : "node",
|
||||
platform: process.platform,
|
||||
},
|
||||
result: {
|
||||
status,
|
||||
},
|
||||
};
|
||||
|
||||
fs.writeFileSync(path.join(runDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function runPatchGate(args: Args): number {
|
||||
if (!args.baselinePatch && !args.candidatePatch) return 0;
|
||||
if (!args.baselinePatch || !args.candidatePatch) {
|
||||
console.error("[router-eval] --baseline-patch and --candidate-patch must be provided together");
|
||||
return 2;
|
||||
}
|
||||
ensureReadable(args.baselinePatch, "baseline patch");
|
||||
ensureReadable(args.candidatePatch, "candidate patch");
|
||||
fs.mkdirSync(path.dirname(args.patchOutput), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(args.patchJsonOutput), { recursive: true });
|
||||
|
||||
const result = runTypeScriptScript([
|
||||
"scripts/router-eval/patch-compare.ts",
|
||||
"--baseline",
|
||||
args.baselinePatch,
|
||||
"--candidate",
|
||||
args.candidatePatch,
|
||||
"--output",
|
||||
args.patchOutput,
|
||||
"--json-output",
|
||||
args.patchJsonOutput,
|
||||
"--run-id",
|
||||
args.runId,
|
||||
"--max-aiq-drop",
|
||||
args.maxPatchAiqDrop,
|
||||
"--max-cost-increase",
|
||||
args.maxPatchCostIncrease,
|
||||
"--max-latency-increase",
|
||||
args.maxPatchLatencyIncrease,
|
||||
"--max-regression-increase",
|
||||
args.maxPatchRegressionIncrease,
|
||||
"--fail-on-regression",
|
||||
]);
|
||||
|
||||
if (result.error) {
|
||||
console.error(`[router-eval] failed to launch patch compare: ${result.error.message}`);
|
||||
return 1;
|
||||
}
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
return result.status ?? 1;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const args = readArgs();
|
||||
ensureReadable(args.baseline, "baseline corpus");
|
||||
ensureReadable(args.candidate, "candidate corpus");
|
||||
fs.mkdirSync(path.dirname(args.output), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(args.jsonOutput), { recursive: true });
|
||||
|
||||
const result = runTypeScriptScript([
|
||||
"scripts/router-eval/index.ts",
|
||||
"--input",
|
||||
args.candidate,
|
||||
"--baseline-input",
|
||||
args.baseline,
|
||||
"--max-aiq-drop",
|
||||
args.maxAiqDrop,
|
||||
"--max-cost-increase",
|
||||
args.maxCostIncrease,
|
||||
"--output",
|
||||
args.output,
|
||||
"--json-output",
|
||||
args.jsonOutput,
|
||||
"--fail-on-regression",
|
||||
]);
|
||||
|
||||
if (result.error) {
|
||||
console.error(`[router-eval] failed to launch evaluator: ${result.error.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
|
||||
if (result.status === 0) {
|
||||
const patchStatus = runPatchGate(args);
|
||||
writeRetainedRun(args, patchStatus);
|
||||
if (patchStatus !== 0) {
|
||||
console.error(`[router-eval] patch gate failed with exit code ${patchStatus}`);
|
||||
process.exit(patchStatus);
|
||||
}
|
||||
const retention = args.artifactDir ? ` retained run ${args.runId}` : " temp run";
|
||||
console.log(`[router-eval] OK -${retention}; artifacts: ${args.output}, ${args.jsonOutput}`);
|
||||
return;
|
||||
}
|
||||
|
||||
writeRetainedRun(args, result.status ?? 1);
|
||||
console.error(`[router-eval] regression gate failed with exit code ${result.status ?? 1}`);
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
main();
|
||||
135
scripts/router-eval/compare.ts
Normal file
135
scripts/router-eval/compare.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type Args = {
|
||||
baseline: string;
|
||||
candidate: string;
|
||||
baselineName: string;
|
||||
candidateName: string;
|
||||
artifactDir: string;
|
||||
runId: string;
|
||||
maxAiqDrop: string;
|
||||
maxCostIncrease: string;
|
||||
failOnRegression: boolean;
|
||||
};
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const isBunRuntime = "Bun" in globalThis;
|
||||
|
||||
function runTypeScriptScript(args: string[]) {
|
||||
return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
return value && !value.startsWith("--") ? value : undefined;
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
"Usage:",
|
||||
" npm run eval:router:compare -- --baseline <baseline.ndjson> --candidate <candidate.ndjson>",
|
||||
" [--baseline-name <name>] [--candidate-name <name>] [--artifact-dir <dir>]",
|
||||
" [--run-id <id>] [--max-aiq-drop <n>] [--max-cost-increase <n>] [--fail-on-regression]",
|
||||
"",
|
||||
"Runs a named baseline-vs-candidate router-eval comparison and retains artifacts.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function requireArg(name: string): string {
|
||||
const value = getArgValue(name);
|
||||
if (!value) {
|
||||
console.error(`Missing required --${name}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readArgs(): Args {
|
||||
const baselineName = getArgValue("baseline-name") ?? "baseline";
|
||||
const candidateName = getArgValue("candidate-name") ?? "candidate";
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
||||
return {
|
||||
baseline: requireArg("baseline"),
|
||||
candidate: requireArg("candidate"),
|
||||
baselineName,
|
||||
candidateName,
|
||||
artifactDir: getArgValue("artifact-dir") ?? "artifacts/router-eval/comparisons",
|
||||
runId: getArgValue("run-id") ?? `${baselineName}-vs-${candidateName}-${timestamp}`,
|
||||
maxAiqDrop: getArgValue("max-aiq-drop") ?? "1",
|
||||
maxCostIncrease: getArgValue("max-cost-increase") ?? "0.05",
|
||||
failOnRegression: process.argv.includes("--fail-on-regression"),
|
||||
};
|
||||
}
|
||||
|
||||
function ensureReadable(filePath: string, label: string): void {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`[router-eval:compare] ${label} missing: ${filePath}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
if (process.argv.includes("--help") || process.argv.includes("-h")) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const args = readArgs();
|
||||
ensureReadable(args.baseline, "baseline corpus");
|
||||
ensureReadable(args.candidate, "candidate corpus");
|
||||
|
||||
const checkArgs = [
|
||||
"scripts/check/check-router-eval-regression.ts",
|
||||
"--baseline",
|
||||
args.baseline,
|
||||
"--candidate",
|
||||
args.candidate,
|
||||
"--artifact-dir",
|
||||
args.artifactDir,
|
||||
"--run-id",
|
||||
args.runId,
|
||||
"--max-aiq-drop",
|
||||
args.maxAiqDrop,
|
||||
"--max-cost-increase",
|
||||
args.maxCostIncrease,
|
||||
];
|
||||
|
||||
const result = runTypeScriptScript(checkArgs);
|
||||
|
||||
if (result.error) {
|
||||
console.error(`[router-eval:compare] failed to launch comparison: ${result.error.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
|
||||
const runDir = path.resolve(args.artifactDir, args.runId);
|
||||
const labels = {
|
||||
baselineName: args.baselineName,
|
||||
candidateName: args.candidateName,
|
||||
baseline: path.relative(runDir, path.resolve(args.baseline)),
|
||||
candidate: path.relative(runDir, path.resolve(args.candidate)),
|
||||
};
|
||||
fs.mkdirSync(runDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(runDir, "comparison.json"), `${JSON.stringify(labels, null, 2)}\n`);
|
||||
|
||||
if (result.status === 0 || !args.failOnRegression) {
|
||||
console.log(`[router-eval:compare] artifacts: ${runDir}`);
|
||||
return;
|
||||
}
|
||||
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
|
||||
main();
|
||||
483
scripts/router-eval/index.ts
Normal file
483
scripts/router-eval/index.ts
Normal file
@@ -0,0 +1,483 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
compareRouterEvalRuns,
|
||||
createRouterEvalArtifact,
|
||||
formatRouterEvalComparison,
|
||||
formatRouterEvalReport,
|
||||
runRouterEval,
|
||||
toRouterObservation,
|
||||
type RouterEvalArtifact,
|
||||
type RouterEvalArtifactMetadata,
|
||||
type RouterObservation,
|
||||
} from "@/lib/routerEval/index.ts";
|
||||
import { SQLITE_FILE } from "@/lib/db/core.ts";
|
||||
|
||||
type DbCallLogRow = {
|
||||
id: string;
|
||||
model: string | null;
|
||||
requested_model: string | null;
|
||||
duration: number | null;
|
||||
tokens_in: number | null;
|
||||
tokens_out: number | null;
|
||||
status: number | null;
|
||||
combo_name: string | null;
|
||||
provider: string | null;
|
||||
error_summary: string | null;
|
||||
timestamp: string | null;
|
||||
correlation_id: string | null;
|
||||
};
|
||||
|
||||
type DbUsageHistoryRow = {
|
||||
id: number;
|
||||
provider: string | null;
|
||||
model: string | null;
|
||||
tokens_input: number | null;
|
||||
tokens_output: number | null;
|
||||
service_tier: string | null;
|
||||
status: string | null;
|
||||
success: number | null;
|
||||
latency_ms: number | null;
|
||||
error_code: string | null;
|
||||
combo_strategy: string | null;
|
||||
timestamp: string | null;
|
||||
};
|
||||
|
||||
type DbReplaySource = "auto" | "call-logs" | "usage-history";
|
||||
|
||||
type SqliteStatement = {
|
||||
get: (...params: unknown[]) => unknown;
|
||||
all: (...params: unknown[]) => unknown[];
|
||||
};
|
||||
|
||||
type SqliteDatabase = {
|
||||
prepare: (sql: string) => SqliteStatement;
|
||||
close: () => void;
|
||||
};
|
||||
|
||||
type ArgSpec = {
|
||||
input?: string;
|
||||
db?: string;
|
||||
dbSource?: DbReplaySource;
|
||||
baselineInput?: string;
|
||||
baselineDb?: string;
|
||||
baselineDbSource?: DbReplaySource;
|
||||
since?: string;
|
||||
limit?: number;
|
||||
aiqDrop?: number;
|
||||
costIncrease?: number;
|
||||
output?: string;
|
||||
jsonOutput?: string;
|
||||
exportCorpus?: string;
|
||||
failOnRegression?: boolean;
|
||||
help?: boolean;
|
||||
};
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
if (!value || value.startsWith("--")) return undefined;
|
||||
return value;
|
||||
}
|
||||
|
||||
function getNumericArg(name: string): number | undefined {
|
||||
const value = getArgValue(name);
|
||||
if (!value) return undefined;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function getFloatArg(name: string): number | undefined {
|
||||
const value = getArgValue(name);
|
||||
if (!value) return undefined;
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseArgs(): ArgSpec {
|
||||
return {
|
||||
input: getArgValue("input"),
|
||||
db: getArgValue("db"),
|
||||
dbSource: parseReplaySource(getArgValue("db-source")),
|
||||
baselineInput: getArgValue("baseline-input"),
|
||||
baselineDb: getArgValue("baseline-db"),
|
||||
baselineDbSource: parseReplaySource(getArgValue("baseline-db-source")),
|
||||
since: getArgValue("since"),
|
||||
limit: getNumericArg("limit"),
|
||||
aiqDrop: getFloatArg("max-aiq-drop"),
|
||||
costIncrease: getFloatArg("max-cost-increase"),
|
||||
output: getArgValue("output"),
|
||||
jsonOutput: getArgValue("json-output"),
|
||||
exportCorpus: getArgValue("export-corpus"),
|
||||
failOnRegression: process.argv.includes("--fail-on-regression"),
|
||||
help: process.argv.includes("--help") || process.argv.includes("-h"),
|
||||
};
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"Usage:",
|
||||
" npm run eval:router -- --input <file.ndjson> [--since <iso>] [--limit <n>]",
|
||||
" npm run eval:router -- --db [path] [--db-source usage-history|call-logs|auto] [--since <iso>] [--limit <n>]",
|
||||
" npm run eval:router -- --input <candidate> --baseline-input <baseline>",
|
||||
" npm run eval:router -- --db <path> --db-source usage-history",
|
||||
" [--max-aiq-drop <n>] [--max-cost-increase <n>] [--fail-on-regression]",
|
||||
"",
|
||||
"Options:",
|
||||
" --input <path> JSONL observation corpus (or omit for stdin)",
|
||||
" --db [path] Read SQLite rows from the routing-replay source",
|
||||
" --db-source <auto|call-logs|usage-history> Source for --db reads (default: auto => call-logs then usage-history)",
|
||||
" --baseline-input <path> Baseline corpus in JSONL",
|
||||
" --baseline-db <path> Baseline corpus in SQLite",
|
||||
" --baseline-db-source <auto|call-logs|usage-history> Source for baseline DB reads",
|
||||
" --since <ISO8601> Filter rows newer than this value",
|
||||
" --limit <n> Limit sample count",
|
||||
" --max-aiq-drop <n> Regression threshold (default: 0)",
|
||||
" --max-cost-increase <n> Relative increase threshold (default: 0)",
|
||||
" --output <path> Write report to file",
|
||||
" --json-output <path> Write machine-readable artifact JSON",
|
||||
" --export-corpus <path> Write normalized RouterObservation JSONL",
|
||||
" --fail-on-regression Exit 1 if candidate regresses vs baseline",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parseInputLine(rawLine: string): RouterObservation | null {
|
||||
const trimmed = rawLine.trim();
|
||||
if (!trimmed) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
return toRouterObservation(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJsonl(inputPath?: string): Promise<RouterObservation[]> {
|
||||
let text: string;
|
||||
if (!inputPath) {
|
||||
text = await new Response(process.stdin, { duplex: "half" }).text();
|
||||
} else {
|
||||
text = await fs.promises.readFile(path.resolve(inputPath), "utf8");
|
||||
}
|
||||
|
||||
const observations: RouterObservation[] = [];
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const parsed = parseInputLine(line);
|
||||
if (parsed) observations.push(parsed);
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
function estimateCost(tokensIn: unknown, tokensOut: unknown): number {
|
||||
const inTokens = typeof tokensIn === "number" ? tokensIn : 0;
|
||||
const outTokens = typeof tokensOut === "number" ? tokensOut : 0;
|
||||
return Number(((inTokens + outTokens) * 0.000001).toFixed(6));
|
||||
}
|
||||
|
||||
function parseReplaySource(rawSource?: string): DbReplaySource {
|
||||
if (!rawSource) return "auto";
|
||||
const normalized = rawSource.toLowerCase();
|
||||
if (normalized === "auto") return "auto";
|
||||
if (normalized === "usage_history" || normalized === "usage-history") return "usage-history";
|
||||
if (normalized === "call_logs" || normalized === "call-logs") return "call-logs";
|
||||
throw new Error(`Unsupported db source: ${rawSource}`);
|
||||
}
|
||||
|
||||
function hasReplayTable(database: SqliteDatabase, tableName: string): boolean {
|
||||
return Boolean(
|
||||
database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(tableName)
|
||||
);
|
||||
}
|
||||
|
||||
function resolveReplaySource(
|
||||
database: SqliteDatabase,
|
||||
requestedSource: DbReplaySource
|
||||
): DbReplaySource {
|
||||
const hasUsageHistory = hasReplayTable(database, "usage_history");
|
||||
const hasCallLogs = hasReplayTable(database, "call_logs");
|
||||
|
||||
if (requestedSource === "usage-history") {
|
||||
if (!hasUsageHistory) throw new Error("Table 'usage_history' missing in database");
|
||||
return "usage-history";
|
||||
}
|
||||
|
||||
if (requestedSource === "call-logs") {
|
||||
if (!hasCallLogs) throw new Error("Table 'call_logs' missing in database");
|
||||
return "call-logs";
|
||||
}
|
||||
|
||||
if (requestedSource === "auto") {
|
||||
if (hasCallLogs) return "call-logs";
|
||||
if (hasUsageHistory) return "usage-history";
|
||||
}
|
||||
|
||||
throw new Error("No replay table found in database (expected usage_history or call_logs)");
|
||||
}
|
||||
|
||||
function toSuccessFromStatus(status: unknown): boolean {
|
||||
if (typeof status === "number") return status >= 200 && status < 400;
|
||||
if (typeof status === "string") {
|
||||
const parsed = Number.parseInt(status, 10);
|
||||
if (Number.isFinite(parsed)) return parsed >= 200 && parsed < 400;
|
||||
const normalized = status.trim().toLowerCase();
|
||||
if (normalized === "ok" || normalized === "success" || normalized === "true") return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function readCallLogDb(db: SqliteDatabase, since?: string, limit?: number): RouterObservation[] {
|
||||
const queryParts = [
|
||||
"SELECT id, model, requested_model, duration, tokens_in, tokens_out, status, combo_name, provider, error_summary, timestamp, correlation_id",
|
||||
"FROM call_logs",
|
||||
"WHERE 1=1",
|
||||
];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (since) {
|
||||
queryParts.push("AND timestamp >= ?");
|
||||
params.push(since);
|
||||
}
|
||||
|
||||
queryParts.push("ORDER BY timestamp ASC");
|
||||
if (limit) {
|
||||
queryParts.push("LIMIT ?");
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
const rows = db.prepare(queryParts.join(" ")).all(...params) as DbCallLogRow[];
|
||||
|
||||
const observations: RouterObservation[] = [];
|
||||
for (const row of rows) {
|
||||
const mapped = toRouterObservation({
|
||||
sampleId: row.id,
|
||||
model: row.model,
|
||||
requestedModel: row.requested_model,
|
||||
latency: row.duration ?? 0,
|
||||
costUsd: estimateCost(row.tokens_in, row.tokens_out),
|
||||
configId: row.combo_name || row.provider || "default",
|
||||
success: row.status != null && row.status >= 200 && row.status < 400,
|
||||
status: row.status ?? 0,
|
||||
error: row.error_summary,
|
||||
routeInput: {
|
||||
correlationId: row.correlation_id ?? "",
|
||||
},
|
||||
timestamp: row.timestamp ?? new Date().toISOString(),
|
||||
});
|
||||
if (mapped) observations.push(mapped);
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
function readUsageHistoryDb(
|
||||
db: SqliteDatabase,
|
||||
since?: string,
|
||||
limit?: number
|
||||
): RouterObservation[] {
|
||||
const queryParts = [
|
||||
"SELECT id, provider, model, tokens_input, tokens_output, service_tier, status, success, latency_ms, error_code, combo_strategy, timestamp",
|
||||
"FROM usage_history",
|
||||
"WHERE 1=1",
|
||||
];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (since) {
|
||||
queryParts.push("AND timestamp >= ?");
|
||||
params.push(since);
|
||||
}
|
||||
|
||||
queryParts.push("ORDER BY timestamp ASC");
|
||||
if (limit) {
|
||||
queryParts.push("LIMIT ?");
|
||||
params.push(limit);
|
||||
}
|
||||
|
||||
const rows = db.prepare(queryParts.join(" ")).all(...params) as DbUsageHistoryRow[];
|
||||
|
||||
const observations: RouterObservation[] = [];
|
||||
for (const row of rows) {
|
||||
const cost = estimateCost(row.tokens_input, row.tokens_output);
|
||||
const rowId = `${row.id}`;
|
||||
const mapped = toRouterObservation({
|
||||
sampleId: rowId,
|
||||
model: row.model,
|
||||
requestedModel: row.model,
|
||||
latency: row.latency_ms ?? 0,
|
||||
costUsd: cost,
|
||||
configId: row.combo_strategy || row.provider || "default",
|
||||
success: toSuccessFromStatus(row.status) || row.success === 1,
|
||||
status: row.success === 1 ? 200 : 0,
|
||||
routeInput: {},
|
||||
metadata: {
|
||||
provider: row.provider,
|
||||
serviceTier: row.service_tier,
|
||||
errorCode: row.error_code,
|
||||
},
|
||||
timestamp: row.timestamp ?? new Date().toISOString(),
|
||||
});
|
||||
if (mapped) observations.push(mapped);
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
async function openSqliteDatabase(sqliteFile: string): Promise<SqliteDatabase> {
|
||||
if ("Bun" in globalThis) {
|
||||
const sqlite = await import("bun:sqlite");
|
||||
return new sqlite.Database(sqliteFile, { readonly: true });
|
||||
}
|
||||
|
||||
const sqlite = await import("better-sqlite3");
|
||||
return new sqlite.default(sqliteFile, { readonly: true });
|
||||
}
|
||||
|
||||
async function readDb(
|
||||
filePath: string,
|
||||
since?: string,
|
||||
limit?: number,
|
||||
source: DbReplaySource = "auto"
|
||||
): Promise<RouterObservation[]> {
|
||||
const sqliteFile = filePath || SQLITE_FILE;
|
||||
if (!sqliteFile) throw new Error("SQLite mode requires a path or SQLITE_FILE");
|
||||
const db = await openSqliteDatabase(sqliteFile);
|
||||
try {
|
||||
const normalized = parseReplaySource(source);
|
||||
const activeSource = resolveReplaySource(db, normalized);
|
||||
if (activeSource === "usage-history") {
|
||||
return readUsageHistoryDb(db, since, limit);
|
||||
}
|
||||
return readCallLogDb(db, since, limit);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDbPath(rawArg?: string): string {
|
||||
if (rawArg) return path.resolve(rawArg);
|
||||
if (SQLITE_FILE) return SQLITE_FILE;
|
||||
throw new Error("No SQLITE_FILE and no --db path provided");
|
||||
}
|
||||
|
||||
function describeInputSource(
|
||||
inputPath: string | undefined,
|
||||
dbPath: string | undefined,
|
||||
dbSource: DbReplaySource | undefined,
|
||||
usesDb: boolean
|
||||
): { source: string; path?: string; dbSource?: string } {
|
||||
if (inputPath) return { source: "jsonl", path: path.resolve(inputPath) };
|
||||
if (usesDb) {
|
||||
return {
|
||||
source: "sqlite",
|
||||
path: resolveDbPath(dbPath),
|
||||
dbSource: dbSource ?? "auto",
|
||||
};
|
||||
}
|
||||
return { source: "stdin" };
|
||||
}
|
||||
|
||||
function buildArtifactMetadata(args: ArgSpec, hasCandidateDb: boolean): RouterEvalArtifactMetadata {
|
||||
const hasBaselineDb = Boolean(args.baselineDb);
|
||||
return {
|
||||
candidate: describeInputSource(args.input, args.db, args.dbSource, hasCandidateDb),
|
||||
baseline:
|
||||
args.baselineInput || hasBaselineDb
|
||||
? describeInputSource(
|
||||
args.baselineInput,
|
||||
args.baselineDb,
|
||||
args.baselineDbSource,
|
||||
hasBaselineDb
|
||||
)
|
||||
: undefined,
|
||||
window: {
|
||||
since: args.since,
|
||||
limit: args.limit,
|
||||
},
|
||||
thresholds: {
|
||||
maxAiqDrop: args.aiqDrop ?? 0,
|
||||
maxCostIncrease: args.costIncrease ?? 0,
|
||||
},
|
||||
outputs: {
|
||||
markdown: args.output ? path.resolve(args.output) : undefined,
|
||||
json: args.jsonOutput ? path.resolve(args.jsonOutput) : undefined,
|
||||
corpus: args.exportCorpus ? path.resolve(args.exportCorpus) : undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function writeCorpus(pathArg: string, observations: RouterObservation[]): Promise<void> {
|
||||
const outPath = path.resolve(pathArg);
|
||||
await fs.promises.mkdir(path.dirname(outPath), { recursive: true });
|
||||
const lines = observations.map((observation) => JSON.stringify(observation));
|
||||
await fs.promises.writeFile(outPath, `${lines.join("\n")}\n`, "utf8");
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const args = parseArgs();
|
||||
if (args.help) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const hasCandidateDb = Boolean(args.db || process.argv.includes("--db"));
|
||||
const candidate: RouterObservation[] = args.input
|
||||
? await readJsonl(args.input)
|
||||
: hasCandidateDb
|
||||
? await readDb(resolveDbPath(args.db), args.since, args.limit, args.dbSource)
|
||||
: await readJsonl();
|
||||
|
||||
const baseline: RouterObservation[] | undefined = args.baselineInput
|
||||
? await readJsonl(args.baselineInput)
|
||||
: args.baselineDb
|
||||
? await readDb(resolveDbPath(args.baselineDb), args.since, args.limit, args.baselineDbSource)
|
||||
: undefined;
|
||||
|
||||
if (candidate.length === 0) {
|
||||
console.error("No candidate observations found");
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.exportCorpus) {
|
||||
await writeCorpus(args.exportCorpus, candidate);
|
||||
}
|
||||
|
||||
const report = runRouterEval(candidate);
|
||||
const metadata = buildArtifactMetadata(args, hasCandidateDb);
|
||||
let output = formatRouterEvalReport(report);
|
||||
let artifact: RouterEvalArtifact = createRouterEvalArtifact(report, metadata);
|
||||
|
||||
if (baseline && baseline.length > 0) {
|
||||
const comparison = compareRouterEvalRuns(runRouterEval(baseline), report, {
|
||||
aiqDrop: args.aiqDrop ?? 0,
|
||||
relativeCostIncrease: args.costIncrease ?? 0,
|
||||
});
|
||||
output = formatRouterEvalComparison(comparison);
|
||||
artifact = createRouterEvalArtifact(comparison, metadata);
|
||||
console.log(output);
|
||||
if (args.failOnRegression && comparison.regressions.length > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} else {
|
||||
console.log(output);
|
||||
}
|
||||
|
||||
if (args.output) {
|
||||
const outPath = path.resolve(args.output);
|
||||
await fs.promises.writeFile(outPath, output, "utf8");
|
||||
}
|
||||
|
||||
if (args.jsonOutput) {
|
||||
const outPath = path.resolve(args.jsonOutput);
|
||||
await fs.promises.writeFile(outPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
run().catch((error) => {
|
||||
if (error && typeof error === "object" && "message" in error) {
|
||||
console.error((error as Error).message);
|
||||
} else {
|
||||
console.error(String(error));
|
||||
}
|
||||
process.exitCode = 1;
|
||||
});
|
||||
315
scripts/router-eval/patch-compare.ts
Normal file
315
scripts/router-eval/patch-compare.ts
Normal file
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
type PatchOperation = {
|
||||
op?: string;
|
||||
path?: string;
|
||||
value?: string;
|
||||
evidence?: {
|
||||
aiq?: number;
|
||||
avgCostUsd?: number;
|
||||
avgLatencyMs?: number;
|
||||
regressions?: number;
|
||||
};
|
||||
rationale?: string;
|
||||
};
|
||||
|
||||
type RouterConfigPatchArtifact = {
|
||||
schemaVersion?: number;
|
||||
kind?: string;
|
||||
generatedAt?: string;
|
||||
applyPolicy?: string;
|
||||
source?: {
|
||||
objective?: string;
|
||||
runId?: string;
|
||||
artifactPath?: string;
|
||||
};
|
||||
operations?: PatchOperation[];
|
||||
};
|
||||
|
||||
type PatchComparison = {
|
||||
schemaVersion: 1;
|
||||
kind: "router-config-patch-comparison";
|
||||
generatedAt: string;
|
||||
runId: string;
|
||||
thresholds: PatchThresholds;
|
||||
baseline: PatchSummary;
|
||||
candidate: PatchSummary;
|
||||
delta: {
|
||||
aiq: number;
|
||||
avgCostUsd: number;
|
||||
costIncreaseRatio: number;
|
||||
avgLatencyMs: number;
|
||||
latencyIncreaseRatio: number;
|
||||
regressions: number;
|
||||
};
|
||||
changedRecommendation: boolean;
|
||||
regressions: string[];
|
||||
result: {
|
||||
passed: boolean;
|
||||
status: 0 | 1;
|
||||
};
|
||||
};
|
||||
|
||||
type PatchThresholds = {
|
||||
maxAiqDrop: number;
|
||||
maxCostIncrease: number;
|
||||
maxLatencyIncrease: number;
|
||||
maxRegressionIncrease: number;
|
||||
};
|
||||
|
||||
type PatchSummary = {
|
||||
name: string;
|
||||
file: string;
|
||||
objective: string;
|
||||
runId: string;
|
||||
recommendedConfigId: string;
|
||||
aiq: number;
|
||||
avgCostUsd: number;
|
||||
avgLatencyMs: number;
|
||||
regressions: number;
|
||||
applyPolicy: string;
|
||||
};
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
return value && !value.startsWith("--") ? value : undefined;
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
"Usage:",
|
||||
" npm run eval:router:patch-compare -- --baseline <router-config.patch.json> --candidate <router-config.patch.json>",
|
||||
" [--baseline-name <name>] [--candidate-name <name>] [--artifact-dir <dir>] [--run-id <id>]",
|
||||
" [--output <file>] [--json-output <file>] [--fail-on-regression]",
|
||||
" [--max-aiq-drop <n>] [--max-cost-increase <ratio>] [--max-latency-increase <ratio>]",
|
||||
" [--max-regression-increase <n>]",
|
||||
"",
|
||||
"Compares two retained router config patch proposals without applying them.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function getNumberArg(name: string, fallback: number): number {
|
||||
const value = getArgValue(name);
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
console.error(`Invalid --${name} value: ${value}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function requireArg(name: string): string {
|
||||
const value = getArgValue(name);
|
||||
if (!value) {
|
||||
console.error(`Missing required --${name}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readPatch(file: string): RouterConfigPatchArtifact {
|
||||
if (!fs.existsSync(file)) {
|
||||
console.error(`[router-eval:patch-compare] patch file missing: ${file}`);
|
||||
process.exit(2);
|
||||
}
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8")) as RouterConfigPatchArtifact;
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[router-eval:patch-compare] invalid JSON in ${file}: ${(error as Error).message}`
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
function requireNumber(value: unknown, field: string, file: string): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
console.error(`[router-eval:patch-compare] invalid numeric evidence field ${field} in ${file}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function summarizePatch(
|
||||
name: string,
|
||||
file: string,
|
||||
patch: RouterConfigPatchArtifact
|
||||
): PatchSummary {
|
||||
if (patch.kind !== "router-config-patch") {
|
||||
console.error(
|
||||
`[router-eval:patch-compare] invalid patch kind in ${file}: ${patch.kind ?? "missing"}`
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
const operation = patch.operations?.[0];
|
||||
if (operation?.op !== "recommend-router-config") {
|
||||
console.error(
|
||||
`[router-eval:patch-compare] missing recommend-router-config operation in ${file}`
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
if (typeof operation.value !== "string" || operation.value.length === 0) {
|
||||
console.error(`[router-eval:patch-compare] invalid recommended config value in ${file}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return {
|
||||
name,
|
||||
file: path.resolve(file),
|
||||
objective: patch.source?.objective ?? "unknown",
|
||||
runId: patch.source?.runId ?? "unknown",
|
||||
recommendedConfigId: operation.value,
|
||||
aiq: requireNumber(operation.evidence?.aiq, "aiq", file),
|
||||
avgCostUsd: requireNumber(operation.evidence?.avgCostUsd, "avgCostUsd", file),
|
||||
avgLatencyMs: requireNumber(operation.evidence?.avgLatencyMs, "avgLatencyMs", file),
|
||||
regressions: requireNumber(operation.evidence?.regressions, "regressions", file),
|
||||
applyPolicy: patch.applyPolicy ?? "unknown",
|
||||
};
|
||||
}
|
||||
|
||||
function increaseRatio(delta: number, baseline: number): number {
|
||||
if (baseline === 0) return delta > 0 ? Number.POSITIVE_INFINITY : 0;
|
||||
return delta / baseline;
|
||||
}
|
||||
|
||||
function findRegressions(
|
||||
baseline: PatchSummary,
|
||||
candidate: PatchSummary,
|
||||
thresholds: PatchThresholds
|
||||
): string[] {
|
||||
const aiqDrop = baseline.aiq - candidate.aiq;
|
||||
const costDelta = candidate.avgCostUsd - baseline.avgCostUsd;
|
||||
const latencyDelta = candidate.avgLatencyMs - baseline.avgLatencyMs;
|
||||
const regressionDelta = candidate.regressions - baseline.regressions;
|
||||
const regressions: string[] = [];
|
||||
if (aiqDrop > thresholds.maxAiqDrop) regressions.push(`AIQ dropped by ${aiqDrop.toFixed(3)}`);
|
||||
if (increaseRatio(costDelta, baseline.avgCostUsd) > thresholds.maxCostIncrease) {
|
||||
regressions.push(
|
||||
`average cost increased by ${increaseRatio(costDelta, baseline.avgCostUsd).toFixed(3)}`
|
||||
);
|
||||
}
|
||||
if (increaseRatio(latencyDelta, baseline.avgLatencyMs) > thresholds.maxLatencyIncrease) {
|
||||
regressions.push(
|
||||
`average latency increased by ${increaseRatio(latencyDelta, baseline.avgLatencyMs).toFixed(3)}`
|
||||
);
|
||||
}
|
||||
if (regressionDelta > thresholds.maxRegressionIncrease) {
|
||||
regressions.push(`regression count increased by ${regressionDelta}`);
|
||||
}
|
||||
return regressions;
|
||||
}
|
||||
|
||||
function comparePatches(
|
||||
runId: string,
|
||||
thresholds: PatchThresholds,
|
||||
failOnRegression: boolean,
|
||||
baseline: PatchSummary,
|
||||
candidate: PatchSummary
|
||||
): PatchComparison {
|
||||
const regressions = findRegressions(baseline, candidate, thresholds);
|
||||
const status = failOnRegression && regressions.length > 0 ? 1 : 0;
|
||||
const costDelta = candidate.avgCostUsd - baseline.avgCostUsd;
|
||||
const latencyDelta = candidate.avgLatencyMs - baseline.avgLatencyMs;
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-patch-comparison",
|
||||
generatedAt: new Date().toISOString(),
|
||||
runId,
|
||||
thresholds,
|
||||
baseline,
|
||||
candidate,
|
||||
delta: {
|
||||
aiq: candidate.aiq - baseline.aiq,
|
||||
avgCostUsd: costDelta,
|
||||
costIncreaseRatio: increaseRatio(costDelta, baseline.avgCostUsd),
|
||||
avgLatencyMs: latencyDelta,
|
||||
latencyIncreaseRatio: increaseRatio(latencyDelta, baseline.avgLatencyMs),
|
||||
regressions: candidate.regressions - baseline.regressions,
|
||||
},
|
||||
changedRecommendation: baseline.recommendedConfigId !== candidate.recommendedConfigId,
|
||||
regressions,
|
||||
result: {
|
||||
passed: regressions.length === 0,
|
||||
status,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatComparison(comparison: PatchComparison): string {
|
||||
return [
|
||||
"# Router Config Patch Comparison",
|
||||
"",
|
||||
`Passed: ${comparison.result.passed ? "yes" : "no"}`,
|
||||
`Changed recommendation: ${comparison.changedRecommendation ? "yes" : "no"}`,
|
||||
...(comparison.regressions.length > 0
|
||||
? ["", "## Regressions", "", ...comparison.regressions.map((item) => `- ${item}`)]
|
||||
: []),
|
||||
"",
|
||||
"| Side | Name | Objective | Recommended Config | AIQ | Avg Cost | Avg Latency | Regressions | Apply Policy |",
|
||||
"| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
|
||||
formatSummaryRow("Baseline", comparison.baseline),
|
||||
formatSummaryRow("Candidate", comparison.candidate),
|
||||
"",
|
||||
"| Delta | AIQ | Avg Cost | Avg Latency | Regressions |",
|
||||
"| --- | ---: | ---: | ---: | ---: |",
|
||||
`| Candidate - Baseline | ${comparison.delta.aiq.toFixed(3)} | $${comparison.delta.avgCostUsd.toFixed(6)} | ${comparison.delta.avgLatencyMs.toFixed(2)}ms | ${comparison.delta.regressions} |`,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatSummaryRow(side: string, summary: PatchSummary): string {
|
||||
return `| ${side} | ${summary.name} | ${summary.objective} | ${summary.recommendedConfigId} | ${summary.aiq.toFixed(3)} | $${summary.avgCostUsd.toFixed(6)} | ${summary.avgLatencyMs.toFixed(2)}ms | ${summary.regressions} | ${summary.applyPolicy} |`;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
if (process.argv.includes("--help") || process.argv.includes("-h")) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const baselineFile = requireArg("baseline");
|
||||
const candidateFile = requireArg("candidate");
|
||||
const baselineName = getArgValue("baseline-name") ?? "baseline";
|
||||
const candidateName = getArgValue("candidate-name") ?? "candidate";
|
||||
const runId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const thresholds: PatchThresholds = {
|
||||
maxAiqDrop: getNumberArg("max-aiq-drop", Number.POSITIVE_INFINITY),
|
||||
maxCostIncrease: getNumberArg("max-cost-increase", Number.POSITIVE_INFINITY),
|
||||
maxLatencyIncrease: getNumberArg("max-latency-increase", Number.POSITIVE_INFINITY),
|
||||
maxRegressionIncrease: getNumberArg("max-regression-increase", Number.POSITIVE_INFINITY),
|
||||
};
|
||||
const failOnRegression = process.argv.includes("--fail-on-regression");
|
||||
const baseline = summarizePatch(baselineName, baselineFile, readPatch(baselineFile));
|
||||
const candidate = summarizePatch(candidateName, candidateFile, readPatch(candidateFile));
|
||||
const comparison = comparePatches(runId, thresholds, failOnRegression, baseline, candidate);
|
||||
const markdown = formatComparison(comparison);
|
||||
const artifactDir = getArgValue("artifact-dir");
|
||||
const output = getArgValue("output");
|
||||
const jsonOutput = getArgValue("json-output");
|
||||
|
||||
if (artifactDir) {
|
||||
const runDir = path.resolve(artifactDir, runId);
|
||||
fs.mkdirSync(runDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(runDir, "patch-comparison.md"), markdown);
|
||||
fs.writeFileSync(
|
||||
path.join(runDir, "patch-comparison.json"),
|
||||
`${JSON.stringify(comparison, null, 2)}\n`
|
||||
);
|
||||
}
|
||||
if (output) {
|
||||
fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true });
|
||||
fs.writeFileSync(output, markdown);
|
||||
}
|
||||
if (jsonOutput) {
|
||||
fs.mkdirSync(path.dirname(path.resolve(jsonOutput)), { recursive: true });
|
||||
fs.writeFileSync(jsonOutput, `${JSON.stringify(comparison, null, 2)}\n`);
|
||||
}
|
||||
console.log(markdown);
|
||||
process.exit(comparison.result.status);
|
||||
}
|
||||
|
||||
main();
|
||||
439
scripts/router-eval/search.ts
Normal file
439
scripts/router-eval/search.ts
Normal file
@@ -0,0 +1,439 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import type { RouterConfigAggregate, RouterEvalArtifact } from "@/lib/routerEval/index.ts";
|
||||
|
||||
type Candidate = {
|
||||
name: string;
|
||||
path: string;
|
||||
};
|
||||
|
||||
type SearchResult = {
|
||||
candidateName: string;
|
||||
configId: string;
|
||||
runId: string;
|
||||
aiq: number;
|
||||
avgCostUsd: number;
|
||||
avgLatencyMs: number;
|
||||
regressions: number;
|
||||
artifactPath: string;
|
||||
};
|
||||
|
||||
type SearchObjective = "balanced" | "quality" | "cost" | "latency";
|
||||
|
||||
type SearchRecommendation = SearchResult & {
|
||||
objective: SearchObjective;
|
||||
rank: number;
|
||||
rationale: string;
|
||||
};
|
||||
|
||||
type RouterConfigSuggestion = {
|
||||
schemaVersion: 1;
|
||||
kind: "router-config-suggestion";
|
||||
generatedAt: string;
|
||||
objective: SearchObjective;
|
||||
recommendedConfigId: string;
|
||||
sourceRunId: string;
|
||||
sourceArtifactPath: string;
|
||||
evidence: {
|
||||
aiq: number;
|
||||
avgCostUsd: number;
|
||||
avgLatencyMs: number;
|
||||
regressions: number;
|
||||
};
|
||||
applyPolicy: "manual-review";
|
||||
rationale: string;
|
||||
};
|
||||
|
||||
type RouterConfigPatchArtifact = {
|
||||
schemaVersion: 1;
|
||||
kind: "router-config-patch";
|
||||
generatedAt: string;
|
||||
applyPolicy: "manual-review";
|
||||
source: {
|
||||
objective: SearchObjective;
|
||||
runId: string;
|
||||
artifactPath: string;
|
||||
};
|
||||
operations: Array<{
|
||||
op: "recommend-router-config";
|
||||
path: "/router/recommendedConfigId";
|
||||
value: string;
|
||||
evidence: RouterConfigSuggestion["evidence"];
|
||||
rationale: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const isBunRuntime = "Bun" in globalThis;
|
||||
|
||||
function runTypeScriptScript(args: string[]) {
|
||||
return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
return value && !value.startsWith("--") ? value : undefined;
|
||||
}
|
||||
|
||||
function getArgValues(name: string): string[] {
|
||||
const values: string[] = [];
|
||||
for (let index = 0; index < process.argv.length; index += 1) {
|
||||
if (process.argv[index] !== `--${name}`) continue;
|
||||
const value = process.argv[index + 1];
|
||||
if (value && !value.startsWith("--")) values.push(value);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
"Usage:",
|
||||
" npm run eval:router:search -- --baseline <baseline.ndjson>",
|
||||
" --candidate <name=path.ndjson> [--candidate <name=path.ndjson> ...]",
|
||||
" [--objective balanced|quality|cost|latency]",
|
||||
" [--artifact-dir <dir>] [--run-id <id>] [--max-aiq-drop <n>] [--max-cost-increase <n>]",
|
||||
"",
|
||||
"Ranks candidate corpora by router-eval AIQ while retaining comparison artifacts.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function requireArg(name: string): string {
|
||||
const value = getArgValue(name);
|
||||
if (!value) {
|
||||
console.error(`Missing required --${name}`);
|
||||
process.exit(2);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseCandidate(raw: string): Candidate {
|
||||
const splitAt = raw.indexOf("=");
|
||||
if (splitAt <= 0 || splitAt === raw.length - 1) {
|
||||
console.error(`Invalid --candidate value: ${raw}. Expected name=path.ndjson`);
|
||||
process.exit(2);
|
||||
}
|
||||
return {
|
||||
name: raw.slice(0, splitAt),
|
||||
path: raw.slice(splitAt + 1),
|
||||
};
|
||||
}
|
||||
|
||||
function ensureReadable(filePath: string, label: string): void {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.error(`[router-eval:search] ${label} missing: ${filePath}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
function readArtifact(filePath: string): RouterEvalArtifact {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8")) as RouterEvalArtifact;
|
||||
}
|
||||
|
||||
function parseObjective(value: string | undefined): SearchObjective {
|
||||
if (!value) return "balanced";
|
||||
if (value === "balanced" || value === "quality" || value === "cost" || value === "latency") {
|
||||
return value;
|
||||
}
|
||||
console.error(
|
||||
`Invalid --objective value: ${value}. Expected balanced, quality, cost, or latency.`
|
||||
);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
function compareConfigsByObjective(
|
||||
objective: SearchObjective,
|
||||
a: RouterConfigAggregate,
|
||||
b: RouterConfigAggregate
|
||||
): number {
|
||||
if (objective === "cost") {
|
||||
return a.avgCostUsd - b.avgCostUsd || b.aiq - a.aiq || a.avgLatencyMs - b.avgLatencyMs;
|
||||
}
|
||||
if (objective === "latency") {
|
||||
return a.avgLatencyMs - b.avgLatencyMs || b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd;
|
||||
}
|
||||
return b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd || a.avgLatencyMs - b.avgLatencyMs;
|
||||
}
|
||||
|
||||
function selectBestConfig(
|
||||
artifact: RouterEvalArtifact,
|
||||
objective: SearchObjective
|
||||
): RouterConfigAggregate | undefined {
|
||||
const configs =
|
||||
artifact.comparison?.candidate.configurations ??
|
||||
artifact.report?.configurations ??
|
||||
artifact.report?.top ??
|
||||
[];
|
||||
return [...configs].sort((a, b) => compareConfigsByObjective(objective, a, b))[0];
|
||||
}
|
||||
|
||||
function resultFromArtifact(
|
||||
candidateName: string,
|
||||
runId: string,
|
||||
artifactPath: string,
|
||||
objective: SearchObjective
|
||||
): SearchResult {
|
||||
const artifact = readArtifact(artifactPath);
|
||||
const best = selectBestConfig(artifact, objective);
|
||||
if (!best) {
|
||||
throw new Error(`No best candidate found in ${artifactPath}`);
|
||||
}
|
||||
return {
|
||||
candidateName,
|
||||
configId: best.configId,
|
||||
runId,
|
||||
aiq: best.aiq,
|
||||
avgCostUsd: best.avgCostUsd,
|
||||
avgLatencyMs: best.avgLatencyMs,
|
||||
regressions: artifact.comparison?.regressions.length ?? 0,
|
||||
artifactPath,
|
||||
};
|
||||
}
|
||||
|
||||
function formatSearch(results: SearchResult[]): string {
|
||||
const lines = [
|
||||
"# Router Eval Search",
|
||||
"",
|
||||
"| Rank | Candidate | AIQ | Avg Cost | Avg Latency | Regressions | Run |",
|
||||
"| ---: | --- | ---: | ---: | ---: | ---: | --- |",
|
||||
];
|
||||
results.forEach((result, index) => {
|
||||
lines.push(
|
||||
`| ${index + 1} | ${result.candidateName} | ${result.aiq.toFixed(3)} | $${result.avgCostUsd.toFixed(6)} | ${result.avgLatencyMs.toFixed(2)}ms | ${result.regressions} | ${result.runId} |`
|
||||
);
|
||||
});
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function compareByObjective(objective: SearchObjective, a: SearchResult, b: SearchResult): number {
|
||||
if (objective === "cost") {
|
||||
return (
|
||||
a.regressions - b.regressions ||
|
||||
a.avgCostUsd - b.avgCostUsd ||
|
||||
b.aiq - a.aiq ||
|
||||
a.avgLatencyMs - b.avgLatencyMs
|
||||
);
|
||||
}
|
||||
if (objective === "latency") {
|
||||
return (
|
||||
a.regressions - b.regressions ||
|
||||
a.avgLatencyMs - b.avgLatencyMs ||
|
||||
b.aiq - a.aiq ||
|
||||
a.avgCostUsd - b.avgCostUsd
|
||||
);
|
||||
}
|
||||
if (objective === "quality") {
|
||||
return (
|
||||
b.aiq - a.aiq ||
|
||||
a.regressions - b.regressions ||
|
||||
a.avgCostUsd - b.avgCostUsd ||
|
||||
a.avgLatencyMs - b.avgLatencyMs
|
||||
);
|
||||
}
|
||||
return (
|
||||
b.aiq - a.aiq ||
|
||||
a.regressions - b.regressions ||
|
||||
a.avgCostUsd - b.avgCostUsd ||
|
||||
a.avgLatencyMs - b.avgLatencyMs
|
||||
);
|
||||
}
|
||||
|
||||
function recommendationRationale(objective: SearchObjective, result: SearchResult): string {
|
||||
if (objective === "cost") {
|
||||
return `${result.candidateName} has the best cost-first rank with ${result.regressions} regressions and $${result.avgCostUsd.toFixed(6)} average cost.`;
|
||||
}
|
||||
if (objective === "latency") {
|
||||
return `${result.candidateName} has the best latency-first rank with ${result.regressions} regressions and ${result.avgLatencyMs.toFixed(2)}ms average latency.`;
|
||||
}
|
||||
if (objective === "quality") {
|
||||
return `${result.candidateName} has the best quality-first rank with ${result.aiq.toFixed(3)} AIQ.`;
|
||||
}
|
||||
return `${result.candidateName} has the best balanced rank with ${result.aiq.toFixed(3)} AIQ, ${result.regressions} regressions, $${result.avgCostUsd.toFixed(6)} average cost, and ${result.avgLatencyMs.toFixed(2)}ms average latency.`;
|
||||
}
|
||||
|
||||
function createRecommendation(
|
||||
objective: SearchObjective,
|
||||
results: SearchResult[]
|
||||
): SearchRecommendation {
|
||||
const winner = results[0];
|
||||
if (!winner) {
|
||||
throw new Error("Cannot create a recommendation without search results");
|
||||
}
|
||||
return {
|
||||
...winner,
|
||||
objective,
|
||||
rank: 1,
|
||||
rationale: recommendationRationale(objective, winner),
|
||||
};
|
||||
}
|
||||
|
||||
function createConfigSuggestion(
|
||||
generatedAt: string,
|
||||
recommendation: SearchRecommendation
|
||||
): RouterConfigSuggestion {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-suggestion",
|
||||
generatedAt,
|
||||
objective: recommendation.objective,
|
||||
recommendedConfigId: recommendation.configId,
|
||||
sourceRunId: recommendation.runId,
|
||||
sourceArtifactPath: recommendation.artifactPath,
|
||||
evidence: {
|
||||
aiq: recommendation.aiq,
|
||||
avgCostUsd: recommendation.avgCostUsd,
|
||||
avgLatencyMs: recommendation.avgLatencyMs,
|
||||
regressions: recommendation.regressions,
|
||||
},
|
||||
applyPolicy: "manual-review",
|
||||
rationale: recommendation.rationale,
|
||||
};
|
||||
}
|
||||
|
||||
function createConfigPatch(suggestion: RouterConfigSuggestion): RouterConfigPatchArtifact {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-patch",
|
||||
generatedAt: suggestion.generatedAt,
|
||||
applyPolicy: "manual-review",
|
||||
source: {
|
||||
objective: suggestion.objective,
|
||||
runId: suggestion.sourceRunId,
|
||||
artifactPath: suggestion.sourceArtifactPath,
|
||||
},
|
||||
operations: [
|
||||
{
|
||||
op: "recommend-router-config",
|
||||
path: "/router/recommendedConfigId",
|
||||
value: suggestion.recommendedConfigId,
|
||||
evidence: suggestion.evidence,
|
||||
rationale: suggestion.rationale,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function formatPatchOperations(patch: RouterConfigPatchArtifact): string {
|
||||
const lines = [
|
||||
"## Patch Operations",
|
||||
"",
|
||||
"| Op | Path | Value | AIQ | Avg Cost | Avg Latency | Regressions | Apply Policy |",
|
||||
"| --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
|
||||
];
|
||||
for (const operation of patch.operations) {
|
||||
lines.push(
|
||||
`| ${operation.op} | ${operation.path} | ${operation.value} | ${operation.evidence.aiq.toFixed(3)} | $${operation.evidence.avgCostUsd.toFixed(6)} | ${operation.evidence.avgLatencyMs.toFixed(2)}ms | ${operation.evidence.regressions} | ${patch.applyPolicy} |`
|
||||
);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
if (process.argv.includes("--help") || process.argv.includes("-h")) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const baseline = requireArg("baseline");
|
||||
ensureReadable(baseline, "baseline corpus");
|
||||
const candidates = getArgValues("candidate").map(parseCandidate);
|
||||
if (candidates.length === 0) {
|
||||
console.error("At least one --candidate <name=path> is required");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const artifactDir = getArgValue("artifact-dir") ?? "artifacts/router-eval/search";
|
||||
const searchId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-");
|
||||
const objective = parseObjective(getArgValue("objective"));
|
||||
const maxAiqDrop = getArgValue("max-aiq-drop") ?? "1";
|
||||
const maxCostIncrease = getArgValue("max-cost-increase") ?? "0.05";
|
||||
const searchDir = path.resolve(artifactDir, searchId);
|
||||
fs.mkdirSync(searchDir, { recursive: true });
|
||||
|
||||
const results: SearchResult[] = [];
|
||||
for (const candidate of candidates) {
|
||||
ensureReadable(candidate.path, `${candidate.name} corpus`);
|
||||
const runId = `${searchId}-${candidate.name}`;
|
||||
const result = runTypeScriptScript([
|
||||
"scripts/router-eval/compare.ts",
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate.path,
|
||||
"--baseline-name",
|
||||
"baseline",
|
||||
"--candidate-name",
|
||||
candidate.name,
|
||||
"--artifact-dir",
|
||||
searchDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
"--max-aiq-drop",
|
||||
maxAiqDrop,
|
||||
"--max-cost-increase",
|
||||
maxCostIncrease,
|
||||
]);
|
||||
if (result.stdout) process.stdout.write(result.stdout);
|
||||
if (result.stderr) process.stderr.write(result.stderr);
|
||||
if (result.error) {
|
||||
console.error(
|
||||
`[router-eval:search] failed to launch ${candidate.name}: ${result.error.message}`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
console.error(
|
||||
`[router-eval:search] comparison failed for ${candidate.name} with exit code ${result.status ?? 1}`
|
||||
);
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
const artifactPath = path.join(searchDir, runId, "router-eval.json");
|
||||
results.push(resultFromArtifact(candidate.name, runId, artifactPath, objective));
|
||||
}
|
||||
|
||||
results.sort((a, b) => compareByObjective(objective, a, b));
|
||||
|
||||
const recommendation = createRecommendation(objective, results);
|
||||
const generatedAt = new Date().toISOString();
|
||||
const suggestion = createConfigSuggestion(generatedAt, recommendation);
|
||||
const patch = createConfigPatch(suggestion);
|
||||
const markdown = `${formatSearch(results)}## Recommendation\n\n${recommendation.rationale}\n\n${formatPatchOperations(patch)}`;
|
||||
const summary = {
|
||||
schemaVersion: 1,
|
||||
kind: "router-eval-search",
|
||||
generatedAt,
|
||||
baseline: path.resolve(baseline),
|
||||
objective,
|
||||
recommendation,
|
||||
suggestion,
|
||||
patch,
|
||||
results,
|
||||
};
|
||||
fs.writeFileSync(path.join(searchDir, "search.md"), markdown);
|
||||
fs.writeFileSync(path.join(searchDir, "search.json"), `${JSON.stringify(summary, null, 2)}\n`);
|
||||
fs.writeFileSync(
|
||||
path.join(searchDir, "recommendation.json"),
|
||||
`${JSON.stringify(recommendation, null, 2)}\n`
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(searchDir, "suggestion.json"),
|
||||
`${JSON.stringify(suggestion, null, 2)}\n`
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(searchDir, "router-config.patch.json"),
|
||||
`${JSON.stringify(patch, null, 2)}\n`
|
||||
);
|
||||
console.log(markdown);
|
||||
console.log(`[router-eval:search] artifacts: ${searchDir}`);
|
||||
}
|
||||
|
||||
main();
|
||||
198
scripts/router-eval/trends.ts
Normal file
198
scripts/router-eval/trends.ts
Normal file
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import type {
|
||||
RouterConfigAggregate,
|
||||
RouterEvalArtifact,
|
||||
RouterEvalArtifactMetadata,
|
||||
RouterEvalComparison,
|
||||
RouterEvalReport,
|
||||
} from "@/lib/routerEval/index.ts";
|
||||
|
||||
type TrendRow = {
|
||||
runId: string;
|
||||
generatedAt: string;
|
||||
kind: RouterEvalArtifact["kind"];
|
||||
bestConfig: string;
|
||||
aiq: number;
|
||||
avgCostUsd: number;
|
||||
avgLatencyMs: number;
|
||||
regressions: number;
|
||||
source: string;
|
||||
window: string;
|
||||
};
|
||||
|
||||
function getArgValue(name: string): string | undefined {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
if (index < 0) return undefined;
|
||||
const value = process.argv[index + 1];
|
||||
return value && !value.startsWith("--") ? value : undefined;
|
||||
}
|
||||
|
||||
function usage(): string {
|
||||
return [
|
||||
"Usage:",
|
||||
" npm run eval:router:trends -- --artifact-dir <dir> [--limit <n>] [--dashboard]",
|
||||
"",
|
||||
"Reads retained router-eval JSON artifacts and prints a markdown trend table or dashboard.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function readJson(filePath: string): RouterEvalArtifact | null {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8")) as RouterEvalArtifact;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function bestFromReport(report: RouterEvalReport): RouterConfigAggregate | undefined {
|
||||
return report.top[0];
|
||||
}
|
||||
|
||||
function bestFromComparison(comparison: RouterEvalComparison): RouterConfigAggregate | undefined {
|
||||
return comparison.candidate.top[0];
|
||||
}
|
||||
|
||||
function toTrendRow(runId: string, artifact: RouterEvalArtifact): TrendRow | null {
|
||||
const best = artifact.comparison
|
||||
? bestFromComparison(artifact.comparison)
|
||||
: artifact.report
|
||||
? bestFromReport(artifact.report)
|
||||
: undefined;
|
||||
|
||||
if (!best) return null;
|
||||
|
||||
return {
|
||||
runId,
|
||||
generatedAt: artifact.generatedAt,
|
||||
kind: artifact.kind,
|
||||
bestConfig: best.configId,
|
||||
aiq: best.aiq,
|
||||
avgCostUsd: best.avgCostUsd,
|
||||
avgLatencyMs: best.avgLatencyMs,
|
||||
regressions: artifact.comparison?.regressions.length ?? 0,
|
||||
source: artifact.metadata?.candidate?.source ?? "unknown",
|
||||
window: formatWindow(artifact.metadata?.window),
|
||||
};
|
||||
}
|
||||
|
||||
function formatWindow(window: RouterEvalArtifactMetadata["window"]): string {
|
||||
if (!window || typeof window !== "object") return "all";
|
||||
const parts: string[] = [];
|
||||
if ("since" in window && typeof window.since === "string") parts.push(`since ${window.since}`);
|
||||
if ("limit" in window && typeof window.limit === "number") parts.push(`limit ${window.limit}`);
|
||||
return parts.length > 0 ? parts.join(", ") : "all";
|
||||
}
|
||||
|
||||
function collectTrendRows(artifactDir: string): TrendRow[] {
|
||||
if (!fs.existsSync(artifactDir)) return [];
|
||||
|
||||
const rows: TrendRow[] = [];
|
||||
for (const entry of fs.readdirSync(artifactDir, { withFileTypes: true })) {
|
||||
const runId = entry.name;
|
||||
const jsonPath = entry.isDirectory()
|
||||
? path.join(artifactDir, runId, "router-eval.json")
|
||||
: entry.isFile() && entry.name.endsWith(".json")
|
||||
? path.join(artifactDir, entry.name)
|
||||
: "";
|
||||
if (!jsonPath) continue;
|
||||
|
||||
const artifact = readJson(jsonPath);
|
||||
if (!artifact || artifact.schemaVersion !== 1) continue;
|
||||
const row = toTrendRow(runId.replace(/\.json$/, ""), artifact);
|
||||
if (row) rows.push(row);
|
||||
}
|
||||
|
||||
return rows.sort((a, b) => a.generatedAt.localeCompare(b.generatedAt));
|
||||
}
|
||||
|
||||
function formatTrend(rows: TrendRow[], limit: number): string {
|
||||
const limited = rows.slice(-limit);
|
||||
const lines = [
|
||||
"# Router Eval Trends",
|
||||
"",
|
||||
"| Run | Kind | Source | Window | Best Config | AIQ | Avg Cost | Avg Latency | Regressions |",
|
||||
"| --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: |",
|
||||
];
|
||||
|
||||
for (const row of limited) {
|
||||
lines.push(
|
||||
`| ${row.runId} | ${row.kind} | ${row.source} | ${row.window} | ${row.bestConfig} | ${row.aiq.toFixed(3)} | $${row.avgCostUsd.toFixed(6)} | ${row.avgLatencyMs.toFixed(2)}ms | ${row.regressions} |`
|
||||
);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function formatDelta(value: number): string {
|
||||
if (value > 0) return `+${value.toFixed(3)}`;
|
||||
return value.toFixed(3);
|
||||
}
|
||||
|
||||
function average(values: number[]): number {
|
||||
if (values.length === 0) return 0;
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function formatDashboard(rows: TrendRow[], limit: number): string {
|
||||
const limited = rows.slice(-limit);
|
||||
const latest = limited[limited.length - 1];
|
||||
const previous = limited[limited.length - 2];
|
||||
const aiqDelta = latest && previous ? latest.aiq - previous.aiq : 0;
|
||||
const latencyDelta = latest && previous ? latest.avgLatencyMs - previous.avgLatencyMs : 0;
|
||||
const costDelta = latest && previous ? latest.avgCostUsd - previous.avgCostUsd : 0;
|
||||
const regressions = limited.reduce((sum, row) => sum + row.regressions, 0);
|
||||
const lines = [
|
||||
"# Router Eval Dashboard",
|
||||
"",
|
||||
`Runs: ${limited.length}`,
|
||||
`Latest: ${latest?.runId ?? "n/a"}`,
|
||||
`Best config: ${latest?.bestConfig ?? "n/a"}`,
|
||||
`AIQ: ${latest ? latest.aiq.toFixed(3) : "0.000"} (${formatDelta(aiqDelta)})`,
|
||||
`Avg latency: ${latest ? latest.avgLatencyMs.toFixed(2) : "0.00"}ms (${formatDelta(latencyDelta)}ms)`,
|
||||
`Avg cost: $${latest ? latest.avgCostUsd.toFixed(6) : "0.000000"} (${formatDelta(costDelta)})`,
|
||||
`Window: ${latest?.window ?? "all"}`,
|
||||
`Source: ${latest?.source ?? "unknown"}`,
|
||||
`Regression count: ${regressions}`,
|
||||
"",
|
||||
"## Rolling Averages",
|
||||
"",
|
||||
`AIQ: ${average(limited.map((row) => row.aiq)).toFixed(3)}`,
|
||||
`Latency: ${average(limited.map((row) => row.avgLatencyMs)).toFixed(2)}ms`,
|
||||
`Cost: $${average(limited.map((row) => row.avgCostUsd)).toFixed(6)}`,
|
||||
];
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
if (process.argv.includes("--help") || process.argv.includes("-h")) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const artifactDir = getArgValue("artifact-dir");
|
||||
if (!artifactDir) {
|
||||
console.error("Missing required --artifact-dir");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const limit = Number.parseInt(getArgValue("limit") ?? "20", 10);
|
||||
const rows = collectTrendRows(path.resolve(artifactDir));
|
||||
if (rows.length === 0) {
|
||||
console.error(`No router-eval artifacts found in ${artifactDir}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const boundedLimit = Number.isFinite(limit) && limit > 0 ? limit : 20;
|
||||
if (process.argv.includes("--dashboard")) {
|
||||
console.log(formatDashboard(rows, boundedLimit));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(formatTrend(rows, boundedLimit));
|
||||
}
|
||||
|
||||
main();
|
||||
428
src/lib/routerEval/index.ts
Normal file
428
src/lib/routerEval/index.ts
Normal file
@@ -0,0 +1,428 @@
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
export type RouterObservation = {
|
||||
sampleId: string;
|
||||
routeInput: JsonRecord;
|
||||
configId: string;
|
||||
selectedModel: string | null;
|
||||
expectedModel: string | null;
|
||||
latencyMs: number;
|
||||
costUsd: number;
|
||||
success: boolean;
|
||||
timestamp: string;
|
||||
metadata: JsonRecord;
|
||||
};
|
||||
|
||||
export type RouterObservationInput = {
|
||||
sampleId?: unknown;
|
||||
id?: unknown;
|
||||
routeInput?: unknown;
|
||||
configId?: unknown;
|
||||
selectedModel?: unknown;
|
||||
model?: unknown;
|
||||
expectedModel?: unknown;
|
||||
requestedModel?: unknown;
|
||||
latencyMs?: unknown;
|
||||
latency?: unknown;
|
||||
durationMs?: unknown;
|
||||
duration?: unknown;
|
||||
costUsd?: unknown;
|
||||
cost?: unknown;
|
||||
success?: unknown;
|
||||
status?: unknown;
|
||||
error?: unknown;
|
||||
timestamp?: unknown;
|
||||
tokens?: {
|
||||
input?: unknown;
|
||||
output?: unknown;
|
||||
};
|
||||
metadata?: unknown;
|
||||
};
|
||||
|
||||
export type RouterConfigAggregate = {
|
||||
configId: string;
|
||||
samples: number;
|
||||
successRate: number;
|
||||
avgLatencyMs: number;
|
||||
p50LatencyMs: number;
|
||||
p95LatencyMs: number;
|
||||
avgCostUsd: number;
|
||||
aiq: number;
|
||||
};
|
||||
|
||||
export type RouterEvalReport = {
|
||||
evaluatedAt: string;
|
||||
summary: {
|
||||
totalSamples: number;
|
||||
validSamples: number;
|
||||
droppedSamples: number;
|
||||
uniqueConfigs: number;
|
||||
};
|
||||
configurations: RouterConfigAggregate[];
|
||||
frontier: RouterConfigAggregate[];
|
||||
top: RouterConfigAggregate[];
|
||||
};
|
||||
|
||||
export type RouterEvalComparison = {
|
||||
baseline: RouterEvalReport;
|
||||
candidate: RouterEvalReport;
|
||||
delta: {
|
||||
aiq: number;
|
||||
costUsd: number;
|
||||
};
|
||||
regressions: string[];
|
||||
};
|
||||
|
||||
export type RouterEvalArtifact = {
|
||||
schemaVersion: 1;
|
||||
kind: "router-eval-report" | "router-eval-comparison";
|
||||
generatedAt: string;
|
||||
metadata?: RouterEvalArtifactMetadata;
|
||||
report?: RouterEvalReport;
|
||||
comparison?: RouterEvalComparison;
|
||||
};
|
||||
|
||||
export type RouterEvalArtifactMetadata = {
|
||||
candidate?: {
|
||||
source: string;
|
||||
path?: string;
|
||||
dbSource?: string;
|
||||
};
|
||||
baseline?: {
|
||||
source: string;
|
||||
path?: string;
|
||||
dbSource?: string;
|
||||
};
|
||||
window?: {
|
||||
since?: string;
|
||||
limit?: number;
|
||||
};
|
||||
thresholds?: {
|
||||
maxAiqDrop: number;
|
||||
maxCostIncrease: number;
|
||||
};
|
||||
outputs?: {
|
||||
markdown?: string;
|
||||
json?: string;
|
||||
corpus?: string;
|
||||
};
|
||||
};
|
||||
|
||||
function asRecord(value: unknown): JsonRecord {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {};
|
||||
}
|
||||
|
||||
function asString(value: unknown, fallback = ""): string {
|
||||
return typeof value === "string" && value.trim().length > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function asNumber(value: unknown, fallback = 0): number {
|
||||
if (typeof value === "number" && Number.isFinite(value)) return value;
|
||||
if (typeof value === "string") {
|
||||
const parsed = Number.parseFloat(value);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function asBoolean(value: unknown, fallback = false): boolean {
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "string") {
|
||||
if (value.toLowerCase() === "true") return true;
|
||||
if (value.toLowerCase() === "false") return false;
|
||||
}
|
||||
if (typeof value === "number") return value !== 0;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function percentile(sortedValues: number[], p: number): number {
|
||||
if (sortedValues.length === 0) return 0;
|
||||
if (sortedValues.length === 1) return sortedValues[0];
|
||||
const bounded = Math.min(1, Math.max(0, p));
|
||||
const index = Math.floor((sortedValues.length - 1) * bounded);
|
||||
return sortedValues[index] ?? 0;
|
||||
}
|
||||
|
||||
function computeAiq(successRate: number, avgLatencyMs: number, avgCostUsd: number): number {
|
||||
const latencyPenalty = avgLatencyMs / 1_000;
|
||||
const costPenalty = avgCostUsd * 1_000;
|
||||
return Number((successRate * 100 - latencyPenalty - costPenalty).toFixed(3));
|
||||
}
|
||||
|
||||
function aggregateValues(values: number[]) {
|
||||
if (values.length === 0) return { avg: 0, p50: 0, p95: 0 };
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const sum = sorted.reduce((acc, value) => acc + value, 0);
|
||||
return {
|
||||
avg: sum / sorted.length,
|
||||
p50: percentile(sorted, 0.5),
|
||||
p95: percentile(sorted, 0.95),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveObservationSampleId(value: RouterObservationInput): string {
|
||||
return asString(value.sampleId ?? value.id, "").trim();
|
||||
}
|
||||
|
||||
function resolveObservationModelFields(value: RouterObservationInput): {
|
||||
selectedModel: string | null;
|
||||
expectedModel: string | null;
|
||||
} {
|
||||
const selectedModel = asString(value.selectedModel ?? value.model);
|
||||
const expectedModel = asString(value.expectedModel ?? value.requestedModel, null);
|
||||
return { selectedModel: selectedModel || null, expectedModel };
|
||||
}
|
||||
|
||||
function resolveObservationLatencyMs(value: RouterObservationInput): number {
|
||||
return asNumber(value.latencyMs ?? value.latency ?? value.durationMs ?? value.duration, 0);
|
||||
}
|
||||
|
||||
function resolveObservationCostUsd(value: RouterObservationInput): number {
|
||||
const explicitCost = asNumber(value.costUsd ?? value.cost, NaN);
|
||||
if (!Number.isNaN(explicitCost)) return explicitCost;
|
||||
|
||||
const promptTokens = asNumber(value.tokens?.input);
|
||||
const completionTokens = asNumber(value.tokens?.output);
|
||||
return Number(((promptTokens + completionTokens) * 0.000001).toFixed(6));
|
||||
}
|
||||
|
||||
function resolveObservationSuccess(value: RouterObservationInput): boolean {
|
||||
const status = asNumber(value.status, 500);
|
||||
const statusIndicatesSuccess = status >= 200 && status < 400;
|
||||
return asBoolean(value.success, statusIndicatesSuccess && !value.error);
|
||||
}
|
||||
|
||||
export function toRouterObservation(input: unknown): RouterObservation | null {
|
||||
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
|
||||
|
||||
const value = input as RouterObservationInput;
|
||||
const sampleId = resolveObservationSampleId(value);
|
||||
if (!sampleId) return null;
|
||||
|
||||
const { selectedModel, expectedModel } = resolveObservationModelFields(value);
|
||||
|
||||
return {
|
||||
sampleId,
|
||||
routeInput: asRecord(value.routeInput, {}),
|
||||
configId: asString(value.configId, "default"),
|
||||
selectedModel,
|
||||
expectedModel,
|
||||
latencyMs: resolveObservationLatencyMs(value),
|
||||
costUsd: resolveObservationCostUsd(value),
|
||||
success: resolveObservationSuccess(value),
|
||||
timestamp: asString(value.timestamp, new Date(0).toISOString()),
|
||||
metadata: asRecord(value.metadata),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeRouterObservations(observations: RouterObservation[]) {
|
||||
const byConfig = new Map<string, RouterObservation[]>();
|
||||
for (const obs of observations) {
|
||||
if (!byConfig.has(obs.configId)) byConfig.set(obs.configId, []);
|
||||
byConfig.get(obs.configId)?.push(obs);
|
||||
}
|
||||
|
||||
const configurations: RouterConfigAggregate[] = [];
|
||||
for (const [configId, rows] of byConfig) {
|
||||
const latencies = rows.map((row) => Math.max(0, row.latencyMs));
|
||||
const costs = rows.map((row) => Math.max(0, row.costUsd));
|
||||
const latencyStats = aggregateValues(latencies);
|
||||
const costStats = aggregateValues(costs);
|
||||
const successCount = rows.filter((row) => row.success).length;
|
||||
const successRate = rows.length === 0 ? 0 : successCount / rows.length;
|
||||
configurations.push({
|
||||
configId,
|
||||
samples: rows.length,
|
||||
successRate,
|
||||
avgLatencyMs: latencyStats.avg,
|
||||
p50LatencyMs: latencyStats.p50,
|
||||
p95LatencyMs: latencyStats.p95,
|
||||
avgCostUsd: costStats.avg,
|
||||
aiq: computeAiq(successRate, latencyStats.avg, costStats.avg),
|
||||
});
|
||||
}
|
||||
|
||||
const top = [...configurations].sort(
|
||||
(a, b) =>
|
||||
b.aiq - a.aiq ||
|
||||
b.successRate - a.successRate ||
|
||||
a.avgLatencyMs - b.avgLatencyMs ||
|
||||
a.avgCostUsd - b.avgCostUsd
|
||||
);
|
||||
const frontier = computeParetoFrontier(configurations);
|
||||
|
||||
return {
|
||||
configurations: configurations.sort((a, b) => b.aiq - a.aiq),
|
||||
frontier,
|
||||
top,
|
||||
};
|
||||
}
|
||||
|
||||
export function runRouterEval(observations: RouterObservation[]): RouterEvalReport {
|
||||
const valid = observations.filter((value) => Boolean(value.selectedModel));
|
||||
const droppedSamples = observations.length - valid.length;
|
||||
const { configurations, frontier, top } = summarizeRouterObservations(valid);
|
||||
|
||||
return {
|
||||
evaluatedAt: new Date().toISOString(),
|
||||
summary: {
|
||||
totalSamples: observations.length,
|
||||
validSamples: valid.length,
|
||||
droppedSamples,
|
||||
uniqueConfigs: configurations.length,
|
||||
},
|
||||
configurations,
|
||||
frontier,
|
||||
top,
|
||||
};
|
||||
}
|
||||
|
||||
export function computeParetoFrontier(configs: RouterConfigAggregate[]) {
|
||||
const ordered = [...configs].sort(
|
||||
(a, b) => a.avgCostUsd - b.avgCostUsd || a.avgLatencyMs - b.avgLatencyMs
|
||||
);
|
||||
const frontier: RouterConfigAggregate[] = [];
|
||||
|
||||
for (const config of ordered) {
|
||||
const dominated = frontier.some(
|
||||
(candidate) =>
|
||||
candidate.aiq >= config.aiq &&
|
||||
candidate.avgCostUsd <= config.avgCostUsd &&
|
||||
candidate.avgLatencyMs <= config.avgLatencyMs
|
||||
);
|
||||
if (!dominated) frontier.push(config);
|
||||
}
|
||||
|
||||
return frontier.sort((a, b) => b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd);
|
||||
}
|
||||
|
||||
export function compareRouterEvalRuns(
|
||||
baseline: RouterEvalReport,
|
||||
candidate: RouterEvalReport,
|
||||
thresholds: {
|
||||
aiqDrop: number;
|
||||
relativeCostIncrease: number;
|
||||
} = { aiqDrop: 0, relativeCostIncrease: 0 }
|
||||
) {
|
||||
const baselineBest = baseline.top[0];
|
||||
const candidateBest = candidate.top[0];
|
||||
const regressions: string[] = [];
|
||||
if (!baselineBest || !candidateBest) {
|
||||
return { baseline, candidate, delta: { aiq: 0, costUsd: 0 }, regressions };
|
||||
}
|
||||
|
||||
const aiq = candidateBest.aiq - baselineBest.aiq;
|
||||
const baselineCost = baselineBest.avgCostUsd;
|
||||
const cost = candidateBest.avgCostUsd;
|
||||
const costIncrease =
|
||||
baselineCost === 0 ? (cost > 0 ? Infinity : 0) : (cost - baselineCost) / baselineCost;
|
||||
|
||||
if (aiq < -Math.abs(thresholds.aiqDrop)) {
|
||||
regressions.push(`AIQ dropped by ${(-aiq).toFixed(3)} below threshold`);
|
||||
}
|
||||
|
||||
if (costIncrease > thresholds.relativeCostIncrease) {
|
||||
regressions.push(`cost increased by ${(costIncrease * 100).toFixed(2)}% above threshold`);
|
||||
}
|
||||
|
||||
return {
|
||||
baseline,
|
||||
candidate,
|
||||
delta: {
|
||||
aiq,
|
||||
costUsd: costIncrease * baselineCost,
|
||||
},
|
||||
regressions,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatRouterEvalReport(report: RouterEvalReport): string {
|
||||
const best = report.top[0];
|
||||
const lines = [
|
||||
"# Router Eval Report",
|
||||
`Generated: ${report.evaluatedAt}`,
|
||||
`Samples: ${report.summary.validSamples}/${report.summary.totalSamples} valid, ${report.summary.uniqueConfigs} config(s)`,
|
||||
"",
|
||||
"## Top Configurations",
|
||||
"",
|
||||
"| Config | Samples | AIQ | Success | Avg Latency | p50 | p95 | Avg Cost |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
];
|
||||
|
||||
for (const config of report.top.slice(0, 10)) {
|
||||
lines.push(
|
||||
`| ${config.configId} | ${config.samples} | ${config.aiq.toFixed(3)} | ${(config.successRate * 100).toFixed(2)}% | ${config.avgLatencyMs.toFixed(2)}ms | ${config.p50LatencyMs.toFixed(2)}ms | ${config.p95LatencyMs.toFixed(2)}ms | $${config.avgCostUsd.toFixed(6)} |`
|
||||
);
|
||||
}
|
||||
|
||||
if (best) {
|
||||
lines.push("");
|
||||
lines.push(`Best: **${best.configId}** (AIQ ${best.aiq.toFixed(3)})`);
|
||||
}
|
||||
|
||||
if (report.frontier.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Pareto Frontier");
|
||||
lines.push("");
|
||||
lines.push("| Config | AIQ | Cost | Latency |");
|
||||
lines.push("| --- | ---: | ---: | ---: |");
|
||||
for (const config of report.frontier) {
|
||||
lines.push(
|
||||
`| ${config.configId} | ${config.aiq.toFixed(3)} | $${config.avgCostUsd.toFixed(6)} | ${config.avgLatencyMs.toFixed(2)}ms |`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
export function formatRouterEvalComparison(comparison: RouterEvalComparison): string {
|
||||
const bestCandidate = comparison.candidate.top[0];
|
||||
const bestBaseline = comparison.baseline.top[0];
|
||||
const passes = comparison.regressions.length === 0;
|
||||
const lines = [
|
||||
"# Router Eval Comparison",
|
||||
`Passed: ${passes ? "✅" : "❌"}`,
|
||||
`AIQ delta: ${comparison.delta.aiq.toFixed(3)}`,
|
||||
`Cost delta: $${comparison.delta.costUsd.toFixed(6)}`,
|
||||
"",
|
||||
];
|
||||
|
||||
if (bestBaseline && bestCandidate) {
|
||||
lines.push(`Baseline best: ${bestBaseline.configId} (AIQ ${bestBaseline.aiq.toFixed(3)})`);
|
||||
lines.push(`Candidate best: ${bestCandidate.configId} (AIQ ${bestCandidate.aiq.toFixed(3)})`);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (comparison.regressions.length > 0) {
|
||||
lines.push("## Regressions");
|
||||
for (const reason of comparison.regressions) {
|
||||
lines.push(`- ${reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
export function createRouterEvalArtifact(
|
||||
value: RouterEvalReport | RouterEvalComparison,
|
||||
metadata?: RouterEvalArtifactMetadata
|
||||
): RouterEvalArtifact {
|
||||
if ("candidate" in value && "baseline" in value) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-eval-comparison",
|
||||
generatedAt: value.candidate.evaluatedAt,
|
||||
metadata,
|
||||
comparison: value,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-eval-report",
|
||||
generatedAt: value.evaluatedAt,
|
||||
metadata,
|
||||
report: value,
|
||||
};
|
||||
}
|
||||
2
tests/fixtures/router-eval/baseline.ndjson
vendored
Normal file
2
tests/fixtures/router-eval/baseline.ndjson
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
{"sampleId":"baseline-1","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":120,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:00:00.000Z"}
|
||||
{"sampleId":"baseline-2","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":130,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:01:00.000Z"}
|
||||
2
tests/fixtures/router-eval/candidate.ndjson
vendored
Normal file
2
tests/fixtures/router-eval/candidate.ndjson
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
{"sampleId":"candidate-1","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":110,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:00:00.000Z"}
|
||||
{"sampleId":"candidate-2","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":125,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:01:00.000Z"}
|
||||
484
tests/unit/router-eval-check.test.ts
Normal file
484
tests/unit/router-eval-check.test.ts
Normal file
@@ -0,0 +1,484 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const checkScript = "scripts/check/check-router-eval-regression.ts";
|
||||
|
||||
function writePatch(
|
||||
file: string,
|
||||
configId: string,
|
||||
aiq: number,
|
||||
costUsd: number,
|
||||
latencyMs: number
|
||||
) {
|
||||
writeFileSync(
|
||||
file,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-patch",
|
||||
generatedAt: "2026-07-03T00:00:00.000Z",
|
||||
applyPolicy: "manual-review",
|
||||
source: {
|
||||
objective: "balanced",
|
||||
runId: `${configId}-run`,
|
||||
artifactPath: `/tmp/${configId}/router-eval.json`,
|
||||
},
|
||||
operations: [
|
||||
{
|
||||
op: "recommend-router-config",
|
||||
path: "/router/recommendedConfigId",
|
||||
value: configId,
|
||||
evidence: {
|
||||
aiq,
|
||||
avgCostUsd: costUsd,
|
||||
avgLatencyMs: latencyMs,
|
||||
regressions: 0,
|
||||
},
|
||||
rationale: `${configId} wins`,
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
);
|
||||
}
|
||||
|
||||
test("router eval check writes artifacts and passes non-regressing corpora", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-check-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const markdown = join(dir, "router-eval.md");
|
||||
const json = join(dir, "router-eval.json");
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--output",
|
||||
markdown,
|
||||
"--json-output",
|
||||
json,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok(readFileSync(markdown, "utf8").includes("Router Eval Comparison"));
|
||||
const artifact = JSON.parse(readFileSync(json, "utf8")) as Record<string, unknown>;
|
||||
assert.equal(artifact.kind, "router-eval-comparison");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval check can include patch compare as a retained gate", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-gate-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const baselinePatch = join(dir, "baseline.patch.json");
|
||||
const candidatePatch = join(dir, "candidate.patch.json");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "patch-gate-001";
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writePatch(baselinePatch, "balanced-v1", 92, 0.005, 150);
|
||||
writePatch(candidatePatch, "balanced-v2", 94, 0.004, 120);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--baseline-patch",
|
||||
baselinePatch,
|
||||
"--candidate-patch",
|
||||
candidatePatch,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "patch-comparison.md"), "utf8").includes(
|
||||
"Router Config Patch Comparison"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "patch-comparison.json"), "utf8").includes(
|
||||
"router-config-patch-comparison"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "inputs", "baseline.patch.json"), "utf8").includes(
|
||||
"router-config-patch"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "inputs", "candidate.patch.json"), "utf8").includes(
|
||||
"router-config-patch"
|
||||
)
|
||||
);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "manifest.json"), "utf8")
|
||||
) as {
|
||||
inputs?: { baselinePatch?: string; candidatePatch?: string };
|
||||
outputs?: { patchMarkdown?: string; patchJson?: string };
|
||||
thresholds?: { patch?: { maxLatencyIncrease?: number } };
|
||||
};
|
||||
assert.equal(manifest.inputs?.baselinePatch, "inputs/baseline.patch.json");
|
||||
assert.equal(manifest.inputs?.candidatePatch, "inputs/candidate.patch.json");
|
||||
assert.equal(manifest.outputs?.patchMarkdown, "patch-comparison.md");
|
||||
assert.equal(manifest.outputs?.patchJson, "patch-comparison.json");
|
||||
assert.equal(manifest.thresholds?.patch?.maxLatencyIncrease, 0.05);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval check fails when patch gate regresses beyond thresholds", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-gate-fail-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const baselinePatch = join(dir, "baseline.patch.json");
|
||||
const candidatePatch = join(dir, "candidate.patch.json");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "patch-gate-fail-001";
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writePatch(baselinePatch, "balanced-v1", 95, 0.004, 120);
|
||||
writePatch(candidatePatch, "balanced-v2", 90, 0.008, 200);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--baseline-patch",
|
||||
baselinePatch,
|
||||
"--candidate-patch",
|
||||
candidatePatch,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
"--max-patch-aiq-drop",
|
||||
"1",
|
||||
"--max-patch-cost-increase",
|
||||
"0.1",
|
||||
"--max-patch-latency-increase",
|
||||
"0.1",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr ?? "", /patch gate failed/);
|
||||
const comparison = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "patch-comparison.json"), "utf8")
|
||||
) as {
|
||||
result?: { passed?: boolean; status?: number };
|
||||
regressions?: string[];
|
||||
};
|
||||
assert.equal(comparison.result?.passed, false);
|
||||
assert.equal(comparison.result?.status, 1);
|
||||
assert.ok((comparison.regressions?.length ?? 0) >= 2);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "manifest.json"), "utf8")
|
||||
) as {
|
||||
result?: { status?: number };
|
||||
};
|
||||
assert.equal(manifest.result?.status, 1);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval check rejects unpaired patch inputs", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-unpaired-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const baselinePatch = join(dir, "baseline.patch.json");
|
||||
const candidatePatch = join(dir, "candidate.patch.json");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "unpaired-001";
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writePatch(baselinePatch, "balanced-v1", 95, 0.004, 120);
|
||||
writePatch(candidatePatch, "balanced-v2", 96, 0.003, 110);
|
||||
|
||||
const baselineOnly = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--baseline-patch",
|
||||
baselinePatch,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
const candidateOnly = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--candidate-patch",
|
||||
candidatePatch,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(baselineOnly.status, 2);
|
||||
assert.match(baselineOnly.stderr ?? "", /baseline-patch and --candidate-patch/);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "manifest.json"), "utf8")
|
||||
) as {
|
||||
result?: { status?: number };
|
||||
inputs?: { baselinePatch?: string; candidatePatch?: string };
|
||||
};
|
||||
assert.equal(manifest.result?.status, 2);
|
||||
assert.equal(manifest.inputs?.baselinePatch, undefined);
|
||||
assert.equal(manifest.inputs?.candidatePatch, undefined);
|
||||
assert.equal(candidateOnly.status, 2);
|
||||
assert.match(candidateOnly.stderr ?? "", /baseline-patch and --candidate-patch/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval check can retain artifacts for trend summaries", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-retained-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "run-001";
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "priority",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
|
||||
const checkResult = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
const trendResult = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/router-eval/trends.ts", "--artifact-dir", artifactDir],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(checkResult.status, 0);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "router-eval.md"), "utf8").includes(
|
||||
"Router Eval Comparison"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "router-eval.json"), "utf8").includes(
|
||||
"router-eval-comparison"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "inputs", "baseline.ndjson"), "utf8").includes(
|
||||
"sampleId"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, runId, "inputs", "candidate.ndjson"), "utf8").includes(
|
||||
"sampleId"
|
||||
)
|
||||
);
|
||||
const manifest = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "manifest.json"), "utf8")
|
||||
) as Record<string, unknown>;
|
||||
assert.equal(manifest.schemaVersion, 1);
|
||||
assert.equal(manifest.kind, "router-eval-gate-run");
|
||||
assert.equal(manifest.runId, runId);
|
||||
assert.deepEqual(manifest.inputs, {
|
||||
baseline: "inputs/baseline.ndjson",
|
||||
candidate: "inputs/candidate.ndjson",
|
||||
});
|
||||
assert.deepEqual(manifest.outputs, {
|
||||
markdown: "router-eval.md",
|
||||
json: "router-eval.json",
|
||||
});
|
||||
assert.equal(trendResult.status, 0);
|
||||
assert.ok((trendResult.stdout ?? "").includes("Router Eval Trends"));
|
||||
assert.ok((trendResult.stdout ?? "").includes(runId));
|
||||
assert.ok((trendResult.stdout ?? "").includes("| jsonl | all |"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
270
tests/unit/router-eval-cli.test.ts
Normal file
270
tests/unit/router-eval-cli.test.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
const scriptPath = "scripts/router-eval/index.ts";
|
||||
|
||||
function runCli(args: string[]) {
|
||||
return spawnSync(process.execPath, ["--import", "tsx", scriptPath, ...args], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
test("router-eval CLI prints a markdown report for JSONL input", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-"));
|
||||
const inputPath = join(dir, "input.ndjson");
|
||||
writeFileSync(
|
||||
inputPath,
|
||||
[
|
||||
JSON.stringify({
|
||||
sampleId: "s1",
|
||||
configId: "combo-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
requestedModel: "gpt-4.1",
|
||||
latencyMs: 120,
|
||||
costUsd: 0.005,
|
||||
success: true,
|
||||
}),
|
||||
JSON.stringify({
|
||||
sampleId: "s2",
|
||||
configId: "combo-b",
|
||||
selectedModel: "gpt-4o",
|
||||
requestedModel: "gpt-4o",
|
||||
latencyMs: 200,
|
||||
costUsd: 0.003,
|
||||
success: true,
|
||||
}),
|
||||
].join("\n")
|
||||
);
|
||||
|
||||
const result = runCli(["--input", inputPath]);
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stderr ?? "").length === 0);
|
||||
assert.ok((result.stdout ?? "").includes("Frontier"));
|
||||
assert.ok((result.stdout ?? "").includes("AIQ"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router-eval CLI exits non-zero when regression threshold is exceeded", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-reg-"));
|
||||
const baselinePath = join(dir, "baseline.ndjson");
|
||||
const candidatePath = join(dir, "candidate.ndjson");
|
||||
|
||||
writeFileSync(
|
||||
baselinePath,
|
||||
[
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "combo-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
requestedModel: "gpt-4.1",
|
||||
latencyMs: 100,
|
||||
costUsd: 0.005,
|
||||
success: true,
|
||||
}),
|
||||
].join("\n")
|
||||
);
|
||||
|
||||
writeFileSync(
|
||||
candidatePath,
|
||||
[
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "combo-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
requestedModel: "gpt-4.1",
|
||||
latencyMs: 400,
|
||||
costUsd: 0.05,
|
||||
success: true,
|
||||
}),
|
||||
].join("\n")
|
||||
);
|
||||
|
||||
const result = runCli([
|
||||
"--input",
|
||||
candidatePath,
|
||||
"--baseline-input",
|
||||
baselinePath,
|
||||
"--max-aiq-drop",
|
||||
"1",
|
||||
"--max-cost-increase",
|
||||
"0.2",
|
||||
"--fail-on-regression",
|
||||
]);
|
||||
try {
|
||||
assert.equal(result.status, 1);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Comparison"));
|
||||
assert.ok((result.stdout ?? "").includes("Regressions"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router-eval CLI writes machine-readable JSON artifacts", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-json-"));
|
||||
const inputPath = join(dir, "input.ndjson");
|
||||
const outputPath = join(dir, "router-eval.json");
|
||||
writeFileSync(
|
||||
inputPath,
|
||||
JSON.stringify({
|
||||
sampleId: "s1",
|
||||
configId: "combo-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
requestedModel: "gpt-4.1",
|
||||
latencyMs: 120,
|
||||
costUsd: 0.005,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
|
||||
const result = runCli(["--input", inputPath, "--json-output", outputPath]);
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
const artifact = JSON.parse(readFileSync(outputPath, "utf8")) as Record<string, unknown>;
|
||||
assert.equal(artifact.schemaVersion, 1);
|
||||
assert.equal(artifact.kind, "router-eval-report");
|
||||
assert.ok("report" in artifact);
|
||||
assert.deepEqual((artifact.metadata as Record<string, unknown>)?.candidate, {
|
||||
source: "jsonl",
|
||||
path: inputPath,
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router-eval CLI reads usage_history DB source", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-usage-"));
|
||||
const dbPath = join(dir, "storage.sqlite");
|
||||
const corpusPath = join(dir, "corpus.ndjson");
|
||||
const artifactPath = join(dir, "artifact.json");
|
||||
const db = new Database(dbPath);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS usage_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT,
|
||||
model TEXT,
|
||||
connection_id TEXT,
|
||||
api_key_id TEXT,
|
||||
api_key_name TEXT,
|
||||
tokens_input INTEGER DEFAULT 0,
|
||||
tokens_output INTEGER DEFAULT 0,
|
||||
tokens_cache_read INTEGER DEFAULT 0,
|
||||
tokens_cache_creation INTEGER DEFAULT 0,
|
||||
tokens_reasoning INTEGER DEFAULT 0,
|
||||
service_tier TEXT DEFAULT 'standard',
|
||||
status TEXT,
|
||||
success INTEGER DEFAULT 1,
|
||||
latency_ms INTEGER DEFAULT 0,
|
||||
ttft_ms INTEGER DEFAULT 0,
|
||||
error_code TEXT,
|
||||
combo_strategy TEXT,
|
||||
endpoint TEXT,
|
||||
timestamp TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO usage_history
|
||||
(provider, model, tokens_input, tokens_output, service_tier, success, latency_ms, status, error_code, combo_strategy, timestamp)
|
||||
VALUES
|
||||
('openrouter', 'gpt-4.1', 120, 80, 'standard', 1, 150, '200', NULL, 'priority', '2026-01-01T00:00:00.000Z')
|
||||
`
|
||||
).run();
|
||||
db.close();
|
||||
|
||||
try {
|
||||
const result = runCli([
|
||||
"--db",
|
||||
dbPath,
|
||||
"--db-source",
|
||||
"usage-history",
|
||||
"--export-corpus",
|
||||
corpusPath,
|
||||
"--json-output",
|
||||
artifactPath,
|
||||
]);
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stderr ?? "").length === 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Report"));
|
||||
assert.ok((result.stdout ?? "").includes("priority"));
|
||||
const corpus = readFileSync(corpusPath, "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
assert.equal(corpus.length, 1);
|
||||
assert.equal(corpus[0].configId, "priority");
|
||||
assert.equal(corpus[0].selectedModel, "gpt-4.1");
|
||||
assert.equal(corpus[0].latencyMs, 150);
|
||||
assert.deepEqual(corpus[0].metadata, {
|
||||
provider: "openrouter",
|
||||
serviceTier: "standard",
|
||||
errorCode: null,
|
||||
});
|
||||
const artifact = JSON.parse(readFileSync(artifactPath, "utf8")) as Record<string, unknown>;
|
||||
assert.deepEqual((artifact.metadata as Record<string, unknown>)?.candidate, {
|
||||
source: "sqlite",
|
||||
path: dbPath,
|
||||
dbSource: "usage-history",
|
||||
});
|
||||
assert.equal(
|
||||
typeof ((artifact.metadata as Record<string, unknown>)?.outputs as Record<string, unknown>)
|
||||
?.corpus,
|
||||
"string"
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router-eval CLI defaults --db to call_logs when available", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-default-"));
|
||||
const dbPath = join(dir, "storage.sqlite");
|
||||
const db = new Database(dbPath);
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS call_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
duration INTEGER,
|
||||
tokens_in INTEGER,
|
||||
tokens_out INTEGER,
|
||||
status INTEGER,
|
||||
combo_name TEXT,
|
||||
requested_model TEXT,
|
||||
model TEXT,
|
||||
provider TEXT,
|
||||
error_summary TEXT,
|
||||
timestamp TEXT NOT NULL,
|
||||
correlation_id TEXT
|
||||
)
|
||||
`);
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO call_logs
|
||||
(id, provider, model, requested_model, tokens_in, tokens_out, status, combo_name, timestamp)
|
||||
VALUES
|
||||
('c1', 'openrouter', 'gpt-4.1', 'gpt-4.1', 120, 80, 200, 'priority', '2026-01-01T00:00:00.000Z')
|
||||
`
|
||||
).run();
|
||||
db.close();
|
||||
|
||||
try {
|
||||
const result = runCli(["--db", dbPath]);
|
||||
assert.equal(result.status, 0);
|
||||
assert.equal((result.stderr ?? "").length, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Report"));
|
||||
assert.ok((result.stdout ?? "").includes("priority"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
83
tests/unit/router-eval-compare.test.ts
Normal file
83
tests/unit/router-eval-compare.test.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const compareScript = "scripts/router-eval/compare.ts";
|
||||
|
||||
test("router eval compare retains named comparison artifacts", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-compare-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const candidate = join(dir, "candidate.ndjson");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "policy-a-vs-policy-b";
|
||||
|
||||
writeFileSync(
|
||||
baseline,
|
||||
JSON.stringify({
|
||||
sampleId: "b1",
|
||||
configId: "policy-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 140,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
JSON.stringify({
|
||||
sampleId: "c1",
|
||||
configId: "policy-b",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 130,
|
||||
costUsd: 0.004,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
compareScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--baseline-name",
|
||||
"policy-a",
|
||||
"--candidate-name",
|
||||
"policy-b",
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Comparison"));
|
||||
const runDir = join(artifactDir, runId);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "router-eval.md"), "utf8").includes("Router Eval Comparison")
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "router-eval.json"), "utf8").includes("router-eval-comparison")
|
||||
);
|
||||
const comparison = JSON.parse(readFileSync(join(runDir, "comparison.json"), "utf8")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
assert.equal(comparison.baselineName, "policy-a");
|
||||
assert.equal(comparison.candidateName, "policy-b");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
135
tests/unit/router-eval-e2e-chain.test.ts
Normal file
135
tests/unit/router-eval-e2e-chain.test.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const searchScript = "scripts/router-eval/search.ts";
|
||||
const checkScript = "scripts/check/check-router-eval-regression.ts";
|
||||
|
||||
function writeObservation(file: string, configId: string, latencyMs: number, costUsd: number) {
|
||||
writeFileSync(
|
||||
file,
|
||||
`${JSON.stringify({
|
||||
sampleId: `${configId}-sample`,
|
||||
configId,
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs,
|
||||
costUsd,
|
||||
success: true,
|
||||
})}\n`
|
||||
);
|
||||
}
|
||||
|
||||
function runSearch(
|
||||
baseline: string,
|
||||
candidateName: string,
|
||||
candidate: string,
|
||||
artifactDir: string,
|
||||
runId: string
|
||||
): string {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
searchScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
`${candidateName}=${candidate}`,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
"--objective",
|
||||
"quality",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
const patch = join(artifactDir, runId, "router-config.patch.json");
|
||||
assert.ok(readFileSync(patch, "utf8").includes("router-config-patch"));
|
||||
return patch;
|
||||
}
|
||||
|
||||
test("router eval retained chain runs search patches through the check wrapper gate", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-e2e-chain-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const oldCandidate = join(dir, "old.ndjson");
|
||||
const newCandidate = join(dir, "new.ndjson");
|
||||
const searchArtifacts = join(dir, "search-artifacts");
|
||||
const gateArtifacts = join(dir, "gate-artifacts");
|
||||
const runId = "chain-001";
|
||||
|
||||
writeObservation(baseline, "baseline", 150, 0.004);
|
||||
writeObservation(oldCandidate, "old-router", 130, 0.004);
|
||||
writeObservation(newCandidate, "new-router", 100, 0.003);
|
||||
|
||||
try {
|
||||
const baselinePatch = runSearch(
|
||||
baseline,
|
||||
"old-router",
|
||||
oldCandidate,
|
||||
searchArtifacts,
|
||||
"old-search"
|
||||
);
|
||||
const candidatePatch = runSearch(
|
||||
baseline,
|
||||
"new-router",
|
||||
newCandidate,
|
||||
searchArtifacts,
|
||||
"new-search"
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
checkScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
newCandidate,
|
||||
"--baseline-patch",
|
||||
baselinePatch,
|
||||
"--candidate-patch",
|
||||
candidatePatch,
|
||||
"--artifact-dir",
|
||||
gateArtifacts,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
assert.equal(result.status, 0);
|
||||
const runDir = join(gateArtifacts, runId);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "router-eval.json"), "utf8").includes("router-eval-comparison")
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "patch-comparison.json"), "utf8").includes(
|
||||
"router-config-patch-comparison"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "inputs", "baseline.patch.json"), "utf8").includes("old-router")
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(runDir, "inputs", "candidate.patch.json"), "utf8").includes("new-router")
|
||||
);
|
||||
const manifest = JSON.parse(readFileSync(join(runDir, "manifest.json"), "utf8")) as {
|
||||
kind?: string;
|
||||
result?: { status?: number };
|
||||
outputs?: { patchJson?: string };
|
||||
};
|
||||
assert.equal(manifest.kind, "router-eval-gate-run");
|
||||
assert.equal(manifest.result?.status, 0);
|
||||
assert.equal(manifest.outputs?.patchJson, "patch-comparison.json");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
312
tests/unit/router-eval-patch-compare.test.ts
Normal file
312
tests/unit/router-eval-patch-compare.test.ts
Normal file
@@ -0,0 +1,312 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const compareScript = "scripts/router-eval/patch-compare.ts";
|
||||
|
||||
function writePatch(
|
||||
file: string,
|
||||
configId: string,
|
||||
aiq: number,
|
||||
costUsd: number,
|
||||
latencyMs: number
|
||||
) {
|
||||
writeFileSync(
|
||||
file,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-patch",
|
||||
generatedAt: "2026-07-03T00:00:00.000Z",
|
||||
applyPolicy: "manual-review",
|
||||
source: {
|
||||
objective: "balanced",
|
||||
runId: `${configId}-run`,
|
||||
artifactPath: `/tmp/${configId}/router-eval.json`,
|
||||
},
|
||||
operations: [
|
||||
{
|
||||
op: "recommend-router-config",
|
||||
path: "/router/recommendedConfigId",
|
||||
value: configId,
|
||||
evidence: {
|
||||
aiq,
|
||||
avgCostUsd: costUsd,
|
||||
avgLatencyMs: latencyMs,
|
||||
regressions: 0,
|
||||
},
|
||||
rationale: `${configId} wins`,
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
);
|
||||
}
|
||||
|
||||
test("router config patch compare reports recommendation and metric deltas", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-"));
|
||||
const baseline = join(dir, "baseline.patch.json");
|
||||
const candidate = join(dir, "candidate.patch.json");
|
||||
const jsonOutput = join(dir, "comparison.json");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
|
||||
writePatch(baseline, "balanced-v1", 91, 0.006, 180);
|
||||
writePatch(candidate, "balanced-v2", 94, 0.004, 120);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
compareScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--baseline-name",
|
||||
"current",
|
||||
"--candidate-name",
|
||||
"proposal",
|
||||
"--json-output",
|
||||
jsonOutput,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
"compare-001",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Config Patch Comparison"));
|
||||
assert.ok((result.stdout ?? "").includes("Changed recommendation: yes"));
|
||||
const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as {
|
||||
kind?: string;
|
||||
changedRecommendation?: boolean;
|
||||
delta?: { aiq?: number; avgCostUsd?: number; avgLatencyMs?: number };
|
||||
candidate?: { recommendedConfigId?: string };
|
||||
};
|
||||
assert.equal(comparison.kind, "router-config-patch-comparison");
|
||||
assert.equal(comparison.changedRecommendation, true);
|
||||
assert.equal(comparison.result?.passed, true);
|
||||
assert.equal(comparison.candidate?.recommendedConfigId, "balanced-v2");
|
||||
assert.equal(comparison.delta?.aiq, 3);
|
||||
assert.equal(comparison.delta?.avgCostUsd, -0.002);
|
||||
assert.equal(comparison.delta?.avgLatencyMs, -60);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, "compare-001", "patch-comparison.md"), "utf8").includes(
|
||||
"balanced-v2"
|
||||
)
|
||||
);
|
||||
assert.ok(
|
||||
readFileSync(join(artifactDir, "compare-001", "patch-comparison.json"), "utf8").includes(
|
||||
"router-config-patch-comparison"
|
||||
)
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router config patch compare only fails threshold regressions when requested", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-gate-"));
|
||||
const baseline = join(dir, "baseline.patch.json");
|
||||
const candidate = join(dir, "candidate.patch.json");
|
||||
const jsonOutput = join(dir, "comparison.json");
|
||||
|
||||
writePatch(baseline, "balanced-v1", 95, 0.004, 120);
|
||||
writePatch(candidate, "balanced-v2", 90, 0.008, 200);
|
||||
|
||||
const warning = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
compareScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--max-aiq-drop",
|
||||
"1",
|
||||
"--max-cost-increase",
|
||||
"0.1",
|
||||
"--max-latency-increase",
|
||||
"0.1",
|
||||
"--json-output",
|
||||
jsonOutput,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
const failing = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
compareScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--max-aiq-drop",
|
||||
"1",
|
||||
"--max-cost-increase",
|
||||
"0.1",
|
||||
"--max-latency-increase",
|
||||
"0.1",
|
||||
"--fail-on-regression",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(warning.status, 0);
|
||||
assert.ok((warning.stdout ?? "").includes("Passed: no"));
|
||||
const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as {
|
||||
regressions?: string[];
|
||||
result?: { passed?: boolean; status?: number };
|
||||
};
|
||||
assert.equal(comparison.result?.passed, false);
|
||||
assert.equal(comparison.result?.status, 0);
|
||||
assert.ok((comparison.regressions?.length ?? 0) >= 2);
|
||||
assert.equal(failing.status, 1);
|
||||
assert.ok((failing.stdout ?? "").includes("Passed: no"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router config patch compare reports unchanged recommendations without failing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-no-change-"));
|
||||
const baseline = join(dir, "baseline.patch.json");
|
||||
const candidate = join(dir, "candidate.patch.json");
|
||||
const jsonOutput = join(dir, "comparison.json");
|
||||
|
||||
writePatch(baseline, "balanced-v1", 94, 0.004, 120);
|
||||
writePatch(candidate, "balanced-v1", 95, 0.003, 110);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
compareScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
candidate,
|
||||
"--json-output",
|
||||
jsonOutput,
|
||||
"--fail-on-regression",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Changed recommendation: no"));
|
||||
const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as {
|
||||
changedRecommendation?: boolean;
|
||||
delta?: { aiq?: number; avgCostUsd?: number; avgLatencyMs?: number };
|
||||
result?: { passed?: boolean; status?: number };
|
||||
};
|
||||
assert.equal(comparison.changedRecommendation, false);
|
||||
assert.equal(comparison.delta?.aiq, 1);
|
||||
assert.equal(comparison.delta?.avgCostUsd, -0.001);
|
||||
assert.equal(comparison.delta?.avgLatencyMs, -10);
|
||||
assert.equal(comparison.result?.passed, true);
|
||||
assert.equal(comparison.result?.status, 0);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router config patch compare rejects invalid patch inputs", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-invalid-"));
|
||||
const baseline = join(dir, "baseline.patch.json");
|
||||
const candidate = join(dir, "candidate.patch.json");
|
||||
|
||||
writePatch(baseline, "balanced-v1", 94, 0.004, 120);
|
||||
writeFileSync(
|
||||
candidate,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-suggestion",
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", candidate],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 2);
|
||||
assert.match(result.stderr ?? "", /invalid patch kind/);
|
||||
assert.match(result.stderr ?? "", /router-config-suggestion/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router config patch compare rejects malformed JSON and invalid evidence", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-malformed-"));
|
||||
const baseline = join(dir, "baseline.patch.json");
|
||||
const malformed = join(dir, "malformed.patch.json");
|
||||
const invalidEvidence = join(dir, "invalid-evidence.patch.json");
|
||||
|
||||
writePatch(baseline, "balanced-v1", 94, 0.004, 120);
|
||||
writeFileSync(malformed, "{not-json");
|
||||
writeFileSync(
|
||||
invalidEvidence,
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
kind: "router-config-patch",
|
||||
applyPolicy: "manual-review",
|
||||
operations: [
|
||||
{
|
||||
op: "recommend-router-config",
|
||||
path: "/router/recommendedConfigId",
|
||||
value: "balanced-v2",
|
||||
evidence: {
|
||||
aiq: "94",
|
||||
avgCostUsd: 0.004,
|
||||
avgLatencyMs: 120,
|
||||
regressions: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
})}\n`
|
||||
);
|
||||
|
||||
const malformedResult = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", malformed],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
const invalidEvidenceResult = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", invalidEvidence],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(malformedResult.status, 2);
|
||||
assert.match(malformedResult.stderr ?? "", /invalid JSON/);
|
||||
assert.equal(invalidEvidenceResult.status, 2);
|
||||
assert.match(invalidEvidenceResult.stderr ?? "", /invalid numeric evidence field aiq/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
258
tests/unit/router-eval-search.test.ts
Normal file
258
tests/unit/router-eval-search.test.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const searchScript = "scripts/router-eval/search.ts";
|
||||
|
||||
function writeCorpus(path: string, configId: string, latencyMs: number, costUsd: number) {
|
||||
writeFileSync(
|
||||
path,
|
||||
JSON.stringify({
|
||||
sampleId: configId,
|
||||
configId,
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs,
|
||||
costUsd,
|
||||
success: true,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function writeCorpusRows(
|
||||
path: string,
|
||||
rows: Array<{ configId: string; latencyMs: number; costUsd: number }>
|
||||
) {
|
||||
writeFileSync(
|
||||
path,
|
||||
rows
|
||||
.map((row) =>
|
||||
JSON.stringify({
|
||||
sampleId: `${row.configId}-sample`,
|
||||
configId: row.configId,
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: row.latencyMs,
|
||||
costUsd: row.costUsd,
|
||||
success: true,
|
||||
})
|
||||
)
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
test("router eval search ranks candidates and writes retained summary artifacts", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-search-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const slow = join(dir, "slow.ndjson");
|
||||
const fast = join(dir, "fast.ndjson");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "search-001";
|
||||
|
||||
writeCorpus(baseline, "baseline", 150, 0.004);
|
||||
writeCorpus(slow, "slow", 200, 0.006);
|
||||
writeCorpus(fast, "fast", 100, 0.003);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
searchScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
`slow=${slow}`,
|
||||
"--candidate",
|
||||
`fast=${fast}`,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Search"));
|
||||
const searchDir = join(artifactDir, runId);
|
||||
const markdown = readFileSync(join(searchDir, "search.md"), "utf8");
|
||||
assert.ok(markdown.indexOf("| 1 | fast |") < markdown.indexOf("| 2 | slow |"));
|
||||
assert.ok(markdown.includes("## Patch Operations"));
|
||||
assert.ok(
|
||||
markdown.includes("| recommend-router-config | /router/recommendedConfigId | fast |")
|
||||
);
|
||||
const summary = JSON.parse(readFileSync(join(searchDir, "search.json"), "utf8")) as {
|
||||
kind?: string;
|
||||
objective?: string;
|
||||
recommendation?: { candidateName: string; rationale: string };
|
||||
suggestion?: { recommendedConfigId: string; applyPolicy: string };
|
||||
patch?: { kind: string; operations: Array<{ value: string }> };
|
||||
results?: Array<{ candidateName: string }>;
|
||||
};
|
||||
assert.equal(summary.kind, "router-eval-search");
|
||||
assert.equal(summary.objective, "balanced");
|
||||
assert.equal(summary.results?.[0]?.candidateName, "fast");
|
||||
assert.equal(summary.recommendation?.candidateName, "fast");
|
||||
assert.match(summary.recommendation?.rationale ?? "", /best balanced rank/);
|
||||
assert.equal(summary.suggestion?.recommendedConfigId, "fast");
|
||||
assert.equal(summary.suggestion?.applyPolicy, "manual-review");
|
||||
assert.equal(summary.patch?.kind, "router-config-patch");
|
||||
assert.equal(summary.patch?.operations[0]?.value, "fast");
|
||||
const recommendation = JSON.parse(
|
||||
readFileSync(join(searchDir, "recommendation.json"), "utf8")
|
||||
) as {
|
||||
candidateName?: string;
|
||||
objective?: string;
|
||||
};
|
||||
assert.equal(recommendation.candidateName, "fast");
|
||||
assert.equal(recommendation.objective, "balanced");
|
||||
const suggestion = JSON.parse(readFileSync(join(searchDir, "suggestion.json"), "utf8")) as {
|
||||
kind?: string;
|
||||
recommendedConfigId?: string;
|
||||
applyPolicy?: string;
|
||||
};
|
||||
assert.equal(suggestion.kind, "router-config-suggestion");
|
||||
assert.equal(suggestion.recommendedConfigId, "fast");
|
||||
assert.equal(suggestion.applyPolicy, "manual-review");
|
||||
const patch = JSON.parse(readFileSync(join(searchDir, "router-config.patch.json"), "utf8")) as {
|
||||
kind?: string;
|
||||
applyPolicy?: string;
|
||||
operations?: Array<{ op?: string; path?: string; value?: string }>;
|
||||
};
|
||||
assert.equal(patch.kind, "router-config-patch");
|
||||
assert.equal(patch.applyPolicy, "manual-review");
|
||||
assert.equal(patch.operations?.[0]?.op, "recommend-router-config");
|
||||
assert.equal(patch.operations?.[0]?.path, "/router/recommendedConfigId");
|
||||
assert.equal(patch.operations?.[0]?.value, "fast");
|
||||
assert.ok(
|
||||
readFileSync(join(searchDir, `${runId}-fast`, "router-eval.json"), "utf8").includes(
|
||||
"router-eval-comparison"
|
||||
)
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval search objective modes can choose different candidates", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-search-objective-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const cheap = join(dir, "cheap.ndjson");
|
||||
const fast = join(dir, "fast.ndjson");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
|
||||
writeCorpus(baseline, "baseline", 150, 0.004);
|
||||
writeCorpus(cheap, "cheap", 900, 0.001);
|
||||
writeCorpus(fast, "fast", 50, 0.009);
|
||||
|
||||
function runSearch(
|
||||
objective: string,
|
||||
runId: string
|
||||
): { recommendedConfigId?: string; objective?: string } {
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
searchScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
`cheap=${cheap}`,
|
||||
"--candidate",
|
||||
`fast=${fast}`,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
"--objective",
|
||||
objective,
|
||||
"--max-aiq-drop",
|
||||
"100",
|
||||
"--max-cost-increase",
|
||||
"100",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
return JSON.parse(readFileSync(join(artifactDir, runId, "suggestion.json"), "utf8")) as {
|
||||
recommendedConfigId?: string;
|
||||
objective?: string;
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const costSuggestion = runSearch("cost", "cost-001");
|
||||
const latencySuggestion = runSearch("latency", "latency-001");
|
||||
const qualitySuggestion = runSearch("quality", "quality-001");
|
||||
assert.equal(costSuggestion.objective, "cost");
|
||||
assert.equal(costSuggestion.recommendedConfigId, "cheap");
|
||||
assert.equal(latencySuggestion.objective, "latency");
|
||||
assert.equal(latencySuggestion.recommendedConfigId, "fast");
|
||||
assert.equal(qualitySuggestion.objective, "quality");
|
||||
assert.equal(qualitySuggestion.recommendedConfigId, "cheap");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval search cost objective can select a non-AIQ-top config inside a candidate corpus", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-search-multi-config-"));
|
||||
const baseline = join(dir, "baseline.ndjson");
|
||||
const mixed = join(dir, "mixed.ndjson");
|
||||
const artifactDir = join(dir, "artifacts");
|
||||
const runId = "multi-config-001";
|
||||
|
||||
writeCorpus(baseline, "baseline", 150, 0.004);
|
||||
writeCorpusRows(mixed, [
|
||||
{ configId: "quality-top", latencyMs: 100, costUsd: 0.002 },
|
||||
{ configId: "cost-top", latencyMs: 3_000, costUsd: 0.0001 },
|
||||
]);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
searchScript,
|
||||
"--baseline",
|
||||
baseline,
|
||||
"--candidate",
|
||||
`mixed=${mixed}`,
|
||||
"--artifact-dir",
|
||||
artifactDir,
|
||||
"--run-id",
|
||||
runId,
|
||||
"--objective",
|
||||
"cost",
|
||||
"--max-aiq-drop",
|
||||
"100",
|
||||
"--max-cost-increase",
|
||||
"100",
|
||||
],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
const suggestion = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "suggestion.json"), "utf8")
|
||||
) as {
|
||||
recommendedConfigId?: string;
|
||||
};
|
||||
const patch = JSON.parse(
|
||||
readFileSync(join(artifactDir, runId, "router-config.patch.json"), "utf8")
|
||||
) as {
|
||||
operations?: Array<{ value?: string }>;
|
||||
};
|
||||
assert.equal(suggestion.recommendedConfigId, "cost-top");
|
||||
assert.equal(patch.operations?.[0]?.value, "cost-top");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
128
tests/unit/router-eval-trends.test.ts
Normal file
128
tests/unit/router-eval-trends.test.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const trendsScript = "scripts/router-eval/trends.ts";
|
||||
|
||||
function artifact(generatedAt: string, configId: string, aiq: number) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
kind: "router-eval-report",
|
||||
generatedAt,
|
||||
report: {
|
||||
evaluatedAt: generatedAt,
|
||||
summary: {
|
||||
totalSamples: 1,
|
||||
validSamples: 1,
|
||||
droppedSamples: 0,
|
||||
uniqueConfigs: 1,
|
||||
},
|
||||
configurations: [],
|
||||
frontier: [],
|
||||
top: [
|
||||
{
|
||||
configId,
|
||||
samples: 1,
|
||||
successRate: 1,
|
||||
avgLatencyMs: 100,
|
||||
p50LatencyMs: 100,
|
||||
p95LatencyMs: 100,
|
||||
avgCostUsd: 0.004,
|
||||
aiq,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("router eval trends reads retained and flat artifacts with limit", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-trends-"));
|
||||
const runA = join(dir, "run-a");
|
||||
const runB = join(dir, "run-b");
|
||||
mkdirSync(runA);
|
||||
mkdirSync(runB);
|
||||
writeFileSync(
|
||||
join(runA, "router-eval.json"),
|
||||
JSON.stringify(artifact("2026-01-01T00:00:00.000Z", "old", 80))
|
||||
);
|
||||
writeFileSync(
|
||||
join(runB, "router-eval.json"),
|
||||
JSON.stringify(artifact("2026-01-02T00:00:00.000Z", "new", 90))
|
||||
);
|
||||
writeFileSync(
|
||||
join(dir, "flat.json"),
|
||||
JSON.stringify(artifact("2026-01-03T00:00:00.000Z", "flat", 95))
|
||||
);
|
||||
writeFileSync(join(dir, "bad.json"), "{not json");
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", trendsScript, "--artifact-dir", dir, "--limit", "2"],
|
||||
{
|
||||
encoding: "utf8",
|
||||
}
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Trends"));
|
||||
assert.ok(!(result.stdout ?? "").includes("run-a"));
|
||||
assert.ok((result.stdout ?? "").includes("run-b"));
|
||||
assert.ok((result.stdout ?? "").includes("flat"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval trends can print dashboard summaries", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-dashboard-"));
|
||||
const runA = join(dir, "run-a");
|
||||
const runB = join(dir, "run-b");
|
||||
mkdirSync(runA);
|
||||
mkdirSync(runB);
|
||||
writeFileSync(
|
||||
join(runA, "router-eval.json"),
|
||||
JSON.stringify(artifact("2026-01-01T00:00:00.000Z", "old", 80))
|
||||
);
|
||||
writeFileSync(
|
||||
join(runB, "router-eval.json"),
|
||||
JSON.stringify(artifact("2026-01-02T00:00:00.000Z", "new", 90))
|
||||
);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", trendsScript, "--artifact-dir", dir, "--dashboard"],
|
||||
{
|
||||
encoding: "utf8",
|
||||
}
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 0);
|
||||
assert.ok((result.stdout ?? "").includes("Router Eval Dashboard"));
|
||||
assert.ok((result.stdout ?? "").includes("Latest: run-b"));
|
||||
assert.ok((result.stdout ?? "").includes("AIQ: 90.000 (+10.000)"));
|
||||
assert.ok((result.stdout ?? "").includes("Rolling Averages"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("router eval trends exits clearly for empty artifact dirs", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "router-eval-trends-empty-"));
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
["--import", "tsx", trendsScript, "--artifact-dir", dir],
|
||||
{ encoding: "utf8" }
|
||||
);
|
||||
|
||||
try {
|
||||
assert.equal(result.status, 2);
|
||||
assert.ok((result.stderr ?? "").includes("No router-eval artifacts found"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
92
tests/unit/router-eval.test.ts
Normal file
92
tests/unit/router-eval.test.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
compareRouterEvalRuns,
|
||||
createRouterEvalArtifact,
|
||||
runRouterEval,
|
||||
toRouterObservation,
|
||||
} from "@/lib/routerEval/index.ts";
|
||||
|
||||
function obs(overrides: Record<string, unknown>) {
|
||||
return toRouterObservation({
|
||||
sampleId: "sample-1",
|
||||
configId: "config",
|
||||
selectedModel: "gpt-4.1",
|
||||
expectedModel: "gpt-4.1",
|
||||
latencyMs: 120,
|
||||
costUsd: 0.05,
|
||||
success: true,
|
||||
timestamp: "2026-01-01T00:00:00.000Z",
|
||||
...overrides,
|
||||
})!;
|
||||
}
|
||||
|
||||
test("toRouterObservation normalizes token cost and booleans", () => {
|
||||
const observation = toRouterObservation({
|
||||
sampleId: "s-1",
|
||||
configId: "combo-a",
|
||||
selectedModel: "gpt-4.1",
|
||||
requestedModel: "gpt-4.1",
|
||||
durationMs: 240,
|
||||
tokens: { input: 100, output: 50 },
|
||||
status: 200,
|
||||
});
|
||||
|
||||
assert.ok(observation);
|
||||
assert.equal(observation!.sampleId, "s-1");
|
||||
assert.equal(observation!.configId, "combo-a");
|
||||
assert.equal(observation!.latencyMs, 240);
|
||||
assert.equal(observation!.costUsd, 0.00015);
|
||||
assert.equal(observation!.success, true);
|
||||
assert.equal(observation!.expectedModel, "gpt-4.1");
|
||||
});
|
||||
|
||||
test("runRouterEval computes summary and frontiers", () => {
|
||||
const report = runRouterEval([
|
||||
obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
||||
obs({ sampleId: "s2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
||||
obs({ sampleId: "s3", configId: "b", latencyMs: 80, costUsd: 0.02, success: true }),
|
||||
obs({ sampleId: "s4", configId: "b", latencyMs: 120, costUsd: 0.02, success: false }),
|
||||
]);
|
||||
|
||||
assert.equal(report.summary.totalSamples, 4);
|
||||
assert.equal(report.summary.validSamples, 4);
|
||||
assert.equal(report.summary.uniqueConfigs, 2);
|
||||
assert.equal(report.configurations.length, 2);
|
||||
assert.equal(report.top[0]?.configId, "a");
|
||||
assert.ok(report.frontier.length >= 1);
|
||||
});
|
||||
|
||||
test("compareRouterEvalRuns captures AIQ and cost regressions", () => {
|
||||
const baseline = runRouterEval([
|
||||
obs({ sampleId: "b1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
||||
obs({ sampleId: "b2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
||||
]);
|
||||
const candidate = runRouterEval([
|
||||
obs({ sampleId: "c1", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }),
|
||||
obs({ sampleId: "c2", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }),
|
||||
]);
|
||||
|
||||
const comparison = compareRouterEvalRuns(baseline, candidate, {
|
||||
aiqDrop: 5,
|
||||
relativeCostIncrease: 1.2,
|
||||
});
|
||||
|
||||
assert.equal(comparison.regressions.length > 0, true);
|
||||
assert.equal(comparison.delta.aiq <= 0, true);
|
||||
assert.equal(comparison.delta.costUsd > 0, true);
|
||||
});
|
||||
|
||||
test("createRouterEvalArtifact wraps reports with a stable schema", () => {
|
||||
const report = runRouterEval([
|
||||
obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }),
|
||||
]);
|
||||
|
||||
const artifact = createRouterEvalArtifact(report);
|
||||
|
||||
assert.equal(artifact.schemaVersion, 1);
|
||||
assert.equal(artifact.kind, "router-eval-report");
|
||||
assert.equal(artifact.generatedAt, report.evaluatedAt);
|
||||
assert.equal(artifact.report?.summary.totalSamples, 1);
|
||||
});
|
||||
Reference in New Issue
Block a user