From f8e56e46152da7f54e44168da073723490f94ca2 Mon Sep 17 00:00:00 2001 From: KooshaPari <42529354+KooshaPari@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:13:39 -0700 Subject: [PATCH] fix(router-eval): retained-optimization gate cleanup (#7318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(eval): add router-eval harness (AIQ scoring, regression gate, Pareto search) Extracts a standalone router-eval evaluation tool that replays routing decisions (from NDJSON corpora or the usage_history/call_logs SQLite tables) into an AIQ (success/latency/cost) score, compares baseline vs. candidate router configs with a retained-run regression gate, and ranks Pareto-optimal candidates across a search space — a sibling to the existing eval:compression harness. New scripts: scripts/router-eval/{index,compare,patch-compare,search, trends}.ts, scripts/check/check-router-eval-regression.ts, and src/lib/routerEval/index.ts, wired via 6 new package.json entries (eval:router, eval:router:compare, eval:router:patch-compare, eval:router:search, eval:router:trends, check:router-eval). Reconstructed onto current release/v3.8.49 from the original ~142-commit stale PR branch: only the genuinely new router-eval payload (17 files) was extracted — the other ~560 changed files in the original diff were base-drift already present on release in newer form. The new package.json scripts now invoke `node --import tsx` instead of `bun`, matching the `eval:compression` precedent (Bun is reserved for a closed 5-script allowlist). The runtime-detection shim (`"Bun" in globalThis`) already present in the harness gracefully falls back to better-sqlite3 under Node, so no logic changes were needed there; the retained-run manifest's previously-hardcoded `runtime: "bun"` field and matching CLI help text were corrected to reflect the actual invocation. All 27 existing router-eval unit tests pass unchanged. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> * refactor(router-eval): decompose toRouterObservation below complexity gate toRouterObservation had cyclomatic complexity 18 (gate max is 15). Extract the per-field parsing/normalization into pure helpers (sampleId, model fields, latency, cost derivation, success) so the entry point is a plain sequential assembly of a RouterObservation. Behavior is unchanged — same tests pass, same fallbacks, same precedence between input aliases. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --------- Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .../features/7318-router-eval-harness.md | 1 + package.json | 6 + scripts/check/check-router-eval-regression.ts | 287 +++++++++++ scripts/router-eval/compare.ts | 135 +++++ scripts/router-eval/index.ts | 483 +++++++++++++++++ scripts/router-eval/patch-compare.ts | 315 ++++++++++++ scripts/router-eval/search.ts | 439 ++++++++++++++++ scripts/router-eval/trends.ts | 198 +++++++ src/lib/routerEval/index.ts | 428 ++++++++++++++++ tests/fixtures/router-eval/baseline.ndjson | 2 + tests/fixtures/router-eval/candidate.ndjson | 2 + tests/unit/router-eval-check.test.ts | 484 ++++++++++++++++++ tests/unit/router-eval-cli.test.ts | 270 ++++++++++ tests/unit/router-eval-compare.test.ts | 83 +++ tests/unit/router-eval-e2e-chain.test.ts | 135 +++++ tests/unit/router-eval-patch-compare.test.ts | 312 +++++++++++ tests/unit/router-eval-search.test.ts | 258 ++++++++++ tests/unit/router-eval-trends.test.ts | 128 +++++ tests/unit/router-eval.test.ts | 92 ++++ 19 files changed, 4058 insertions(+) create mode 100644 changelog.d/features/7318-router-eval-harness.md create mode 100644 scripts/check/check-router-eval-regression.ts create mode 100644 scripts/router-eval/compare.ts create mode 100644 scripts/router-eval/index.ts create mode 100644 scripts/router-eval/patch-compare.ts create mode 100644 scripts/router-eval/search.ts create mode 100644 scripts/router-eval/trends.ts create mode 100644 src/lib/routerEval/index.ts create mode 100644 tests/fixtures/router-eval/baseline.ndjson create mode 100644 tests/fixtures/router-eval/candidate.ndjson create mode 100644 tests/unit/router-eval-check.test.ts create mode 100644 tests/unit/router-eval-cli.test.ts create mode 100644 tests/unit/router-eval-compare.test.ts create mode 100644 tests/unit/router-eval-e2e-chain.test.ts create mode 100644 tests/unit/router-eval-patch-compare.test.ts create mode 100644 tests/unit/router-eval-search.test.ts create mode 100644 tests/unit/router-eval-trends.test.ts create mode 100644 tests/unit/router-eval.test.ts diff --git a/changelog.d/features/7318-router-eval-harness.md b/changelog.d/features/7318-router-eval-harness.md new file mode 100644 index 0000000000..f88326e9e0 --- /dev/null +++ b/changelog.d/features/7318-router-eval-harness.md @@ -0,0 +1 @@ +- **feat(eval):** added a router-eval harness (`npm run eval:router`, `eval:router:compare`, `eval:router:patch-compare`, `eval:router:search`, `eval:router:trends`, `check:router-eval`) that replays routing decisions — from NDJSON corpora or the `usage_history`/`call_logs` SQLite tables — into an AIQ (success/latency/cost) score, compares baseline vs. candidate router configs with a retained-run regression gate, and ranks Pareto-optimal candidates; a sibling tool to the existing `eval:compression` harness (#7318 — thanks @KooshaPari). diff --git a/package.json b/package.json index b92584773d..fa272ed3d6 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,11 @@ "gen:provider-reference": "bun scripts/docs/gen-provider-reference.ts", "bench:compression": "bun scripts/compression/benchmark.ts", "eval:compression": "node --import tsx scripts/compression-eval/index.ts", + "eval:router": "node --import tsx scripts/router-eval/index.ts", + "eval:router:compare": "node --import tsx scripts/router-eval/compare.ts", + "eval:router:patch-compare": "node --import tsx scripts/router-eval/patch-compare.ts", + "eval:router:search": "node --import tsx scripts/router-eval/search.ts", + "eval:router:trends": "node --import tsx scripts/router-eval/trends.ts", "release:sync-changelog-i18n": "node scripts/release/sync-changelog-i18n.mjs", "build": "node scripts/build/build-next-isolated.mjs", "build:secure": "OMNIROUTE_BUILD_PROFILE=minimal node scripts/build/build-next-isolated.mjs", @@ -118,6 +123,7 @@ "check:docs-counts": "node scripts/check/check-docs-counts-sync.mjs", "check:deprecated-versions": "node scripts/check/check-deprecated-versions.mjs", "check:compression-budget": "bun scripts/check/check-compression-budget.ts", + "check:router-eval": "node --import tsx scripts/check/check-router-eval-regression.ts", "check:doc-links": "node scripts/check/check-doc-links.mjs", "check:fabricated-docs": "node scripts/check/check-fabricated-docs.mjs --strict", "check:docs-all": "npm run check:docs-sync && npm run check:docs-counts && npm run check:env-doc-sync && npm run check:deprecated-versions && npm run check:doc-links && npm run check:fabricated-docs", diff --git a/scripts/check/check-router-eval-regression.ts b/scripts/check/check-router-eval-regression.ts new file mode 100644 index 0000000000..46d696d6b8 --- /dev/null +++ b/scripts/check/check-router-eval-regression.ts @@ -0,0 +1,287 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +type Args = { + baseline: string; + candidate: string; + baselinePatch?: string; + candidatePatch?: string; + output: string; + jsonOutput: string; + patchOutput: string; + patchJsonOutput: string; + artifactDir?: string; + runId: string; + maxAiqDrop: string; + maxCostIncrease: string; + maxPatchAiqDrop: string; + maxPatchCostIncrease: string; + maxPatchLatencyIncrease: string; + maxPatchRegressionIncrease: string; +}; + +type GateManifest = { + schemaVersion: 1; + kind: "router-eval-gate-run"; + runId: string; + generatedAt: string; + command: string[]; + thresholds: { + maxAiqDrop: number; + maxCostIncrease: number; + patch?: { + maxAiqDrop: number; + maxCostIncrease: number; + maxLatencyIncrease: number; + maxRegressionIncrease: number; + }; + }; + inputs: { + baseline: string; + candidate: string; + baselinePatch?: string; + candidatePatch?: string; + }; + outputs: { + markdown: string; + json: string; + patchMarkdown?: string; + patchJson?: string; + }; + environment: { + runtime: "bun" | "node"; + platform: NodeJS.Platform; + }; + result: { + status: number; + }; +}; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const isBunRuntime = "Bun" in globalThis; + +function runTypeScriptScript(args: string[]) { + return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} +const defaultFixtureDir = path.join(repoRoot, "tests/fixtures/router-eval"); +const defaultArtifactDir = path.join(os.tmpdir(), "omniroute-router-eval"); + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function readArgs(): Args { + const artifactDir = getArgValue("artifact-dir"); + const runId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-"); + const retainedDir = artifactDir ? path.join(path.resolve(artifactDir), runId) : undefined; + return { + baseline: getArgValue("baseline") ?? path.join(defaultFixtureDir, "baseline.ndjson"), + candidate: getArgValue("candidate") ?? path.join(defaultFixtureDir, "candidate.ndjson"), + baselinePatch: getArgValue("baseline-patch"), + candidatePatch: getArgValue("candidate-patch"), + output: getArgValue("output") ?? path.join(retainedDir ?? defaultArtifactDir, "router-eval.md"), + jsonOutput: + getArgValue("json-output") ?? + path.join(retainedDir ?? defaultArtifactDir, "router-eval.json"), + patchOutput: + getArgValue("patch-output") ?? + path.join(retainedDir ?? defaultArtifactDir, "patch-comparison.md"), + patchJsonOutput: + getArgValue("patch-json-output") ?? + path.join(retainedDir ?? defaultArtifactDir, "patch-comparison.json"), + artifactDir, + runId, + maxAiqDrop: getArgValue("max-aiq-drop") ?? "1", + maxCostIncrease: getArgValue("max-cost-increase") ?? "0.05", + maxPatchAiqDrop: getArgValue("max-patch-aiq-drop") ?? "1", + maxPatchCostIncrease: getArgValue("max-patch-cost-increase") ?? "0.05", + maxPatchLatencyIncrease: getArgValue("max-patch-latency-increase") ?? "0.05", + maxPatchRegressionIncrease: getArgValue("max-patch-regression-increase") ?? "0", + }; +} + +function ensureReadable(filePath: string, label: string): void { + if (!fs.existsSync(filePath)) { + console.error(`[router-eval] ${label} missing: ${filePath}`); + process.exit(2); + } +} + +function writeRetainedRun(args: Args, status: number): void { + if (!args.artifactDir) return; + + const runDir = path.join(path.resolve(args.artifactDir), args.runId); + const inputDir = path.join(runDir, "inputs"); + fs.mkdirSync(inputDir, { recursive: true }); + + const baselineCopy = path.join(inputDir, "baseline.ndjson"); + const candidateCopy = path.join(inputDir, "candidate.ndjson"); + fs.copyFileSync(args.baseline, baselineCopy); + fs.copyFileSync(args.candidate, candidateCopy); + const baselinePatchCopy = args.baselinePatch + ? path.join(inputDir, "baseline.patch.json") + : undefined; + const candidatePatchCopy = args.candidatePatch + ? path.join(inputDir, "candidate.patch.json") + : undefined; + if (args.baselinePatch && baselinePatchCopy) + fs.copyFileSync(args.baselinePatch, baselinePatchCopy); + if (args.candidatePatch && candidatePatchCopy) + fs.copyFileSync(args.candidatePatch, candidatePatchCopy); + + const manifest: GateManifest = { + schemaVersion: 1, + kind: "router-eval-gate-run", + runId: args.runId, + generatedAt: new Date().toISOString(), + command: process.argv.slice(1), + thresholds: { + maxAiqDrop: Number.parseFloat(args.maxAiqDrop), + maxCostIncrease: Number.parseFloat(args.maxCostIncrease), + ...(args.baselinePatch && args.candidatePatch + ? { + patch: { + maxAiqDrop: Number.parseFloat(args.maxPatchAiqDrop), + maxCostIncrease: Number.parseFloat(args.maxPatchCostIncrease), + maxLatencyIncrease: Number.parseFloat(args.maxPatchLatencyIncrease), + maxRegressionIncrease: Number.parseFloat(args.maxPatchRegressionIncrease), + }, + } + : {}), + }, + inputs: { + baseline: path.relative(runDir, baselineCopy), + candidate: path.relative(runDir, candidateCopy), + ...(baselinePatchCopy && candidatePatchCopy + ? { + baselinePatch: path.relative(runDir, baselinePatchCopy), + candidatePatch: path.relative(runDir, candidatePatchCopy), + } + : {}), + }, + outputs: { + markdown: path.relative(runDir, args.output), + json: path.relative(runDir, args.jsonOutput), + ...(args.baselinePatch && args.candidatePatch + ? { + patchMarkdown: path.relative(runDir, args.patchOutput), + patchJson: path.relative(runDir, args.patchJsonOutput), + } + : {}), + }, + environment: { + runtime: isBunRuntime ? "bun" : "node", + platform: process.platform, + }, + result: { + status, + }, + }; + + fs.writeFileSync(path.join(runDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); +} + +function runPatchGate(args: Args): number { + if (!args.baselinePatch && !args.candidatePatch) return 0; + if (!args.baselinePatch || !args.candidatePatch) { + console.error("[router-eval] --baseline-patch and --candidate-patch must be provided together"); + return 2; + } + ensureReadable(args.baselinePatch, "baseline patch"); + ensureReadable(args.candidatePatch, "candidate patch"); + fs.mkdirSync(path.dirname(args.patchOutput), { recursive: true }); + fs.mkdirSync(path.dirname(args.patchJsonOutput), { recursive: true }); + + const result = runTypeScriptScript([ + "scripts/router-eval/patch-compare.ts", + "--baseline", + args.baselinePatch, + "--candidate", + args.candidatePatch, + "--output", + args.patchOutput, + "--json-output", + args.patchJsonOutput, + "--run-id", + args.runId, + "--max-aiq-drop", + args.maxPatchAiqDrop, + "--max-cost-increase", + args.maxPatchCostIncrease, + "--max-latency-increase", + args.maxPatchLatencyIncrease, + "--max-regression-increase", + args.maxPatchRegressionIncrease, + "--fail-on-regression", + ]); + + if (result.error) { + console.error(`[router-eval] failed to launch patch compare: ${result.error.message}`); + return 1; + } + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + return result.status ?? 1; +} + +function main(): void { + const args = readArgs(); + ensureReadable(args.baseline, "baseline corpus"); + ensureReadable(args.candidate, "candidate corpus"); + fs.mkdirSync(path.dirname(args.output), { recursive: true }); + fs.mkdirSync(path.dirname(args.jsonOutput), { recursive: true }); + + const result = runTypeScriptScript([ + "scripts/router-eval/index.ts", + "--input", + args.candidate, + "--baseline-input", + args.baseline, + "--max-aiq-drop", + args.maxAiqDrop, + "--max-cost-increase", + args.maxCostIncrease, + "--output", + args.output, + "--json-output", + args.jsonOutput, + "--fail-on-regression", + ]); + + if (result.error) { + console.error(`[router-eval] failed to launch evaluator: ${result.error.message}`); + process.exit(1); + } + + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + + if (result.status === 0) { + const patchStatus = runPatchGate(args); + writeRetainedRun(args, patchStatus); + if (patchStatus !== 0) { + console.error(`[router-eval] patch gate failed with exit code ${patchStatus}`); + process.exit(patchStatus); + } + const retention = args.artifactDir ? ` retained run ${args.runId}` : " temp run"; + console.log(`[router-eval] OK -${retention}; artifacts: ${args.output}, ${args.jsonOutput}`); + return; + } + + writeRetainedRun(args, result.status ?? 1); + console.error(`[router-eval] regression gate failed with exit code ${result.status ?? 1}`); + process.exit(result.status ?? 1); +} + +main(); diff --git a/scripts/router-eval/compare.ts b/scripts/router-eval/compare.ts new file mode 100644 index 0000000000..31ea7f3677 --- /dev/null +++ b/scripts/router-eval/compare.ts @@ -0,0 +1,135 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +type Args = { + baseline: string; + candidate: string; + baselineName: string; + candidateName: string; + artifactDir: string; + runId: string; + maxAiqDrop: string; + maxCostIncrease: string; + failOnRegression: boolean; +}; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const isBunRuntime = "Bun" in globalThis; + +function runTypeScriptScript(args: string[]) { + return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function usage(): string { + return [ + "Usage:", + " npm run eval:router:compare -- --baseline --candidate ", + " [--baseline-name ] [--candidate-name ] [--artifact-dir ]", + " [--run-id ] [--max-aiq-drop ] [--max-cost-increase ] [--fail-on-regression]", + "", + "Runs a named baseline-vs-candidate router-eval comparison and retains artifacts.", + ].join("\n"); +} + +function requireArg(name: string): string { + const value = getArgValue(name); + if (!value) { + console.error(`Missing required --${name}`); + process.exit(2); + } + return value; +} + +function readArgs(): Args { + const baselineName = getArgValue("baseline-name") ?? "baseline"; + const candidateName = getArgValue("candidate-name") ?? "candidate"; + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + return { + baseline: requireArg("baseline"), + candidate: requireArg("candidate"), + baselineName, + candidateName, + artifactDir: getArgValue("artifact-dir") ?? "artifacts/router-eval/comparisons", + runId: getArgValue("run-id") ?? `${baselineName}-vs-${candidateName}-${timestamp}`, + maxAiqDrop: getArgValue("max-aiq-drop") ?? "1", + maxCostIncrease: getArgValue("max-cost-increase") ?? "0.05", + failOnRegression: process.argv.includes("--fail-on-regression"), + }; +} + +function ensureReadable(filePath: string, label: string): void { + if (!fs.existsSync(filePath)) { + console.error(`[router-eval:compare] ${label} missing: ${filePath}`); + process.exit(2); + } +} + +function main(): void { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(usage()); + return; + } + + const args = readArgs(); + ensureReadable(args.baseline, "baseline corpus"); + ensureReadable(args.candidate, "candidate corpus"); + + const checkArgs = [ + "scripts/check/check-router-eval-regression.ts", + "--baseline", + args.baseline, + "--candidate", + args.candidate, + "--artifact-dir", + args.artifactDir, + "--run-id", + args.runId, + "--max-aiq-drop", + args.maxAiqDrop, + "--max-cost-increase", + args.maxCostIncrease, + ]; + + const result = runTypeScriptScript(checkArgs); + + if (result.error) { + console.error(`[router-eval:compare] failed to launch comparison: ${result.error.message}`); + process.exit(1); + } + + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + + const runDir = path.resolve(args.artifactDir, args.runId); + const labels = { + baselineName: args.baselineName, + candidateName: args.candidateName, + baseline: path.relative(runDir, path.resolve(args.baseline)), + candidate: path.relative(runDir, path.resolve(args.candidate)), + }; + fs.mkdirSync(runDir, { recursive: true }); + fs.writeFileSync(path.join(runDir, "comparison.json"), `${JSON.stringify(labels, null, 2)}\n`); + + if (result.status === 0 || !args.failOnRegression) { + console.log(`[router-eval:compare] artifacts: ${runDir}`); + return; + } + + process.exit(result.status ?? 1); +} + +main(); diff --git a/scripts/router-eval/index.ts b/scripts/router-eval/index.ts new file mode 100644 index 0000000000..6b0b7312e9 --- /dev/null +++ b/scripts/router-eval/index.ts @@ -0,0 +1,483 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; + +import { + compareRouterEvalRuns, + createRouterEvalArtifact, + formatRouterEvalComparison, + formatRouterEvalReport, + runRouterEval, + toRouterObservation, + type RouterEvalArtifact, + type RouterEvalArtifactMetadata, + type RouterObservation, +} from "@/lib/routerEval/index.ts"; +import { SQLITE_FILE } from "@/lib/db/core.ts"; + +type DbCallLogRow = { + id: string; + model: string | null; + requested_model: string | null; + duration: number | null; + tokens_in: number | null; + tokens_out: number | null; + status: number | null; + combo_name: string | null; + provider: string | null; + error_summary: string | null; + timestamp: string | null; + correlation_id: string | null; +}; + +type DbUsageHistoryRow = { + id: number; + provider: string | null; + model: string | null; + tokens_input: number | null; + tokens_output: number | null; + service_tier: string | null; + status: string | null; + success: number | null; + latency_ms: number | null; + error_code: string | null; + combo_strategy: string | null; + timestamp: string | null; +}; + +type DbReplaySource = "auto" | "call-logs" | "usage-history"; + +type SqliteStatement = { + get: (...params: unknown[]) => unknown; + all: (...params: unknown[]) => unknown[]; +}; + +type SqliteDatabase = { + prepare: (sql: string) => SqliteStatement; + close: () => void; +}; + +type ArgSpec = { + input?: string; + db?: string; + dbSource?: DbReplaySource; + baselineInput?: string; + baselineDb?: string; + baselineDbSource?: DbReplaySource; + since?: string; + limit?: number; + aiqDrop?: number; + costIncrease?: number; + output?: string; + jsonOutput?: string; + exportCorpus?: string; + failOnRegression?: boolean; + help?: boolean; +}; + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + if (!value || value.startsWith("--")) return undefined; + return value; +} + +function getNumericArg(name: string): number | undefined { + const value = getArgValue(name); + if (!value) return undefined; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined; +} + +function getFloatArg(name: string): number | undefined { + const value = getArgValue(name); + if (!value) return undefined; + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseArgs(): ArgSpec { + return { + input: getArgValue("input"), + db: getArgValue("db"), + dbSource: parseReplaySource(getArgValue("db-source")), + baselineInput: getArgValue("baseline-input"), + baselineDb: getArgValue("baseline-db"), + baselineDbSource: parseReplaySource(getArgValue("baseline-db-source")), + since: getArgValue("since"), + limit: getNumericArg("limit"), + aiqDrop: getFloatArg("max-aiq-drop"), + costIncrease: getFloatArg("max-cost-increase"), + output: getArgValue("output"), + jsonOutput: getArgValue("json-output"), + exportCorpus: getArgValue("export-corpus"), + failOnRegression: process.argv.includes("--fail-on-regression"), + help: process.argv.includes("--help") || process.argv.includes("-h"), + }; +} + +function usage() { + return [ + "Usage:", + " npm run eval:router -- --input [--since ] [--limit ]", + " npm run eval:router -- --db [path] [--db-source usage-history|call-logs|auto] [--since ] [--limit ]", + " npm run eval:router -- --input --baseline-input ", + " npm run eval:router -- --db --db-source usage-history", + " [--max-aiq-drop ] [--max-cost-increase ] [--fail-on-regression]", + "", + "Options:", + " --input JSONL observation corpus (or omit for stdin)", + " --db [path] Read SQLite rows from the routing-replay source", + " --db-source Source for --db reads (default: auto => call-logs then usage-history)", + " --baseline-input Baseline corpus in JSONL", + " --baseline-db Baseline corpus in SQLite", + " --baseline-db-source Source for baseline DB reads", + " --since Filter rows newer than this value", + " --limit Limit sample count", + " --max-aiq-drop Regression threshold (default: 0)", + " --max-cost-increase Relative increase threshold (default: 0)", + " --output Write report to file", + " --json-output Write machine-readable artifact JSON", + " --export-corpus Write normalized RouterObservation JSONL", + " --fail-on-regression Exit 1 if candidate regresses vs baseline", + ].join("\n"); +} + +function parseInputLine(rawLine: string): RouterObservation | null { + const trimmed = rawLine.trim(); + if (!trimmed) return null; + try { + const parsed = JSON.parse(trimmed); + return toRouterObservation(parsed); + } catch { + return null; + } +} + +async function readJsonl(inputPath?: string): Promise { + let text: string; + if (!inputPath) { + text = await new Response(process.stdin, { duplex: "half" }).text(); + } else { + text = await fs.promises.readFile(path.resolve(inputPath), "utf8"); + } + + const observations: RouterObservation[] = []; + for (const line of text.split(/\r?\n/)) { + const parsed = parseInputLine(line); + if (parsed) observations.push(parsed); + } + return observations; +} + +function estimateCost(tokensIn: unknown, tokensOut: unknown): number { + const inTokens = typeof tokensIn === "number" ? tokensIn : 0; + const outTokens = typeof tokensOut === "number" ? tokensOut : 0; + return Number(((inTokens + outTokens) * 0.000001).toFixed(6)); +} + +function parseReplaySource(rawSource?: string): DbReplaySource { + if (!rawSource) return "auto"; + const normalized = rawSource.toLowerCase(); + if (normalized === "auto") return "auto"; + if (normalized === "usage_history" || normalized === "usage-history") return "usage-history"; + if (normalized === "call_logs" || normalized === "call-logs") return "call-logs"; + throw new Error(`Unsupported db source: ${rawSource}`); +} + +function hasReplayTable(database: SqliteDatabase, tableName: string): boolean { + return Boolean( + database.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(tableName) + ); +} + +function resolveReplaySource( + database: SqliteDatabase, + requestedSource: DbReplaySource +): DbReplaySource { + const hasUsageHistory = hasReplayTable(database, "usage_history"); + const hasCallLogs = hasReplayTable(database, "call_logs"); + + if (requestedSource === "usage-history") { + if (!hasUsageHistory) throw new Error("Table 'usage_history' missing in database"); + return "usage-history"; + } + + if (requestedSource === "call-logs") { + if (!hasCallLogs) throw new Error("Table 'call_logs' missing in database"); + return "call-logs"; + } + + if (requestedSource === "auto") { + if (hasCallLogs) return "call-logs"; + if (hasUsageHistory) return "usage-history"; + } + + throw new Error("No replay table found in database (expected usage_history or call_logs)"); +} + +function toSuccessFromStatus(status: unknown): boolean { + if (typeof status === "number") return status >= 200 && status < 400; + if (typeof status === "string") { + const parsed = Number.parseInt(status, 10); + if (Number.isFinite(parsed)) return parsed >= 200 && parsed < 400; + const normalized = status.trim().toLowerCase(); + if (normalized === "ok" || normalized === "success" || normalized === "true") return true; + } + return false; +} + +function readCallLogDb(db: SqliteDatabase, since?: string, limit?: number): RouterObservation[] { + const queryParts = [ + "SELECT id, model, requested_model, duration, tokens_in, tokens_out, status, combo_name, provider, error_summary, timestamp, correlation_id", + "FROM call_logs", + "WHERE 1=1", + ]; + const params: unknown[] = []; + + if (since) { + queryParts.push("AND timestamp >= ?"); + params.push(since); + } + + queryParts.push("ORDER BY timestamp ASC"); + if (limit) { + queryParts.push("LIMIT ?"); + params.push(limit); + } + + const rows = db.prepare(queryParts.join(" ")).all(...params) as DbCallLogRow[]; + + const observations: RouterObservation[] = []; + for (const row of rows) { + const mapped = toRouterObservation({ + sampleId: row.id, + model: row.model, + requestedModel: row.requested_model, + latency: row.duration ?? 0, + costUsd: estimateCost(row.tokens_in, row.tokens_out), + configId: row.combo_name || row.provider || "default", + success: row.status != null && row.status >= 200 && row.status < 400, + status: row.status ?? 0, + error: row.error_summary, + routeInput: { + correlationId: row.correlation_id ?? "", + }, + timestamp: row.timestamp ?? new Date().toISOString(), + }); + if (mapped) observations.push(mapped); + } + return observations; +} + +function readUsageHistoryDb( + db: SqliteDatabase, + since?: string, + limit?: number +): RouterObservation[] { + const queryParts = [ + "SELECT id, provider, model, tokens_input, tokens_output, service_tier, status, success, latency_ms, error_code, combo_strategy, timestamp", + "FROM usage_history", + "WHERE 1=1", + ]; + const params: unknown[] = []; + + if (since) { + queryParts.push("AND timestamp >= ?"); + params.push(since); + } + + queryParts.push("ORDER BY timestamp ASC"); + if (limit) { + queryParts.push("LIMIT ?"); + params.push(limit); + } + + const rows = db.prepare(queryParts.join(" ")).all(...params) as DbUsageHistoryRow[]; + + const observations: RouterObservation[] = []; + for (const row of rows) { + const cost = estimateCost(row.tokens_input, row.tokens_output); + const rowId = `${row.id}`; + const mapped = toRouterObservation({ + sampleId: rowId, + model: row.model, + requestedModel: row.model, + latency: row.latency_ms ?? 0, + costUsd: cost, + configId: row.combo_strategy || row.provider || "default", + success: toSuccessFromStatus(row.status) || row.success === 1, + status: row.success === 1 ? 200 : 0, + routeInput: {}, + metadata: { + provider: row.provider, + serviceTier: row.service_tier, + errorCode: row.error_code, + }, + timestamp: row.timestamp ?? new Date().toISOString(), + }); + if (mapped) observations.push(mapped); + } + return observations; +} + +async function openSqliteDatabase(sqliteFile: string): Promise { + if ("Bun" in globalThis) { + const sqlite = await import("bun:sqlite"); + return new sqlite.Database(sqliteFile, { readonly: true }); + } + + const sqlite = await import("better-sqlite3"); + return new sqlite.default(sqliteFile, { readonly: true }); +} + +async function readDb( + filePath: string, + since?: string, + limit?: number, + source: DbReplaySource = "auto" +): Promise { + const sqliteFile = filePath || SQLITE_FILE; + if (!sqliteFile) throw new Error("SQLite mode requires a path or SQLITE_FILE"); + const db = await openSqliteDatabase(sqliteFile); + try { + const normalized = parseReplaySource(source); + const activeSource = resolveReplaySource(db, normalized); + if (activeSource === "usage-history") { + return readUsageHistoryDb(db, since, limit); + } + return readCallLogDb(db, since, limit); + } finally { + db.close(); + } +} + +function resolveDbPath(rawArg?: string): string { + if (rawArg) return path.resolve(rawArg); + if (SQLITE_FILE) return SQLITE_FILE; + throw new Error("No SQLITE_FILE and no --db path provided"); +} + +function describeInputSource( + inputPath: string | undefined, + dbPath: string | undefined, + dbSource: DbReplaySource | undefined, + usesDb: boolean +): { source: string; path?: string; dbSource?: string } { + if (inputPath) return { source: "jsonl", path: path.resolve(inputPath) }; + if (usesDb) { + return { + source: "sqlite", + path: resolveDbPath(dbPath), + dbSource: dbSource ?? "auto", + }; + } + return { source: "stdin" }; +} + +function buildArtifactMetadata(args: ArgSpec, hasCandidateDb: boolean): RouterEvalArtifactMetadata { + const hasBaselineDb = Boolean(args.baselineDb); + return { + candidate: describeInputSource(args.input, args.db, args.dbSource, hasCandidateDb), + baseline: + args.baselineInput || hasBaselineDb + ? describeInputSource( + args.baselineInput, + args.baselineDb, + args.baselineDbSource, + hasBaselineDb + ) + : undefined, + window: { + since: args.since, + limit: args.limit, + }, + thresholds: { + maxAiqDrop: args.aiqDrop ?? 0, + maxCostIncrease: args.costIncrease ?? 0, + }, + outputs: { + markdown: args.output ? path.resolve(args.output) : undefined, + json: args.jsonOutput ? path.resolve(args.jsonOutput) : undefined, + corpus: args.exportCorpus ? path.resolve(args.exportCorpus) : undefined, + }, + }; +} + +async function writeCorpus(pathArg: string, observations: RouterObservation[]): Promise { + const outPath = path.resolve(pathArg); + await fs.promises.mkdir(path.dirname(outPath), { recursive: true }); + const lines = observations.map((observation) => JSON.stringify(observation)); + await fs.promises.writeFile(outPath, `${lines.join("\n")}\n`, "utf8"); +} + +async function run() { + const args = parseArgs(); + if (args.help) { + console.log(usage()); + return; + } + + const hasCandidateDb = Boolean(args.db || process.argv.includes("--db")); + const candidate: RouterObservation[] = args.input + ? await readJsonl(args.input) + : hasCandidateDb + ? await readDb(resolveDbPath(args.db), args.since, args.limit, args.dbSource) + : await readJsonl(); + + const baseline: RouterObservation[] | undefined = args.baselineInput + ? await readJsonl(args.baselineInput) + : args.baselineDb + ? await readDb(resolveDbPath(args.baselineDb), args.since, args.limit, args.baselineDbSource) + : undefined; + + if (candidate.length === 0) { + console.error("No candidate observations found"); + process.exitCode = 2; + return; + } + + if (args.exportCorpus) { + await writeCorpus(args.exportCorpus, candidate); + } + + const report = runRouterEval(candidate); + const metadata = buildArtifactMetadata(args, hasCandidateDb); + let output = formatRouterEvalReport(report); + let artifact: RouterEvalArtifact = createRouterEvalArtifact(report, metadata); + + if (baseline && baseline.length > 0) { + const comparison = compareRouterEvalRuns(runRouterEval(baseline), report, { + aiqDrop: args.aiqDrop ?? 0, + relativeCostIncrease: args.costIncrease ?? 0, + }); + output = formatRouterEvalComparison(comparison); + artifact = createRouterEvalArtifact(comparison, metadata); + console.log(output); + if (args.failOnRegression && comparison.regressions.length > 0) { + process.exitCode = 1; + } + } else { + console.log(output); + } + + if (args.output) { + const outPath = path.resolve(args.output); + await fs.promises.writeFile(outPath, output, "utf8"); + } + + if (args.jsonOutput) { + const outPath = path.resolve(args.jsonOutput); + await fs.promises.writeFile(outPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8"); + } +} + +run().catch((error) => { + if (error && typeof error === "object" && "message" in error) { + console.error((error as Error).message); + } else { + console.error(String(error)); + } + process.exitCode = 1; +}); diff --git a/scripts/router-eval/patch-compare.ts b/scripts/router-eval/patch-compare.ts new file mode 100644 index 0000000000..32fe80d902 --- /dev/null +++ b/scripts/router-eval/patch-compare.ts @@ -0,0 +1,315 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; + +type PatchOperation = { + op?: string; + path?: string; + value?: string; + evidence?: { + aiq?: number; + avgCostUsd?: number; + avgLatencyMs?: number; + regressions?: number; + }; + rationale?: string; +}; + +type RouterConfigPatchArtifact = { + schemaVersion?: number; + kind?: string; + generatedAt?: string; + applyPolicy?: string; + source?: { + objective?: string; + runId?: string; + artifactPath?: string; + }; + operations?: PatchOperation[]; +}; + +type PatchComparison = { + schemaVersion: 1; + kind: "router-config-patch-comparison"; + generatedAt: string; + runId: string; + thresholds: PatchThresholds; + baseline: PatchSummary; + candidate: PatchSummary; + delta: { + aiq: number; + avgCostUsd: number; + costIncreaseRatio: number; + avgLatencyMs: number; + latencyIncreaseRatio: number; + regressions: number; + }; + changedRecommendation: boolean; + regressions: string[]; + result: { + passed: boolean; + status: 0 | 1; + }; +}; + +type PatchThresholds = { + maxAiqDrop: number; + maxCostIncrease: number; + maxLatencyIncrease: number; + maxRegressionIncrease: number; +}; + +type PatchSummary = { + name: string; + file: string; + objective: string; + runId: string; + recommendedConfigId: string; + aiq: number; + avgCostUsd: number; + avgLatencyMs: number; + regressions: number; + applyPolicy: string; +}; + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function usage(): string { + return [ + "Usage:", + " npm run eval:router:patch-compare -- --baseline --candidate ", + " [--baseline-name ] [--candidate-name ] [--artifact-dir ] [--run-id ]", + " [--output ] [--json-output ] [--fail-on-regression]", + " [--max-aiq-drop ] [--max-cost-increase ] [--max-latency-increase ]", + " [--max-regression-increase ]", + "", + "Compares two retained router config patch proposals without applying them.", + ].join("\n"); +} + +function getNumberArg(name: string, fallback: number): number { + const value = getArgValue(name); + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + console.error(`Invalid --${name} value: ${value}`); + process.exit(2); + } + return parsed; +} + +function requireArg(name: string): string { + const value = getArgValue(name); + if (!value) { + console.error(`Missing required --${name}`); + process.exit(2); + } + return value; +} + +function readPatch(file: string): RouterConfigPatchArtifact { + if (!fs.existsSync(file)) { + console.error(`[router-eval:patch-compare] patch file missing: ${file}`); + process.exit(2); + } + try { + return JSON.parse(fs.readFileSync(file, "utf8")) as RouterConfigPatchArtifact; + } catch (error) { + console.error( + `[router-eval:patch-compare] invalid JSON in ${file}: ${(error as Error).message}` + ); + process.exit(2); + } +} + +function requireNumber(value: unknown, field: string, file: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + console.error(`[router-eval:patch-compare] invalid numeric evidence field ${field} in ${file}`); + process.exit(2); + } + return value; +} + +function summarizePatch( + name: string, + file: string, + patch: RouterConfigPatchArtifact +): PatchSummary { + if (patch.kind !== "router-config-patch") { + console.error( + `[router-eval:patch-compare] invalid patch kind in ${file}: ${patch.kind ?? "missing"}` + ); + process.exit(2); + } + const operation = patch.operations?.[0]; + if (operation?.op !== "recommend-router-config") { + console.error( + `[router-eval:patch-compare] missing recommend-router-config operation in ${file}` + ); + process.exit(2); + } + if (typeof operation.value !== "string" || operation.value.length === 0) { + console.error(`[router-eval:patch-compare] invalid recommended config value in ${file}`); + process.exit(2); + } + return { + name, + file: path.resolve(file), + objective: patch.source?.objective ?? "unknown", + runId: patch.source?.runId ?? "unknown", + recommendedConfigId: operation.value, + aiq: requireNumber(operation.evidence?.aiq, "aiq", file), + avgCostUsd: requireNumber(operation.evidence?.avgCostUsd, "avgCostUsd", file), + avgLatencyMs: requireNumber(operation.evidence?.avgLatencyMs, "avgLatencyMs", file), + regressions: requireNumber(operation.evidence?.regressions, "regressions", file), + applyPolicy: patch.applyPolicy ?? "unknown", + }; +} + +function increaseRatio(delta: number, baseline: number): number { + if (baseline === 0) return delta > 0 ? Number.POSITIVE_INFINITY : 0; + return delta / baseline; +} + +function findRegressions( + baseline: PatchSummary, + candidate: PatchSummary, + thresholds: PatchThresholds +): string[] { + const aiqDrop = baseline.aiq - candidate.aiq; + const costDelta = candidate.avgCostUsd - baseline.avgCostUsd; + const latencyDelta = candidate.avgLatencyMs - baseline.avgLatencyMs; + const regressionDelta = candidate.regressions - baseline.regressions; + const regressions: string[] = []; + if (aiqDrop > thresholds.maxAiqDrop) regressions.push(`AIQ dropped by ${aiqDrop.toFixed(3)}`); + if (increaseRatio(costDelta, baseline.avgCostUsd) > thresholds.maxCostIncrease) { + regressions.push( + `average cost increased by ${increaseRatio(costDelta, baseline.avgCostUsd).toFixed(3)}` + ); + } + if (increaseRatio(latencyDelta, baseline.avgLatencyMs) > thresholds.maxLatencyIncrease) { + regressions.push( + `average latency increased by ${increaseRatio(latencyDelta, baseline.avgLatencyMs).toFixed(3)}` + ); + } + if (regressionDelta > thresholds.maxRegressionIncrease) { + regressions.push(`regression count increased by ${regressionDelta}`); + } + return regressions; +} + +function comparePatches( + runId: string, + thresholds: PatchThresholds, + failOnRegression: boolean, + baseline: PatchSummary, + candidate: PatchSummary +): PatchComparison { + const regressions = findRegressions(baseline, candidate, thresholds); + const status = failOnRegression && regressions.length > 0 ? 1 : 0; + const costDelta = candidate.avgCostUsd - baseline.avgCostUsd; + const latencyDelta = candidate.avgLatencyMs - baseline.avgLatencyMs; + return { + schemaVersion: 1, + kind: "router-config-patch-comparison", + generatedAt: new Date().toISOString(), + runId, + thresholds, + baseline, + candidate, + delta: { + aiq: candidate.aiq - baseline.aiq, + avgCostUsd: costDelta, + costIncreaseRatio: increaseRatio(costDelta, baseline.avgCostUsd), + avgLatencyMs: latencyDelta, + latencyIncreaseRatio: increaseRatio(latencyDelta, baseline.avgLatencyMs), + regressions: candidate.regressions - baseline.regressions, + }, + changedRecommendation: baseline.recommendedConfigId !== candidate.recommendedConfigId, + regressions, + result: { + passed: regressions.length === 0, + status, + }, + }; +} + +function formatComparison(comparison: PatchComparison): string { + return [ + "# Router Config Patch Comparison", + "", + `Passed: ${comparison.result.passed ? "yes" : "no"}`, + `Changed recommendation: ${comparison.changedRecommendation ? "yes" : "no"}`, + ...(comparison.regressions.length > 0 + ? ["", "## Regressions", "", ...comparison.regressions.map((item) => `- ${item}`)] + : []), + "", + "| Side | Name | Objective | Recommended Config | AIQ | Avg Cost | Avg Latency | Regressions | Apply Policy |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- |", + formatSummaryRow("Baseline", comparison.baseline), + formatSummaryRow("Candidate", comparison.candidate), + "", + "| Delta | AIQ | Avg Cost | Avg Latency | Regressions |", + "| --- | ---: | ---: | ---: | ---: |", + `| Candidate - Baseline | ${comparison.delta.aiq.toFixed(3)} | $${comparison.delta.avgCostUsd.toFixed(6)} | ${comparison.delta.avgLatencyMs.toFixed(2)}ms | ${comparison.delta.regressions} |`, + "", + ].join("\n"); +} + +function formatSummaryRow(side: string, summary: PatchSummary): string { + return `| ${side} | ${summary.name} | ${summary.objective} | ${summary.recommendedConfigId} | ${summary.aiq.toFixed(3)} | $${summary.avgCostUsd.toFixed(6)} | ${summary.avgLatencyMs.toFixed(2)}ms | ${summary.regressions} | ${summary.applyPolicy} |`; +} + +function main(): void { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(usage()); + return; + } + + const baselineFile = requireArg("baseline"); + const candidateFile = requireArg("candidate"); + const baselineName = getArgValue("baseline-name") ?? "baseline"; + const candidateName = getArgValue("candidate-name") ?? "candidate"; + const runId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-"); + const thresholds: PatchThresholds = { + maxAiqDrop: getNumberArg("max-aiq-drop", Number.POSITIVE_INFINITY), + maxCostIncrease: getNumberArg("max-cost-increase", Number.POSITIVE_INFINITY), + maxLatencyIncrease: getNumberArg("max-latency-increase", Number.POSITIVE_INFINITY), + maxRegressionIncrease: getNumberArg("max-regression-increase", Number.POSITIVE_INFINITY), + }; + const failOnRegression = process.argv.includes("--fail-on-regression"); + const baseline = summarizePatch(baselineName, baselineFile, readPatch(baselineFile)); + const candidate = summarizePatch(candidateName, candidateFile, readPatch(candidateFile)); + const comparison = comparePatches(runId, thresholds, failOnRegression, baseline, candidate); + const markdown = formatComparison(comparison); + const artifactDir = getArgValue("artifact-dir"); + const output = getArgValue("output"); + const jsonOutput = getArgValue("json-output"); + + if (artifactDir) { + const runDir = path.resolve(artifactDir, runId); + fs.mkdirSync(runDir, { recursive: true }); + fs.writeFileSync(path.join(runDir, "patch-comparison.md"), markdown); + fs.writeFileSync( + path.join(runDir, "patch-comparison.json"), + `${JSON.stringify(comparison, null, 2)}\n` + ); + } + if (output) { + fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true }); + fs.writeFileSync(output, markdown); + } + if (jsonOutput) { + fs.mkdirSync(path.dirname(path.resolve(jsonOutput)), { recursive: true }); + fs.writeFileSync(jsonOutput, `${JSON.stringify(comparison, null, 2)}\n`); + } + console.log(markdown); + process.exit(comparison.result.status); +} + +main(); diff --git a/scripts/router-eval/search.ts b/scripts/router-eval/search.ts new file mode 100644 index 0000000000..1683585e55 --- /dev/null +++ b/scripts/router-eval/search.ts @@ -0,0 +1,439 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import type { RouterConfigAggregate, RouterEvalArtifact } from "@/lib/routerEval/index.ts"; + +type Candidate = { + name: string; + path: string; +}; + +type SearchResult = { + candidateName: string; + configId: string; + runId: string; + aiq: number; + avgCostUsd: number; + avgLatencyMs: number; + regressions: number; + artifactPath: string; +}; + +type SearchObjective = "balanced" | "quality" | "cost" | "latency"; + +type SearchRecommendation = SearchResult & { + objective: SearchObjective; + rank: number; + rationale: string; +}; + +type RouterConfigSuggestion = { + schemaVersion: 1; + kind: "router-config-suggestion"; + generatedAt: string; + objective: SearchObjective; + recommendedConfigId: string; + sourceRunId: string; + sourceArtifactPath: string; + evidence: { + aiq: number; + avgCostUsd: number; + avgLatencyMs: number; + regressions: number; + }; + applyPolicy: "manual-review"; + rationale: string; +}; + +type RouterConfigPatchArtifact = { + schemaVersion: 1; + kind: "router-config-patch"; + generatedAt: string; + applyPolicy: "manual-review"; + source: { + objective: SearchObjective; + runId: string; + artifactPath: string; + }; + operations: Array<{ + op: "recommend-router-config"; + path: "/router/recommendedConfigId"; + value: string; + evidence: RouterConfigSuggestion["evidence"]; + rationale: string; + }>; +}; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const isBunRuntime = "Bun" in globalThis; + +function runTypeScriptScript(args: string[]) { + return spawnSync(process.execPath, isBunRuntime ? args : ["--import", "tsx", ...args], { + cwd: repoRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function getArgValues(name: string): string[] { + const values: string[] = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] !== `--${name}`) continue; + const value = process.argv[index + 1]; + if (value && !value.startsWith("--")) values.push(value); + } + return values; +} + +function usage(): string { + return [ + "Usage:", + " npm run eval:router:search -- --baseline ", + " --candidate [--candidate ...]", + " [--objective balanced|quality|cost|latency]", + " [--artifact-dir ] [--run-id ] [--max-aiq-drop ] [--max-cost-increase ]", + "", + "Ranks candidate corpora by router-eval AIQ while retaining comparison artifacts.", + ].join("\n"); +} + +function requireArg(name: string): string { + const value = getArgValue(name); + if (!value) { + console.error(`Missing required --${name}`); + process.exit(2); + } + return value; +} + +function parseCandidate(raw: string): Candidate { + const splitAt = raw.indexOf("="); + if (splitAt <= 0 || splitAt === raw.length - 1) { + console.error(`Invalid --candidate value: ${raw}. Expected name=path.ndjson`); + process.exit(2); + } + return { + name: raw.slice(0, splitAt), + path: raw.slice(splitAt + 1), + }; +} + +function ensureReadable(filePath: string, label: string): void { + if (!fs.existsSync(filePath)) { + console.error(`[router-eval:search] ${label} missing: ${filePath}`); + process.exit(2); + } +} + +function readArtifact(filePath: string): RouterEvalArtifact { + return JSON.parse(fs.readFileSync(filePath, "utf8")) as RouterEvalArtifact; +} + +function parseObjective(value: string | undefined): SearchObjective { + if (!value) return "balanced"; + if (value === "balanced" || value === "quality" || value === "cost" || value === "latency") { + return value; + } + console.error( + `Invalid --objective value: ${value}. Expected balanced, quality, cost, or latency.` + ); + process.exit(2); +} + +function compareConfigsByObjective( + objective: SearchObjective, + a: RouterConfigAggregate, + b: RouterConfigAggregate +): number { + if (objective === "cost") { + return a.avgCostUsd - b.avgCostUsd || b.aiq - a.aiq || a.avgLatencyMs - b.avgLatencyMs; + } + if (objective === "latency") { + return a.avgLatencyMs - b.avgLatencyMs || b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd; + } + return b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd || a.avgLatencyMs - b.avgLatencyMs; +} + +function selectBestConfig( + artifact: RouterEvalArtifact, + objective: SearchObjective +): RouterConfigAggregate | undefined { + const configs = + artifact.comparison?.candidate.configurations ?? + artifact.report?.configurations ?? + artifact.report?.top ?? + []; + return [...configs].sort((a, b) => compareConfigsByObjective(objective, a, b))[0]; +} + +function resultFromArtifact( + candidateName: string, + runId: string, + artifactPath: string, + objective: SearchObjective +): SearchResult { + const artifact = readArtifact(artifactPath); + const best = selectBestConfig(artifact, objective); + if (!best) { + throw new Error(`No best candidate found in ${artifactPath}`); + } + return { + candidateName, + configId: best.configId, + runId, + aiq: best.aiq, + avgCostUsd: best.avgCostUsd, + avgLatencyMs: best.avgLatencyMs, + regressions: artifact.comparison?.regressions.length ?? 0, + artifactPath, + }; +} + +function formatSearch(results: SearchResult[]): string { + const lines = [ + "# Router Eval Search", + "", + "| Rank | Candidate | AIQ | Avg Cost | Avg Latency | Regressions | Run |", + "| ---: | --- | ---: | ---: | ---: | ---: | --- |", + ]; + results.forEach((result, index) => { + lines.push( + `| ${index + 1} | ${result.candidateName} | ${result.aiq.toFixed(3)} | $${result.avgCostUsd.toFixed(6)} | ${result.avgLatencyMs.toFixed(2)}ms | ${result.regressions} | ${result.runId} |` + ); + }); + return `${lines.join("\n")}\n`; +} + +function compareByObjective(objective: SearchObjective, a: SearchResult, b: SearchResult): number { + if (objective === "cost") { + return ( + a.regressions - b.regressions || + a.avgCostUsd - b.avgCostUsd || + b.aiq - a.aiq || + a.avgLatencyMs - b.avgLatencyMs + ); + } + if (objective === "latency") { + return ( + a.regressions - b.regressions || + a.avgLatencyMs - b.avgLatencyMs || + b.aiq - a.aiq || + a.avgCostUsd - b.avgCostUsd + ); + } + if (objective === "quality") { + return ( + b.aiq - a.aiq || + a.regressions - b.regressions || + a.avgCostUsd - b.avgCostUsd || + a.avgLatencyMs - b.avgLatencyMs + ); + } + return ( + b.aiq - a.aiq || + a.regressions - b.regressions || + a.avgCostUsd - b.avgCostUsd || + a.avgLatencyMs - b.avgLatencyMs + ); +} + +function recommendationRationale(objective: SearchObjective, result: SearchResult): string { + if (objective === "cost") { + return `${result.candidateName} has the best cost-first rank with ${result.regressions} regressions and $${result.avgCostUsd.toFixed(6)} average cost.`; + } + if (objective === "latency") { + return `${result.candidateName} has the best latency-first rank with ${result.regressions} regressions and ${result.avgLatencyMs.toFixed(2)}ms average latency.`; + } + if (objective === "quality") { + return `${result.candidateName} has the best quality-first rank with ${result.aiq.toFixed(3)} AIQ.`; + } + return `${result.candidateName} has the best balanced rank with ${result.aiq.toFixed(3)} AIQ, ${result.regressions} regressions, $${result.avgCostUsd.toFixed(6)} average cost, and ${result.avgLatencyMs.toFixed(2)}ms average latency.`; +} + +function createRecommendation( + objective: SearchObjective, + results: SearchResult[] +): SearchRecommendation { + const winner = results[0]; + if (!winner) { + throw new Error("Cannot create a recommendation without search results"); + } + return { + ...winner, + objective, + rank: 1, + rationale: recommendationRationale(objective, winner), + }; +} + +function createConfigSuggestion( + generatedAt: string, + recommendation: SearchRecommendation +): RouterConfigSuggestion { + return { + schemaVersion: 1, + kind: "router-config-suggestion", + generatedAt, + objective: recommendation.objective, + recommendedConfigId: recommendation.configId, + sourceRunId: recommendation.runId, + sourceArtifactPath: recommendation.artifactPath, + evidence: { + aiq: recommendation.aiq, + avgCostUsd: recommendation.avgCostUsd, + avgLatencyMs: recommendation.avgLatencyMs, + regressions: recommendation.regressions, + }, + applyPolicy: "manual-review", + rationale: recommendation.rationale, + }; +} + +function createConfigPatch(suggestion: RouterConfigSuggestion): RouterConfigPatchArtifact { + return { + schemaVersion: 1, + kind: "router-config-patch", + generatedAt: suggestion.generatedAt, + applyPolicy: "manual-review", + source: { + objective: suggestion.objective, + runId: suggestion.sourceRunId, + artifactPath: suggestion.sourceArtifactPath, + }, + operations: [ + { + op: "recommend-router-config", + path: "/router/recommendedConfigId", + value: suggestion.recommendedConfigId, + evidence: suggestion.evidence, + rationale: suggestion.rationale, + }, + ], + }; +} + +function formatPatchOperations(patch: RouterConfigPatchArtifact): string { + const lines = [ + "## Patch Operations", + "", + "| Op | Path | Value | AIQ | Avg Cost | Avg Latency | Regressions | Apply Policy |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |", + ]; + for (const operation of patch.operations) { + lines.push( + `| ${operation.op} | ${operation.path} | ${operation.value} | ${operation.evidence.aiq.toFixed(3)} | $${operation.evidence.avgCostUsd.toFixed(6)} | ${operation.evidence.avgLatencyMs.toFixed(2)}ms | ${operation.evidence.regressions} | ${patch.applyPolicy} |` + ); + } + return `${lines.join("\n")}\n`; +} + +function main(): void { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(usage()); + return; + } + + const baseline = requireArg("baseline"); + ensureReadable(baseline, "baseline corpus"); + const candidates = getArgValues("candidate").map(parseCandidate); + if (candidates.length === 0) { + console.error("At least one --candidate is required"); + process.exit(2); + } + + const artifactDir = getArgValue("artifact-dir") ?? "artifacts/router-eval/search"; + const searchId = getArgValue("run-id") ?? new Date().toISOString().replace(/[:.]/g, "-"); + const objective = parseObjective(getArgValue("objective")); + const maxAiqDrop = getArgValue("max-aiq-drop") ?? "1"; + const maxCostIncrease = getArgValue("max-cost-increase") ?? "0.05"; + const searchDir = path.resolve(artifactDir, searchId); + fs.mkdirSync(searchDir, { recursive: true }); + + const results: SearchResult[] = []; + for (const candidate of candidates) { + ensureReadable(candidate.path, `${candidate.name} corpus`); + const runId = `${searchId}-${candidate.name}`; + const result = runTypeScriptScript([ + "scripts/router-eval/compare.ts", + "--baseline", + baseline, + "--candidate", + candidate.path, + "--baseline-name", + "baseline", + "--candidate-name", + candidate.name, + "--artifact-dir", + searchDir, + "--run-id", + runId, + "--max-aiq-drop", + maxAiqDrop, + "--max-cost-increase", + maxCostIncrease, + ]); + if (result.stdout) process.stdout.write(result.stdout); + if (result.stderr) process.stderr.write(result.stderr); + if (result.error) { + console.error( + `[router-eval:search] failed to launch ${candidate.name}: ${result.error.message}` + ); + process.exit(1); + } + if (result.status !== 0) { + console.error( + `[router-eval:search] comparison failed for ${candidate.name} with exit code ${result.status ?? 1}` + ); + process.exit(result.status ?? 1); + } + const artifactPath = path.join(searchDir, runId, "router-eval.json"); + results.push(resultFromArtifact(candidate.name, runId, artifactPath, objective)); + } + + results.sort((a, b) => compareByObjective(objective, a, b)); + + const recommendation = createRecommendation(objective, results); + const generatedAt = new Date().toISOString(); + const suggestion = createConfigSuggestion(generatedAt, recommendation); + const patch = createConfigPatch(suggestion); + const markdown = `${formatSearch(results)}## Recommendation\n\n${recommendation.rationale}\n\n${formatPatchOperations(patch)}`; + const summary = { + schemaVersion: 1, + kind: "router-eval-search", + generatedAt, + baseline: path.resolve(baseline), + objective, + recommendation, + suggestion, + patch, + results, + }; + fs.writeFileSync(path.join(searchDir, "search.md"), markdown); + fs.writeFileSync(path.join(searchDir, "search.json"), `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync( + path.join(searchDir, "recommendation.json"), + `${JSON.stringify(recommendation, null, 2)}\n` + ); + fs.writeFileSync( + path.join(searchDir, "suggestion.json"), + `${JSON.stringify(suggestion, null, 2)}\n` + ); + fs.writeFileSync( + path.join(searchDir, "router-config.patch.json"), + `${JSON.stringify(patch, null, 2)}\n` + ); + console.log(markdown); + console.log(`[router-eval:search] artifacts: ${searchDir}`); +} + +main(); diff --git a/scripts/router-eval/trends.ts b/scripts/router-eval/trends.ts new file mode 100644 index 0000000000..70f277c110 --- /dev/null +++ b/scripts/router-eval/trends.ts @@ -0,0 +1,198 @@ +#!/usr/bin/env node +import fs from "node:fs"; +import path from "node:path"; + +import type { + RouterConfigAggregate, + RouterEvalArtifact, + RouterEvalArtifactMetadata, + RouterEvalComparison, + RouterEvalReport, +} from "@/lib/routerEval/index.ts"; + +type TrendRow = { + runId: string; + generatedAt: string; + kind: RouterEvalArtifact["kind"]; + bestConfig: string; + aiq: number; + avgCostUsd: number; + avgLatencyMs: number; + regressions: number; + source: string; + window: string; +}; + +function getArgValue(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + const value = process.argv[index + 1]; + return value && !value.startsWith("--") ? value : undefined; +} + +function usage(): string { + return [ + "Usage:", + " npm run eval:router:trends -- --artifact-dir [--limit ] [--dashboard]", + "", + "Reads retained router-eval JSON artifacts and prints a markdown trend table or dashboard.", + ].join("\n"); +} + +function readJson(filePath: string): RouterEvalArtifact | null { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")) as RouterEvalArtifact; + } catch { + return null; + } +} + +function bestFromReport(report: RouterEvalReport): RouterConfigAggregate | undefined { + return report.top[0]; +} + +function bestFromComparison(comparison: RouterEvalComparison): RouterConfigAggregate | undefined { + return comparison.candidate.top[0]; +} + +function toTrendRow(runId: string, artifact: RouterEvalArtifact): TrendRow | null { + const best = artifact.comparison + ? bestFromComparison(artifact.comparison) + : artifact.report + ? bestFromReport(artifact.report) + : undefined; + + if (!best) return null; + + return { + runId, + generatedAt: artifact.generatedAt, + kind: artifact.kind, + bestConfig: best.configId, + aiq: best.aiq, + avgCostUsd: best.avgCostUsd, + avgLatencyMs: best.avgLatencyMs, + regressions: artifact.comparison?.regressions.length ?? 0, + source: artifact.metadata?.candidate?.source ?? "unknown", + window: formatWindow(artifact.metadata?.window), + }; +} + +function formatWindow(window: RouterEvalArtifactMetadata["window"]): string { + if (!window || typeof window !== "object") return "all"; + const parts: string[] = []; + if ("since" in window && typeof window.since === "string") parts.push(`since ${window.since}`); + if ("limit" in window && typeof window.limit === "number") parts.push(`limit ${window.limit}`); + return parts.length > 0 ? parts.join(", ") : "all"; +} + +function collectTrendRows(artifactDir: string): TrendRow[] { + if (!fs.existsSync(artifactDir)) return []; + + const rows: TrendRow[] = []; + for (const entry of fs.readdirSync(artifactDir, { withFileTypes: true })) { + const runId = entry.name; + const jsonPath = entry.isDirectory() + ? path.join(artifactDir, runId, "router-eval.json") + : entry.isFile() && entry.name.endsWith(".json") + ? path.join(artifactDir, entry.name) + : ""; + if (!jsonPath) continue; + + const artifact = readJson(jsonPath); + if (!artifact || artifact.schemaVersion !== 1) continue; + const row = toTrendRow(runId.replace(/\.json$/, ""), artifact); + if (row) rows.push(row); + } + + return rows.sort((a, b) => a.generatedAt.localeCompare(b.generatedAt)); +} + +function formatTrend(rows: TrendRow[], limit: number): string { + const limited = rows.slice(-limit); + const lines = [ + "# Router Eval Trends", + "", + "| Run | Kind | Source | Window | Best Config | AIQ | Avg Cost | Avg Latency | Regressions |", + "| --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: |", + ]; + + for (const row of limited) { + lines.push( + `| ${row.runId} | ${row.kind} | ${row.source} | ${row.window} | ${row.bestConfig} | ${row.aiq.toFixed(3)} | $${row.avgCostUsd.toFixed(6)} | ${row.avgLatencyMs.toFixed(2)}ms | ${row.regressions} |` + ); + } + + return `${lines.join("\n")}\n`; +} + +function formatDelta(value: number): string { + if (value > 0) return `+${value.toFixed(3)}`; + return value.toFixed(3); +} + +function average(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function formatDashboard(rows: TrendRow[], limit: number): string { + const limited = rows.slice(-limit); + const latest = limited[limited.length - 1]; + const previous = limited[limited.length - 2]; + const aiqDelta = latest && previous ? latest.aiq - previous.aiq : 0; + const latencyDelta = latest && previous ? latest.avgLatencyMs - previous.avgLatencyMs : 0; + const costDelta = latest && previous ? latest.avgCostUsd - previous.avgCostUsd : 0; + const regressions = limited.reduce((sum, row) => sum + row.regressions, 0); + const lines = [ + "# Router Eval Dashboard", + "", + `Runs: ${limited.length}`, + `Latest: ${latest?.runId ?? "n/a"}`, + `Best config: ${latest?.bestConfig ?? "n/a"}`, + `AIQ: ${latest ? latest.aiq.toFixed(3) : "0.000"} (${formatDelta(aiqDelta)})`, + `Avg latency: ${latest ? latest.avgLatencyMs.toFixed(2) : "0.00"}ms (${formatDelta(latencyDelta)}ms)`, + `Avg cost: $${latest ? latest.avgCostUsd.toFixed(6) : "0.000000"} (${formatDelta(costDelta)})`, + `Window: ${latest?.window ?? "all"}`, + `Source: ${latest?.source ?? "unknown"}`, + `Regression count: ${regressions}`, + "", + "## Rolling Averages", + "", + `AIQ: ${average(limited.map((row) => row.aiq)).toFixed(3)}`, + `Latency: ${average(limited.map((row) => row.avgLatencyMs)).toFixed(2)}ms`, + `Cost: $${average(limited.map((row) => row.avgCostUsd)).toFixed(6)}`, + ]; + + return `${lines.join("\n")}\n`; +} + +function main(): void { + if (process.argv.includes("--help") || process.argv.includes("-h")) { + console.log(usage()); + return; + } + + const artifactDir = getArgValue("artifact-dir"); + if (!artifactDir) { + console.error("Missing required --artifact-dir"); + process.exit(2); + } + + const limit = Number.parseInt(getArgValue("limit") ?? "20", 10); + const rows = collectTrendRows(path.resolve(artifactDir)); + if (rows.length === 0) { + console.error(`No router-eval artifacts found in ${artifactDir}`); + process.exit(2); + } + + const boundedLimit = Number.isFinite(limit) && limit > 0 ? limit : 20; + if (process.argv.includes("--dashboard")) { + console.log(formatDashboard(rows, boundedLimit)); + return; + } + + console.log(formatTrend(rows, boundedLimit)); +} + +main(); diff --git a/src/lib/routerEval/index.ts b/src/lib/routerEval/index.ts new file mode 100644 index 0000000000..9368ae4f78 --- /dev/null +++ b/src/lib/routerEval/index.ts @@ -0,0 +1,428 @@ +type JsonRecord = Record; + +export type RouterObservation = { + sampleId: string; + routeInput: JsonRecord; + configId: string; + selectedModel: string | null; + expectedModel: string | null; + latencyMs: number; + costUsd: number; + success: boolean; + timestamp: string; + metadata: JsonRecord; +}; + +export type RouterObservationInput = { + sampleId?: unknown; + id?: unknown; + routeInput?: unknown; + configId?: unknown; + selectedModel?: unknown; + model?: unknown; + expectedModel?: unknown; + requestedModel?: unknown; + latencyMs?: unknown; + latency?: unknown; + durationMs?: unknown; + duration?: unknown; + costUsd?: unknown; + cost?: unknown; + success?: unknown; + status?: unknown; + error?: unknown; + timestamp?: unknown; + tokens?: { + input?: unknown; + output?: unknown; + }; + metadata?: unknown; +}; + +export type RouterConfigAggregate = { + configId: string; + samples: number; + successRate: number; + avgLatencyMs: number; + p50LatencyMs: number; + p95LatencyMs: number; + avgCostUsd: number; + aiq: number; +}; + +export type RouterEvalReport = { + evaluatedAt: string; + summary: { + totalSamples: number; + validSamples: number; + droppedSamples: number; + uniqueConfigs: number; + }; + configurations: RouterConfigAggregate[]; + frontier: RouterConfigAggregate[]; + top: RouterConfigAggregate[]; +}; + +export type RouterEvalComparison = { + baseline: RouterEvalReport; + candidate: RouterEvalReport; + delta: { + aiq: number; + costUsd: number; + }; + regressions: string[]; +}; + +export type RouterEvalArtifact = { + schemaVersion: 1; + kind: "router-eval-report" | "router-eval-comparison"; + generatedAt: string; + metadata?: RouterEvalArtifactMetadata; + report?: RouterEvalReport; + comparison?: RouterEvalComparison; +}; + +export type RouterEvalArtifactMetadata = { + candidate?: { + source: string; + path?: string; + dbSource?: string; + }; + baseline?: { + source: string; + path?: string; + dbSource?: string; + }; + window?: { + since?: string; + limit?: number; + }; + thresholds?: { + maxAiqDrop: number; + maxCostIncrease: number; + }; + outputs?: { + markdown?: string; + json?: string; + corpus?: string; + }; +}; + +function asRecord(value: unknown): JsonRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as JsonRecord) : {}; +} + +function asString(value: unknown, fallback = ""): string { + return typeof value === "string" && value.trim().length > 0 ? value : fallback; +} + +function asNumber(value: unknown, fallback = 0): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : fallback; + } + return fallback; +} + +function asBoolean(value: unknown, fallback = false): boolean { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + if (value.toLowerCase() === "true") return true; + if (value.toLowerCase() === "false") return false; + } + if (typeof value === "number") return value !== 0; + return fallback; +} + +function percentile(sortedValues: number[], p: number): number { + if (sortedValues.length === 0) return 0; + if (sortedValues.length === 1) return sortedValues[0]; + const bounded = Math.min(1, Math.max(0, p)); + const index = Math.floor((sortedValues.length - 1) * bounded); + return sortedValues[index] ?? 0; +} + +function computeAiq(successRate: number, avgLatencyMs: number, avgCostUsd: number): number { + const latencyPenalty = avgLatencyMs / 1_000; + const costPenalty = avgCostUsd * 1_000; + return Number((successRate * 100 - latencyPenalty - costPenalty).toFixed(3)); +} + +function aggregateValues(values: number[]) { + if (values.length === 0) return { avg: 0, p50: 0, p95: 0 }; + const sorted = [...values].sort((a, b) => a - b); + const sum = sorted.reduce((acc, value) => acc + value, 0); + return { + avg: sum / sorted.length, + p50: percentile(sorted, 0.5), + p95: percentile(sorted, 0.95), + }; +} + +function resolveObservationSampleId(value: RouterObservationInput): string { + return asString(value.sampleId ?? value.id, "").trim(); +} + +function resolveObservationModelFields(value: RouterObservationInput): { + selectedModel: string | null; + expectedModel: string | null; +} { + const selectedModel = asString(value.selectedModel ?? value.model); + const expectedModel = asString(value.expectedModel ?? value.requestedModel, null); + return { selectedModel: selectedModel || null, expectedModel }; +} + +function resolveObservationLatencyMs(value: RouterObservationInput): number { + return asNumber(value.latencyMs ?? value.latency ?? value.durationMs ?? value.duration, 0); +} + +function resolveObservationCostUsd(value: RouterObservationInput): number { + const explicitCost = asNumber(value.costUsd ?? value.cost, NaN); + if (!Number.isNaN(explicitCost)) return explicitCost; + + const promptTokens = asNumber(value.tokens?.input); + const completionTokens = asNumber(value.tokens?.output); + return Number(((promptTokens + completionTokens) * 0.000001).toFixed(6)); +} + +function resolveObservationSuccess(value: RouterObservationInput): boolean { + const status = asNumber(value.status, 500); + const statusIndicatesSuccess = status >= 200 && status < 400; + return asBoolean(value.success, statusIndicatesSuccess && !value.error); +} + +export function toRouterObservation(input: unknown): RouterObservation | null { + if (!input || typeof input !== "object" || Array.isArray(input)) return null; + + const value = input as RouterObservationInput; + const sampleId = resolveObservationSampleId(value); + if (!sampleId) return null; + + const { selectedModel, expectedModel } = resolveObservationModelFields(value); + + return { + sampleId, + routeInput: asRecord(value.routeInput, {}), + configId: asString(value.configId, "default"), + selectedModel, + expectedModel, + latencyMs: resolveObservationLatencyMs(value), + costUsd: resolveObservationCostUsd(value), + success: resolveObservationSuccess(value), + timestamp: asString(value.timestamp, new Date(0).toISOString()), + metadata: asRecord(value.metadata), + }; +} + +export function summarizeRouterObservations(observations: RouterObservation[]) { + const byConfig = new Map(); + for (const obs of observations) { + if (!byConfig.has(obs.configId)) byConfig.set(obs.configId, []); + byConfig.get(obs.configId)?.push(obs); + } + + const configurations: RouterConfigAggregate[] = []; + for (const [configId, rows] of byConfig) { + const latencies = rows.map((row) => Math.max(0, row.latencyMs)); + const costs = rows.map((row) => Math.max(0, row.costUsd)); + const latencyStats = aggregateValues(latencies); + const costStats = aggregateValues(costs); + const successCount = rows.filter((row) => row.success).length; + const successRate = rows.length === 0 ? 0 : successCount / rows.length; + configurations.push({ + configId, + samples: rows.length, + successRate, + avgLatencyMs: latencyStats.avg, + p50LatencyMs: latencyStats.p50, + p95LatencyMs: latencyStats.p95, + avgCostUsd: costStats.avg, + aiq: computeAiq(successRate, latencyStats.avg, costStats.avg), + }); + } + + const top = [...configurations].sort( + (a, b) => + b.aiq - a.aiq || + b.successRate - a.successRate || + a.avgLatencyMs - b.avgLatencyMs || + a.avgCostUsd - b.avgCostUsd + ); + const frontier = computeParetoFrontier(configurations); + + return { + configurations: configurations.sort((a, b) => b.aiq - a.aiq), + frontier, + top, + }; +} + +export function runRouterEval(observations: RouterObservation[]): RouterEvalReport { + const valid = observations.filter((value) => Boolean(value.selectedModel)); + const droppedSamples = observations.length - valid.length; + const { configurations, frontier, top } = summarizeRouterObservations(valid); + + return { + evaluatedAt: new Date().toISOString(), + summary: { + totalSamples: observations.length, + validSamples: valid.length, + droppedSamples, + uniqueConfigs: configurations.length, + }, + configurations, + frontier, + top, + }; +} + +export function computeParetoFrontier(configs: RouterConfigAggregate[]) { + const ordered = [...configs].sort( + (a, b) => a.avgCostUsd - b.avgCostUsd || a.avgLatencyMs - b.avgLatencyMs + ); + const frontier: RouterConfigAggregate[] = []; + + for (const config of ordered) { + const dominated = frontier.some( + (candidate) => + candidate.aiq >= config.aiq && + candidate.avgCostUsd <= config.avgCostUsd && + candidate.avgLatencyMs <= config.avgLatencyMs + ); + if (!dominated) frontier.push(config); + } + + return frontier.sort((a, b) => b.aiq - a.aiq || a.avgCostUsd - b.avgCostUsd); +} + +export function compareRouterEvalRuns( + baseline: RouterEvalReport, + candidate: RouterEvalReport, + thresholds: { + aiqDrop: number; + relativeCostIncrease: number; + } = { aiqDrop: 0, relativeCostIncrease: 0 } +) { + const baselineBest = baseline.top[0]; + const candidateBest = candidate.top[0]; + const regressions: string[] = []; + if (!baselineBest || !candidateBest) { + return { baseline, candidate, delta: { aiq: 0, costUsd: 0 }, regressions }; + } + + const aiq = candidateBest.aiq - baselineBest.aiq; + const baselineCost = baselineBest.avgCostUsd; + const cost = candidateBest.avgCostUsd; + const costIncrease = + baselineCost === 0 ? (cost > 0 ? Infinity : 0) : (cost - baselineCost) / baselineCost; + + if (aiq < -Math.abs(thresholds.aiqDrop)) { + regressions.push(`AIQ dropped by ${(-aiq).toFixed(3)} below threshold`); + } + + if (costIncrease > thresholds.relativeCostIncrease) { + regressions.push(`cost increased by ${(costIncrease * 100).toFixed(2)}% above threshold`); + } + + return { + baseline, + candidate, + delta: { + aiq, + costUsd: costIncrease * baselineCost, + }, + regressions, + }; +} + +export function formatRouterEvalReport(report: RouterEvalReport): string { + const best = report.top[0]; + const lines = [ + "# Router Eval Report", + `Generated: ${report.evaluatedAt}`, + `Samples: ${report.summary.validSamples}/${report.summary.totalSamples} valid, ${report.summary.uniqueConfigs} config(s)`, + "", + "## Top Configurations", + "", + "| Config | Samples | AIQ | Success | Avg Latency | p50 | p95 | Avg Cost |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ]; + + for (const config of report.top.slice(0, 10)) { + lines.push( + `| ${config.configId} | ${config.samples} | ${config.aiq.toFixed(3)} | ${(config.successRate * 100).toFixed(2)}% | ${config.avgLatencyMs.toFixed(2)}ms | ${config.p50LatencyMs.toFixed(2)}ms | ${config.p95LatencyMs.toFixed(2)}ms | $${config.avgCostUsd.toFixed(6)} |` + ); + } + + if (best) { + lines.push(""); + lines.push(`Best: **${best.configId}** (AIQ ${best.aiq.toFixed(3)})`); + } + + if (report.frontier.length > 0) { + lines.push(""); + lines.push("## Pareto Frontier"); + lines.push(""); + lines.push("| Config | AIQ | Cost | Latency |"); + lines.push("| --- | ---: | ---: | ---: |"); + for (const config of report.frontier) { + lines.push( + `| ${config.configId} | ${config.aiq.toFixed(3)} | $${config.avgCostUsd.toFixed(6)} | ${config.avgLatencyMs.toFixed(2)}ms |` + ); + } + } + + return `${lines.join("\n")}\n`; +} + +export function formatRouterEvalComparison(comparison: RouterEvalComparison): string { + const bestCandidate = comparison.candidate.top[0]; + const bestBaseline = comparison.baseline.top[0]; + const passes = comparison.regressions.length === 0; + const lines = [ + "# Router Eval Comparison", + `Passed: ${passes ? "✅" : "❌"}`, + `AIQ delta: ${comparison.delta.aiq.toFixed(3)}`, + `Cost delta: $${comparison.delta.costUsd.toFixed(6)}`, + "", + ]; + + if (bestBaseline && bestCandidate) { + lines.push(`Baseline best: ${bestBaseline.configId} (AIQ ${bestBaseline.aiq.toFixed(3)})`); + lines.push(`Candidate best: ${bestCandidate.configId} (AIQ ${bestCandidate.aiq.toFixed(3)})`); + lines.push(""); + } + + if (comparison.regressions.length > 0) { + lines.push("## Regressions"); + for (const reason of comparison.regressions) { + lines.push(`- ${reason}`); + } + } + + return `${lines.join("\n")}\n`; +} + +export function createRouterEvalArtifact( + value: RouterEvalReport | RouterEvalComparison, + metadata?: RouterEvalArtifactMetadata +): RouterEvalArtifact { + if ("candidate" in value && "baseline" in value) { + return { + schemaVersion: 1, + kind: "router-eval-comparison", + generatedAt: value.candidate.evaluatedAt, + metadata, + comparison: value, + }; + } + + return { + schemaVersion: 1, + kind: "router-eval-report", + generatedAt: value.evaluatedAt, + metadata, + report: value, + }; +} diff --git a/tests/fixtures/router-eval/baseline.ndjson b/tests/fixtures/router-eval/baseline.ndjson new file mode 100644 index 0000000000..ddc07175fc --- /dev/null +++ b/tests/fixtures/router-eval/baseline.ndjson @@ -0,0 +1,2 @@ +{"sampleId":"baseline-1","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":120,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:00:00.000Z"} +{"sampleId":"baseline-2","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":130,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:01:00.000Z"} diff --git a/tests/fixtures/router-eval/candidate.ndjson b/tests/fixtures/router-eval/candidate.ndjson new file mode 100644 index 0000000000..a6e9160375 --- /dev/null +++ b/tests/fixtures/router-eval/candidate.ndjson @@ -0,0 +1,2 @@ +{"sampleId":"candidate-1","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":110,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:00:00.000Z"} +{"sampleId":"candidate-2","configId":"priority","selectedModel":"gpt-4.1","expectedModel":"gpt-4.1","latencyMs":125,"costUsd":0.004,"success":true,"timestamp":"2026-01-01T00:01:00.000Z"} diff --git a/tests/unit/router-eval-check.test.ts b/tests/unit/router-eval-check.test.ts new file mode 100644 index 0000000000..51ae202d03 --- /dev/null +++ b/tests/unit/router-eval-check.test.ts @@ -0,0 +1,484 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const checkScript = "scripts/check/check-router-eval-regression.ts"; + +function writePatch( + file: string, + configId: string, + aiq: number, + costUsd: number, + latencyMs: number +) { + writeFileSync( + file, + `${JSON.stringify( + { + schemaVersion: 1, + kind: "router-config-patch", + generatedAt: "2026-07-03T00:00:00.000Z", + applyPolicy: "manual-review", + source: { + objective: "balanced", + runId: `${configId}-run`, + artifactPath: `/tmp/${configId}/router-eval.json`, + }, + operations: [ + { + op: "recommend-router-config", + path: "/router/recommendedConfigId", + value: configId, + evidence: { + aiq, + avgCostUsd: costUsd, + avgLatencyMs: latencyMs, + regressions: 0, + }, + rationale: `${configId} wins`, + }, + ], + }, + null, + 2 + )}\n` + ); +} + +test("router eval check writes artifacts and passes non-regressing corpora", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-check-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const markdown = join(dir, "router-eval.md"); + const json = join(dir, "router-eval.json"); + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--output", + markdown, + "--json-output", + json, + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok(readFileSync(markdown, "utf8").includes("Router Eval Comparison")); + const artifact = JSON.parse(readFileSync(json, "utf8")) as Record; + assert.equal(artifact.kind, "router-eval-comparison"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval check can include patch compare as a retained gate", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-gate-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const baselinePatch = join(dir, "baseline.patch.json"); + const candidatePatch = join(dir, "candidate.patch.json"); + const artifactDir = join(dir, "artifacts"); + const runId = "patch-gate-001"; + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + writePatch(baselinePatch, "balanced-v1", 92, 0.005, 150); + writePatch(candidatePatch, "balanced-v2", 94, 0.004, 120); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--baseline-patch", + baselinePatch, + "--candidate-patch", + candidatePatch, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok( + readFileSync(join(artifactDir, runId, "patch-comparison.md"), "utf8").includes( + "Router Config Patch Comparison" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "patch-comparison.json"), "utf8").includes( + "router-config-patch-comparison" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "inputs", "baseline.patch.json"), "utf8").includes( + "router-config-patch" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "inputs", "candidate.patch.json"), "utf8").includes( + "router-config-patch" + ) + ); + const manifest = JSON.parse( + readFileSync(join(artifactDir, runId, "manifest.json"), "utf8") + ) as { + inputs?: { baselinePatch?: string; candidatePatch?: string }; + outputs?: { patchMarkdown?: string; patchJson?: string }; + thresholds?: { patch?: { maxLatencyIncrease?: number } }; + }; + assert.equal(manifest.inputs?.baselinePatch, "inputs/baseline.patch.json"); + assert.equal(manifest.inputs?.candidatePatch, "inputs/candidate.patch.json"); + assert.equal(manifest.outputs?.patchMarkdown, "patch-comparison.md"); + assert.equal(manifest.outputs?.patchJson, "patch-comparison.json"); + assert.equal(manifest.thresholds?.patch?.maxLatencyIncrease, 0.05); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval check fails when patch gate regresses beyond thresholds", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-gate-fail-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const baselinePatch = join(dir, "baseline.patch.json"); + const candidatePatch = join(dir, "candidate.patch.json"); + const artifactDir = join(dir, "artifacts"); + const runId = "patch-gate-fail-001"; + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + writePatch(baselinePatch, "balanced-v1", 95, 0.004, 120); + writePatch(candidatePatch, "balanced-v2", 90, 0.008, 200); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--baseline-patch", + baselinePatch, + "--candidate-patch", + candidatePatch, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + "--max-patch-aiq-drop", + "1", + "--max-patch-cost-increase", + "0.1", + "--max-patch-latency-increase", + "0.1", + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 1); + assert.match(result.stderr ?? "", /patch gate failed/); + const comparison = JSON.parse( + readFileSync(join(artifactDir, runId, "patch-comparison.json"), "utf8") + ) as { + result?: { passed?: boolean; status?: number }; + regressions?: string[]; + }; + assert.equal(comparison.result?.passed, false); + assert.equal(comparison.result?.status, 1); + assert.ok((comparison.regressions?.length ?? 0) >= 2); + const manifest = JSON.parse( + readFileSync(join(artifactDir, runId, "manifest.json"), "utf8") + ) as { + result?: { status?: number }; + }; + assert.equal(manifest.result?.status, 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval check rejects unpaired patch inputs", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-patch-unpaired-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const baselinePatch = join(dir, "baseline.patch.json"); + const candidatePatch = join(dir, "candidate.patch.json"); + const artifactDir = join(dir, "artifacts"); + const runId = "unpaired-001"; + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + writePatch(baselinePatch, "balanced-v1", 95, 0.004, 120); + writePatch(candidatePatch, "balanced-v2", 96, 0.003, 110); + + const baselineOnly = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--baseline-patch", + baselinePatch, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + const candidateOnly = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--candidate-patch", + candidatePatch, + ], + { encoding: "utf8" } + ); + + try { + assert.equal(baselineOnly.status, 2); + assert.match(baselineOnly.stderr ?? "", /baseline-patch and --candidate-patch/); + const manifest = JSON.parse( + readFileSync(join(artifactDir, runId, "manifest.json"), "utf8") + ) as { + result?: { status?: number }; + inputs?: { baselinePatch?: string; candidatePatch?: string }; + }; + assert.equal(manifest.result?.status, 2); + assert.equal(manifest.inputs?.baselinePatch, undefined); + assert.equal(manifest.inputs?.candidatePatch, undefined); + assert.equal(candidateOnly.status, 2); + assert.match(candidateOnly.stderr ?? "", /baseline-patch and --candidate-patch/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval check can retain artifacts for trend summaries", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-retained-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const artifactDir = join(dir, "artifacts"); + const runId = "run-001"; + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "priority", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + + const checkResult = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + + const trendResult = spawnSync( + process.execPath, + ["--import", "tsx", "scripts/router-eval/trends.ts", "--artifact-dir", artifactDir], + { encoding: "utf8" } + ); + + try { + assert.equal(checkResult.status, 0); + assert.ok( + readFileSync(join(artifactDir, runId, "router-eval.md"), "utf8").includes( + "Router Eval Comparison" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "router-eval.json"), "utf8").includes( + "router-eval-comparison" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "inputs", "baseline.ndjson"), "utf8").includes( + "sampleId" + ) + ); + assert.ok( + readFileSync(join(artifactDir, runId, "inputs", "candidate.ndjson"), "utf8").includes( + "sampleId" + ) + ); + const manifest = JSON.parse( + readFileSync(join(artifactDir, runId, "manifest.json"), "utf8") + ) as Record; + assert.equal(manifest.schemaVersion, 1); + assert.equal(manifest.kind, "router-eval-gate-run"); + assert.equal(manifest.runId, runId); + assert.deepEqual(manifest.inputs, { + baseline: "inputs/baseline.ndjson", + candidate: "inputs/candidate.ndjson", + }); + assert.deepEqual(manifest.outputs, { + markdown: "router-eval.md", + json: "router-eval.json", + }); + assert.equal(trendResult.status, 0); + assert.ok((trendResult.stdout ?? "").includes("Router Eval Trends")); + assert.ok((trendResult.stdout ?? "").includes(runId)); + assert.ok((trendResult.stdout ?? "").includes("| jsonl | all |")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-cli.test.ts b/tests/unit/router-eval-cli.test.ts new file mode 100644 index 0000000000..0eb2c1388e --- /dev/null +++ b/tests/unit/router-eval-cli.test.ts @@ -0,0 +1,270 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import Database from "better-sqlite3"; + +const scriptPath = "scripts/router-eval/index.ts"; + +function runCli(args: string[]) { + return spawnSync(process.execPath, ["--import", "tsx", scriptPath, ...args], { + encoding: "utf8", + }); +} + +test("router-eval CLI prints a markdown report for JSONL input", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-")); + const inputPath = join(dir, "input.ndjson"); + writeFileSync( + inputPath, + [ + JSON.stringify({ + sampleId: "s1", + configId: "combo-a", + selectedModel: "gpt-4.1", + requestedModel: "gpt-4.1", + latencyMs: 120, + costUsd: 0.005, + success: true, + }), + JSON.stringify({ + sampleId: "s2", + configId: "combo-b", + selectedModel: "gpt-4o", + requestedModel: "gpt-4o", + latencyMs: 200, + costUsd: 0.003, + success: true, + }), + ].join("\n") + ); + + const result = runCli(["--input", inputPath]); + try { + assert.equal(result.status, 0); + assert.ok((result.stderr ?? "").length === 0); + assert.ok((result.stdout ?? "").includes("Frontier")); + assert.ok((result.stdout ?? "").includes("AIQ")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router-eval CLI exits non-zero when regression threshold is exceeded", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-reg-")); + const baselinePath = join(dir, "baseline.ndjson"); + const candidatePath = join(dir, "candidate.ndjson"); + + writeFileSync( + baselinePath, + [ + JSON.stringify({ + sampleId: "b1", + configId: "combo-a", + selectedModel: "gpt-4.1", + requestedModel: "gpt-4.1", + latencyMs: 100, + costUsd: 0.005, + success: true, + }), + ].join("\n") + ); + + writeFileSync( + candidatePath, + [ + JSON.stringify({ + sampleId: "c1", + configId: "combo-a", + selectedModel: "gpt-4.1", + requestedModel: "gpt-4.1", + latencyMs: 400, + costUsd: 0.05, + success: true, + }), + ].join("\n") + ); + + const result = runCli([ + "--input", + candidatePath, + "--baseline-input", + baselinePath, + "--max-aiq-drop", + "1", + "--max-cost-increase", + "0.2", + "--fail-on-regression", + ]); + try { + assert.equal(result.status, 1); + assert.ok((result.stdout ?? "").includes("Router Eval Comparison")); + assert.ok((result.stdout ?? "").includes("Regressions")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router-eval CLI writes machine-readable JSON artifacts", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-json-")); + const inputPath = join(dir, "input.ndjson"); + const outputPath = join(dir, "router-eval.json"); + writeFileSync( + inputPath, + JSON.stringify({ + sampleId: "s1", + configId: "combo-a", + selectedModel: "gpt-4.1", + requestedModel: "gpt-4.1", + latencyMs: 120, + costUsd: 0.005, + success: true, + }) + ); + + const result = runCli(["--input", inputPath, "--json-output", outputPath]); + try { + assert.equal(result.status, 0); + const artifact = JSON.parse(readFileSync(outputPath, "utf8")) as Record; + assert.equal(artifact.schemaVersion, 1); + assert.equal(artifact.kind, "router-eval-report"); + assert.ok("report" in artifact); + assert.deepEqual((artifact.metadata as Record)?.candidate, { + source: "jsonl", + path: inputPath, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router-eval CLI reads usage_history DB source", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-usage-")); + const dbPath = join(dir, "storage.sqlite"); + const corpusPath = join(dir, "corpus.ndjson"); + const artifactPath = join(dir, "artifact.json"); + const db = new Database(dbPath); + + db.exec(` + CREATE TABLE IF NOT EXISTS usage_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider TEXT, + model TEXT, + connection_id TEXT, + api_key_id TEXT, + api_key_name TEXT, + tokens_input INTEGER DEFAULT 0, + tokens_output INTEGER DEFAULT 0, + tokens_cache_read INTEGER DEFAULT 0, + tokens_cache_creation INTEGER DEFAULT 0, + tokens_reasoning INTEGER DEFAULT 0, + service_tier TEXT DEFAULT 'standard', + status TEXT, + success INTEGER DEFAULT 1, + latency_ms INTEGER DEFAULT 0, + ttft_ms INTEGER DEFAULT 0, + error_code TEXT, + combo_strategy TEXT, + endpoint TEXT, + timestamp TEXT NOT NULL + ) + `); + + db.prepare( + ` + INSERT INTO usage_history + (provider, model, tokens_input, tokens_output, service_tier, success, latency_ms, status, error_code, combo_strategy, timestamp) + VALUES + ('openrouter', 'gpt-4.1', 120, 80, 'standard', 1, 150, '200', NULL, 'priority', '2026-01-01T00:00:00.000Z') + ` + ).run(); + db.close(); + + try { + const result = runCli([ + "--db", + dbPath, + "--db-source", + "usage-history", + "--export-corpus", + corpusPath, + "--json-output", + artifactPath, + ]); + assert.equal(result.status, 0); + assert.ok((result.stderr ?? "").length === 0); + assert.ok((result.stdout ?? "").includes("Router Eval Report")); + assert.ok((result.stdout ?? "").includes("priority")); + const corpus = readFileSync(corpusPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + assert.equal(corpus.length, 1); + assert.equal(corpus[0].configId, "priority"); + assert.equal(corpus[0].selectedModel, "gpt-4.1"); + assert.equal(corpus[0].latencyMs, 150); + assert.deepEqual(corpus[0].metadata, { + provider: "openrouter", + serviceTier: "standard", + errorCode: null, + }); + const artifact = JSON.parse(readFileSync(artifactPath, "utf8")) as Record; + assert.deepEqual((artifact.metadata as Record)?.candidate, { + source: "sqlite", + path: dbPath, + dbSource: "usage-history", + }); + assert.equal( + typeof ((artifact.metadata as Record)?.outputs as Record) + ?.corpus, + "string" + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router-eval CLI defaults --db to call_logs when available", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-cli-default-")); + const dbPath = join(dir, "storage.sqlite"); + const db = new Database(dbPath); + + db.exec(` + CREATE TABLE IF NOT EXISTS call_logs ( + id TEXT PRIMARY KEY, + duration INTEGER, + tokens_in INTEGER, + tokens_out INTEGER, + status INTEGER, + combo_name TEXT, + requested_model TEXT, + model TEXT, + provider TEXT, + error_summary TEXT, + timestamp TEXT NOT NULL, + correlation_id TEXT + ) + `); + + db.prepare( + ` + INSERT INTO call_logs + (id, provider, model, requested_model, tokens_in, tokens_out, status, combo_name, timestamp) + VALUES + ('c1', 'openrouter', 'gpt-4.1', 'gpt-4.1', 120, 80, 200, 'priority', '2026-01-01T00:00:00.000Z') + ` + ).run(); + db.close(); + + try { + const result = runCli(["--db", dbPath]); + assert.equal(result.status, 0); + assert.equal((result.stderr ?? "").length, 0); + assert.ok((result.stdout ?? "").includes("Router Eval Report")); + assert.ok((result.stdout ?? "").includes("priority")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-compare.test.ts b/tests/unit/router-eval-compare.test.ts new file mode 100644 index 0000000000..5eef0c2edb --- /dev/null +++ b/tests/unit/router-eval-compare.test.ts @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const compareScript = "scripts/router-eval/compare.ts"; + +test("router eval compare retains named comparison artifacts", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-compare-")); + const baseline = join(dir, "baseline.ndjson"); + const candidate = join(dir, "candidate.ndjson"); + const artifactDir = join(dir, "artifacts"); + const runId = "policy-a-vs-policy-b"; + + writeFileSync( + baseline, + JSON.stringify({ + sampleId: "b1", + configId: "policy-a", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 140, + costUsd: 0.004, + success: true, + }) + ); + writeFileSync( + candidate, + JSON.stringify({ + sampleId: "c1", + configId: "policy-b", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 130, + costUsd: 0.004, + success: true, + }) + ); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + compareScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--baseline-name", + "policy-a", + "--candidate-name", + "policy-b", + "--artifact-dir", + artifactDir, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Router Eval Comparison")); + const runDir = join(artifactDir, runId); + assert.ok( + readFileSync(join(runDir, "router-eval.md"), "utf8").includes("Router Eval Comparison") + ); + assert.ok( + readFileSync(join(runDir, "router-eval.json"), "utf8").includes("router-eval-comparison") + ); + const comparison = JSON.parse(readFileSync(join(runDir, "comparison.json"), "utf8")) as Record< + string, + unknown + >; + assert.equal(comparison.baselineName, "policy-a"); + assert.equal(comparison.candidateName, "policy-b"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-e2e-chain.test.ts b/tests/unit/router-eval-e2e-chain.test.ts new file mode 100644 index 0000000000..aea92f6df0 --- /dev/null +++ b/tests/unit/router-eval-e2e-chain.test.ts @@ -0,0 +1,135 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const searchScript = "scripts/router-eval/search.ts"; +const checkScript = "scripts/check/check-router-eval-regression.ts"; + +function writeObservation(file: string, configId: string, latencyMs: number, costUsd: number) { + writeFileSync( + file, + `${JSON.stringify({ + sampleId: `${configId}-sample`, + configId, + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs, + costUsd, + success: true, + })}\n` + ); +} + +function runSearch( + baseline: string, + candidateName: string, + candidate: string, + artifactDir: string, + runId: string +): string { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + searchScript, + "--baseline", + baseline, + "--candidate", + `${candidateName}=${candidate}`, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + "--objective", + "quality", + ], + { encoding: "utf8" } + ); + assert.equal(result.status, 0); + const patch = join(artifactDir, runId, "router-config.patch.json"); + assert.ok(readFileSync(patch, "utf8").includes("router-config-patch")); + return patch; +} + +test("router eval retained chain runs search patches through the check wrapper gate", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-e2e-chain-")); + const baseline = join(dir, "baseline.ndjson"); + const oldCandidate = join(dir, "old.ndjson"); + const newCandidate = join(dir, "new.ndjson"); + const searchArtifacts = join(dir, "search-artifacts"); + const gateArtifacts = join(dir, "gate-artifacts"); + const runId = "chain-001"; + + writeObservation(baseline, "baseline", 150, 0.004); + writeObservation(oldCandidate, "old-router", 130, 0.004); + writeObservation(newCandidate, "new-router", 100, 0.003); + + try { + const baselinePatch = runSearch( + baseline, + "old-router", + oldCandidate, + searchArtifacts, + "old-search" + ); + const candidatePatch = runSearch( + baseline, + "new-router", + newCandidate, + searchArtifacts, + "new-search" + ); + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + checkScript, + "--baseline", + baseline, + "--candidate", + newCandidate, + "--baseline-patch", + baselinePatch, + "--candidate-patch", + candidatePatch, + "--artifact-dir", + gateArtifacts, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + + assert.equal(result.status, 0); + const runDir = join(gateArtifacts, runId); + assert.ok( + readFileSync(join(runDir, "router-eval.json"), "utf8").includes("router-eval-comparison") + ); + assert.ok( + readFileSync(join(runDir, "patch-comparison.json"), "utf8").includes( + "router-config-patch-comparison" + ) + ); + assert.ok( + readFileSync(join(runDir, "inputs", "baseline.patch.json"), "utf8").includes("old-router") + ); + assert.ok( + readFileSync(join(runDir, "inputs", "candidate.patch.json"), "utf8").includes("new-router") + ); + const manifest = JSON.parse(readFileSync(join(runDir, "manifest.json"), "utf8")) as { + kind?: string; + result?: { status?: number }; + outputs?: { patchJson?: string }; + }; + assert.equal(manifest.kind, "router-eval-gate-run"); + assert.equal(manifest.result?.status, 0); + assert.equal(manifest.outputs?.patchJson, "patch-comparison.json"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-patch-compare.test.ts b/tests/unit/router-eval-patch-compare.test.ts new file mode 100644 index 0000000000..87db7d65ac --- /dev/null +++ b/tests/unit/router-eval-patch-compare.test.ts @@ -0,0 +1,312 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const compareScript = "scripts/router-eval/patch-compare.ts"; + +function writePatch( + file: string, + configId: string, + aiq: number, + costUsd: number, + latencyMs: number +) { + writeFileSync( + file, + `${JSON.stringify( + { + schemaVersion: 1, + kind: "router-config-patch", + generatedAt: "2026-07-03T00:00:00.000Z", + applyPolicy: "manual-review", + source: { + objective: "balanced", + runId: `${configId}-run`, + artifactPath: `/tmp/${configId}/router-eval.json`, + }, + operations: [ + { + op: "recommend-router-config", + path: "/router/recommendedConfigId", + value: configId, + evidence: { + aiq, + avgCostUsd: costUsd, + avgLatencyMs: latencyMs, + regressions: 0, + }, + rationale: `${configId} wins`, + }, + ], + }, + null, + 2 + )}\n` + ); +} + +test("router config patch compare reports recommendation and metric deltas", () => { + const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-")); + const baseline = join(dir, "baseline.patch.json"); + const candidate = join(dir, "candidate.patch.json"); + const jsonOutput = join(dir, "comparison.json"); + const artifactDir = join(dir, "artifacts"); + + writePatch(baseline, "balanced-v1", 91, 0.006, 180); + writePatch(candidate, "balanced-v2", 94, 0.004, 120); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + compareScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--baseline-name", + "current", + "--candidate-name", + "proposal", + "--json-output", + jsonOutput, + "--artifact-dir", + artifactDir, + "--run-id", + "compare-001", + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Router Config Patch Comparison")); + assert.ok((result.stdout ?? "").includes("Changed recommendation: yes")); + const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as { + kind?: string; + changedRecommendation?: boolean; + delta?: { aiq?: number; avgCostUsd?: number; avgLatencyMs?: number }; + candidate?: { recommendedConfigId?: string }; + }; + assert.equal(comparison.kind, "router-config-patch-comparison"); + assert.equal(comparison.changedRecommendation, true); + assert.equal(comparison.result?.passed, true); + assert.equal(comparison.candidate?.recommendedConfigId, "balanced-v2"); + assert.equal(comparison.delta?.aiq, 3); + assert.equal(comparison.delta?.avgCostUsd, -0.002); + assert.equal(comparison.delta?.avgLatencyMs, -60); + assert.ok( + readFileSync(join(artifactDir, "compare-001", "patch-comparison.md"), "utf8").includes( + "balanced-v2" + ) + ); + assert.ok( + readFileSync(join(artifactDir, "compare-001", "patch-comparison.json"), "utf8").includes( + "router-config-patch-comparison" + ) + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router config patch compare only fails threshold regressions when requested", () => { + const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-gate-")); + const baseline = join(dir, "baseline.patch.json"); + const candidate = join(dir, "candidate.patch.json"); + const jsonOutput = join(dir, "comparison.json"); + + writePatch(baseline, "balanced-v1", 95, 0.004, 120); + writePatch(candidate, "balanced-v2", 90, 0.008, 200); + + const warning = spawnSync( + process.execPath, + [ + "--import", + "tsx", + compareScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--max-aiq-drop", + "1", + "--max-cost-increase", + "0.1", + "--max-latency-increase", + "0.1", + "--json-output", + jsonOutput, + ], + { encoding: "utf8" } + ); + const failing = spawnSync( + process.execPath, + [ + "--import", + "tsx", + compareScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--max-aiq-drop", + "1", + "--max-cost-increase", + "0.1", + "--max-latency-increase", + "0.1", + "--fail-on-regression", + ], + { encoding: "utf8" } + ); + + try { + assert.equal(warning.status, 0); + assert.ok((warning.stdout ?? "").includes("Passed: no")); + const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as { + regressions?: string[]; + result?: { passed?: boolean; status?: number }; + }; + assert.equal(comparison.result?.passed, false); + assert.equal(comparison.result?.status, 0); + assert.ok((comparison.regressions?.length ?? 0) >= 2); + assert.equal(failing.status, 1); + assert.ok((failing.stdout ?? "").includes("Passed: no")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router config patch compare reports unchanged recommendations without failing", () => { + const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-no-change-")); + const baseline = join(dir, "baseline.patch.json"); + const candidate = join(dir, "candidate.patch.json"); + const jsonOutput = join(dir, "comparison.json"); + + writePatch(baseline, "balanced-v1", 94, 0.004, 120); + writePatch(candidate, "balanced-v1", 95, 0.003, 110); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + compareScript, + "--baseline", + baseline, + "--candidate", + candidate, + "--json-output", + jsonOutput, + "--fail-on-regression", + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Changed recommendation: no")); + const comparison = JSON.parse(readFileSync(jsonOutput, "utf8")) as { + changedRecommendation?: boolean; + delta?: { aiq?: number; avgCostUsd?: number; avgLatencyMs?: number }; + result?: { passed?: boolean; status?: number }; + }; + assert.equal(comparison.changedRecommendation, false); + assert.equal(comparison.delta?.aiq, 1); + assert.equal(comparison.delta?.avgCostUsd, -0.001); + assert.equal(comparison.delta?.avgLatencyMs, -10); + assert.equal(comparison.result?.passed, true); + assert.equal(comparison.result?.status, 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router config patch compare rejects invalid patch inputs", () => { + const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-invalid-")); + const baseline = join(dir, "baseline.patch.json"); + const candidate = join(dir, "candidate.patch.json"); + + writePatch(baseline, "balanced-v1", 94, 0.004, 120); + writeFileSync( + candidate, + `${JSON.stringify( + { + schemaVersion: 1, + kind: "router-config-suggestion", + }, + null, + 2 + )}\n` + ); + + const result = spawnSync( + process.execPath, + ["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", candidate], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 2); + assert.match(result.stderr ?? "", /invalid patch kind/); + assert.match(result.stderr ?? "", /router-config-suggestion/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router config patch compare rejects malformed JSON and invalid evidence", () => { + const dir = mkdtempSync(join(tmpdir(), "router-patch-compare-malformed-")); + const baseline = join(dir, "baseline.patch.json"); + const malformed = join(dir, "malformed.patch.json"); + const invalidEvidence = join(dir, "invalid-evidence.patch.json"); + + writePatch(baseline, "balanced-v1", 94, 0.004, 120); + writeFileSync(malformed, "{not-json"); + writeFileSync( + invalidEvidence, + `${JSON.stringify({ + schemaVersion: 1, + kind: "router-config-patch", + applyPolicy: "manual-review", + operations: [ + { + op: "recommend-router-config", + path: "/router/recommendedConfigId", + value: "balanced-v2", + evidence: { + aiq: "94", + avgCostUsd: 0.004, + avgLatencyMs: 120, + regressions: 0, + }, + }, + ], + })}\n` + ); + + const malformedResult = spawnSync( + process.execPath, + ["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", malformed], + { encoding: "utf8" } + ); + const invalidEvidenceResult = spawnSync( + process.execPath, + ["--import", "tsx", compareScript, "--baseline", baseline, "--candidate", invalidEvidence], + { encoding: "utf8" } + ); + + try { + assert.equal(malformedResult.status, 2); + assert.match(malformedResult.stderr ?? "", /invalid JSON/); + assert.equal(invalidEvidenceResult.status, 2); + assert.match(invalidEvidenceResult.stderr ?? "", /invalid numeric evidence field aiq/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-search.test.ts b/tests/unit/router-eval-search.test.ts new file mode 100644 index 0000000000..7e6b40507e --- /dev/null +++ b/tests/unit/router-eval-search.test.ts @@ -0,0 +1,258 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const searchScript = "scripts/router-eval/search.ts"; + +function writeCorpus(path: string, configId: string, latencyMs: number, costUsd: number) { + writeFileSync( + path, + JSON.stringify({ + sampleId: configId, + configId, + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs, + costUsd, + success: true, + }) + ); +} + +function writeCorpusRows( + path: string, + rows: Array<{ configId: string; latencyMs: number; costUsd: number }> +) { + writeFileSync( + path, + rows + .map((row) => + JSON.stringify({ + sampleId: `${row.configId}-sample`, + configId: row.configId, + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: row.latencyMs, + costUsd: row.costUsd, + success: true, + }) + ) + .join("\n") + ); +} + +test("router eval search ranks candidates and writes retained summary artifacts", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-search-")); + const baseline = join(dir, "baseline.ndjson"); + const slow = join(dir, "slow.ndjson"); + const fast = join(dir, "fast.ndjson"); + const artifactDir = join(dir, "artifacts"); + const runId = "search-001"; + + writeCorpus(baseline, "baseline", 150, 0.004); + writeCorpus(slow, "slow", 200, 0.006); + writeCorpus(fast, "fast", 100, 0.003); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + searchScript, + "--baseline", + baseline, + "--candidate", + `slow=${slow}`, + "--candidate", + `fast=${fast}`, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Router Eval Search")); + const searchDir = join(artifactDir, runId); + const markdown = readFileSync(join(searchDir, "search.md"), "utf8"); + assert.ok(markdown.indexOf("| 1 | fast |") < markdown.indexOf("| 2 | slow |")); + assert.ok(markdown.includes("## Patch Operations")); + assert.ok( + markdown.includes("| recommend-router-config | /router/recommendedConfigId | fast |") + ); + const summary = JSON.parse(readFileSync(join(searchDir, "search.json"), "utf8")) as { + kind?: string; + objective?: string; + recommendation?: { candidateName: string; rationale: string }; + suggestion?: { recommendedConfigId: string; applyPolicy: string }; + patch?: { kind: string; operations: Array<{ value: string }> }; + results?: Array<{ candidateName: string }>; + }; + assert.equal(summary.kind, "router-eval-search"); + assert.equal(summary.objective, "balanced"); + assert.equal(summary.results?.[0]?.candidateName, "fast"); + assert.equal(summary.recommendation?.candidateName, "fast"); + assert.match(summary.recommendation?.rationale ?? "", /best balanced rank/); + assert.equal(summary.suggestion?.recommendedConfigId, "fast"); + assert.equal(summary.suggestion?.applyPolicy, "manual-review"); + assert.equal(summary.patch?.kind, "router-config-patch"); + assert.equal(summary.patch?.operations[0]?.value, "fast"); + const recommendation = JSON.parse( + readFileSync(join(searchDir, "recommendation.json"), "utf8") + ) as { + candidateName?: string; + objective?: string; + }; + assert.equal(recommendation.candidateName, "fast"); + assert.equal(recommendation.objective, "balanced"); + const suggestion = JSON.parse(readFileSync(join(searchDir, "suggestion.json"), "utf8")) as { + kind?: string; + recommendedConfigId?: string; + applyPolicy?: string; + }; + assert.equal(suggestion.kind, "router-config-suggestion"); + assert.equal(suggestion.recommendedConfigId, "fast"); + assert.equal(suggestion.applyPolicy, "manual-review"); + const patch = JSON.parse(readFileSync(join(searchDir, "router-config.patch.json"), "utf8")) as { + kind?: string; + applyPolicy?: string; + operations?: Array<{ op?: string; path?: string; value?: string }>; + }; + assert.equal(patch.kind, "router-config-patch"); + assert.equal(patch.applyPolicy, "manual-review"); + assert.equal(patch.operations?.[0]?.op, "recommend-router-config"); + assert.equal(patch.operations?.[0]?.path, "/router/recommendedConfigId"); + assert.equal(patch.operations?.[0]?.value, "fast"); + assert.ok( + readFileSync(join(searchDir, `${runId}-fast`, "router-eval.json"), "utf8").includes( + "router-eval-comparison" + ) + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval search objective modes can choose different candidates", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-search-objective-")); + const baseline = join(dir, "baseline.ndjson"); + const cheap = join(dir, "cheap.ndjson"); + const fast = join(dir, "fast.ndjson"); + const artifactDir = join(dir, "artifacts"); + + writeCorpus(baseline, "baseline", 150, 0.004); + writeCorpus(cheap, "cheap", 900, 0.001); + writeCorpus(fast, "fast", 50, 0.009); + + function runSearch( + objective: string, + runId: string + ): { recommendedConfigId?: string; objective?: string } { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + searchScript, + "--baseline", + baseline, + "--candidate", + `cheap=${cheap}`, + "--candidate", + `fast=${fast}`, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + "--objective", + objective, + "--max-aiq-drop", + "100", + "--max-cost-increase", + "100", + ], + { encoding: "utf8" } + ); + assert.equal(result.status, 0); + return JSON.parse(readFileSync(join(artifactDir, runId, "suggestion.json"), "utf8")) as { + recommendedConfigId?: string; + objective?: string; + }; + } + + try { + const costSuggestion = runSearch("cost", "cost-001"); + const latencySuggestion = runSearch("latency", "latency-001"); + const qualitySuggestion = runSearch("quality", "quality-001"); + assert.equal(costSuggestion.objective, "cost"); + assert.equal(costSuggestion.recommendedConfigId, "cheap"); + assert.equal(latencySuggestion.objective, "latency"); + assert.equal(latencySuggestion.recommendedConfigId, "fast"); + assert.equal(qualitySuggestion.objective, "quality"); + assert.equal(qualitySuggestion.recommendedConfigId, "cheap"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval search cost objective can select a non-AIQ-top config inside a candidate corpus", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-search-multi-config-")); + const baseline = join(dir, "baseline.ndjson"); + const mixed = join(dir, "mixed.ndjson"); + const artifactDir = join(dir, "artifacts"); + const runId = "multi-config-001"; + + writeCorpus(baseline, "baseline", 150, 0.004); + writeCorpusRows(mixed, [ + { configId: "quality-top", latencyMs: 100, costUsd: 0.002 }, + { configId: "cost-top", latencyMs: 3_000, costUsd: 0.0001 }, + ]); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + searchScript, + "--baseline", + baseline, + "--candidate", + `mixed=${mixed}`, + "--artifact-dir", + artifactDir, + "--run-id", + runId, + "--objective", + "cost", + "--max-aiq-drop", + "100", + "--max-cost-increase", + "100", + ], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 0); + const suggestion = JSON.parse( + readFileSync(join(artifactDir, runId, "suggestion.json"), "utf8") + ) as { + recommendedConfigId?: string; + }; + const patch = JSON.parse( + readFileSync(join(artifactDir, runId, "router-config.patch.json"), "utf8") + ) as { + operations?: Array<{ value?: string }>; + }; + assert.equal(suggestion.recommendedConfigId, "cost-top"); + assert.equal(patch.operations?.[0]?.value, "cost-top"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval-trends.test.ts b/tests/unit/router-eval-trends.test.ts new file mode 100644 index 0000000000..989e594f13 --- /dev/null +++ b/tests/unit/router-eval-trends.test.ts @@ -0,0 +1,128 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const trendsScript = "scripts/router-eval/trends.ts"; + +function artifact(generatedAt: string, configId: string, aiq: number) { + return { + schemaVersion: 1, + kind: "router-eval-report", + generatedAt, + report: { + evaluatedAt: generatedAt, + summary: { + totalSamples: 1, + validSamples: 1, + droppedSamples: 0, + uniqueConfigs: 1, + }, + configurations: [], + frontier: [], + top: [ + { + configId, + samples: 1, + successRate: 1, + avgLatencyMs: 100, + p50LatencyMs: 100, + p95LatencyMs: 100, + avgCostUsd: 0.004, + aiq, + }, + ], + }, + }; +} + +test("router eval trends reads retained and flat artifacts with limit", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-trends-")); + const runA = join(dir, "run-a"); + const runB = join(dir, "run-b"); + mkdirSync(runA); + mkdirSync(runB); + writeFileSync( + join(runA, "router-eval.json"), + JSON.stringify(artifact("2026-01-01T00:00:00.000Z", "old", 80)) + ); + writeFileSync( + join(runB, "router-eval.json"), + JSON.stringify(artifact("2026-01-02T00:00:00.000Z", "new", 90)) + ); + writeFileSync( + join(dir, "flat.json"), + JSON.stringify(artifact("2026-01-03T00:00:00.000Z", "flat", 95)) + ); + writeFileSync(join(dir, "bad.json"), "{not json"); + + const result = spawnSync( + process.execPath, + ["--import", "tsx", trendsScript, "--artifact-dir", dir, "--limit", "2"], + { + encoding: "utf8", + } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Router Eval Trends")); + assert.ok(!(result.stdout ?? "").includes("run-a")); + assert.ok((result.stdout ?? "").includes("run-b")); + assert.ok((result.stdout ?? "").includes("flat")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval trends can print dashboard summaries", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-dashboard-")); + const runA = join(dir, "run-a"); + const runB = join(dir, "run-b"); + mkdirSync(runA); + mkdirSync(runB); + writeFileSync( + join(runA, "router-eval.json"), + JSON.stringify(artifact("2026-01-01T00:00:00.000Z", "old", 80)) + ); + writeFileSync( + join(runB, "router-eval.json"), + JSON.stringify(artifact("2026-01-02T00:00:00.000Z", "new", 90)) + ); + + const result = spawnSync( + process.execPath, + ["--import", "tsx", trendsScript, "--artifact-dir", dir, "--dashboard"], + { + encoding: "utf8", + } + ); + + try { + assert.equal(result.status, 0); + assert.ok((result.stdout ?? "").includes("Router Eval Dashboard")); + assert.ok((result.stdout ?? "").includes("Latest: run-b")); + assert.ok((result.stdout ?? "").includes("AIQ: 90.000 (+10.000)")); + assert.ok((result.stdout ?? "").includes("Rolling Averages")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("router eval trends exits clearly for empty artifact dirs", () => { + const dir = mkdtempSync(join(tmpdir(), "router-eval-trends-empty-")); + const result = spawnSync( + process.execPath, + ["--import", "tsx", trendsScript, "--artifact-dir", dir], + { encoding: "utf8" } + ); + + try { + assert.equal(result.status, 2); + assert.ok((result.stderr ?? "").includes("No router-eval artifacts found")); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/unit/router-eval.test.ts b/tests/unit/router-eval.test.ts new file mode 100644 index 0000000000..ba520a6c86 --- /dev/null +++ b/tests/unit/router-eval.test.ts @@ -0,0 +1,92 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + compareRouterEvalRuns, + createRouterEvalArtifact, + runRouterEval, + toRouterObservation, +} from "@/lib/routerEval/index.ts"; + +function obs(overrides: Record) { + return toRouterObservation({ + sampleId: "sample-1", + configId: "config", + selectedModel: "gpt-4.1", + expectedModel: "gpt-4.1", + latencyMs: 120, + costUsd: 0.05, + success: true, + timestamp: "2026-01-01T00:00:00.000Z", + ...overrides, + })!; +} + +test("toRouterObservation normalizes token cost and booleans", () => { + const observation = toRouterObservation({ + sampleId: "s-1", + configId: "combo-a", + selectedModel: "gpt-4.1", + requestedModel: "gpt-4.1", + durationMs: 240, + tokens: { input: 100, output: 50 }, + status: 200, + }); + + assert.ok(observation); + assert.equal(observation!.sampleId, "s-1"); + assert.equal(observation!.configId, "combo-a"); + assert.equal(observation!.latencyMs, 240); + assert.equal(observation!.costUsd, 0.00015); + assert.equal(observation!.success, true); + assert.equal(observation!.expectedModel, "gpt-4.1"); +}); + +test("runRouterEval computes summary and frontiers", () => { + const report = runRouterEval([ + obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }), + obs({ sampleId: "s2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }), + obs({ sampleId: "s3", configId: "b", latencyMs: 80, costUsd: 0.02, success: true }), + obs({ sampleId: "s4", configId: "b", latencyMs: 120, costUsd: 0.02, success: false }), + ]); + + assert.equal(report.summary.totalSamples, 4); + assert.equal(report.summary.validSamples, 4); + assert.equal(report.summary.uniqueConfigs, 2); + assert.equal(report.configurations.length, 2); + assert.equal(report.top[0]?.configId, "a"); + assert.ok(report.frontier.length >= 1); +}); + +test("compareRouterEvalRuns captures AIQ and cost regressions", () => { + const baseline = runRouterEval([ + obs({ sampleId: "b1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }), + obs({ sampleId: "b2", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }), + ]); + const candidate = runRouterEval([ + obs({ sampleId: "c1", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }), + obs({ sampleId: "c2", configId: "a", latencyMs: 300, costUsd: 0.03, success: true }), + ]); + + const comparison = compareRouterEvalRuns(baseline, candidate, { + aiqDrop: 5, + relativeCostIncrease: 1.2, + }); + + assert.equal(comparison.regressions.length > 0, true); + assert.equal(comparison.delta.aiq <= 0, true); + assert.equal(comparison.delta.costUsd > 0, true); +}); + +test("createRouterEvalArtifact wraps reports with a stable schema", () => { + const report = runRouterEval([ + obs({ sampleId: "s1", configId: "a", latencyMs: 100, costUsd: 0.01, success: true }), + ]); + + const artifact = createRouterEvalArtifact(report); + + assert.equal(artifact.schemaVersion, 1); + assert.equal(artifact.kind, "router-eval-report"); + assert.equal(artifact.generatedAt, report.evaluatedAt); + assert.equal(artifact.report?.summary.totalSamples, 1); +});