mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-02 21:32:10 +03:00
1c1aa53caf
Move the 7 static built-in eval suites (golden-set, coding-proficiency, reasoning-logic, multilingual, safety-guardrails, instruction-following, codex-comparison) plus the builtInSuites aggregate into the pure-data leaf src/lib/evals/evalRunner/builtinSuites.ts (zero imports, no side effects). evalRunner.ts keeps all logic (register/get/list/evaluate/run/scorecard/reset) and registers the leaf suites at module load, mirroring the original inline calls. Public API is unchanged (7 exported functions; the suite consts were already module-private). Host 960->301 LOC; leaf 676 LOC (< 800 cap); host was frozen-satisfied (961), so this is debt reduction. Suite data moved verbatim (652 data lines byte-identical). New split-guard test characterizes the suite ids/case counts/key cases and proves the host registers every leaf suite at load.
106 lines
3.3 KiB
TypeScript
106 lines
3.3 KiB
TypeScript
/**
|
|
* Split-guard — evalRunner ↔ evalRunner/builtinSuites
|
|
*
|
|
* Guards the extraction of the 7 built-in golden-set suites (pure data) into
|
|
* the leaf `src/lib/evals/evalRunner/builtinSuites.ts`. Characterizes the suite
|
|
* data (ids, case counts, key cases) and proves the host still registers every
|
|
* leaf suite at module load. DB-free by design: only getSuite(builtInId) is
|
|
* exercised, which short-circuits on the in-memory Map before any DB fallback.
|
|
*/
|
|
|
|
import { describe, it, after } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
|
|
import {
|
|
goldenSet,
|
|
codingSuite,
|
|
reasoningSuite,
|
|
multilingualSuite,
|
|
safetySuite,
|
|
instructionSuite,
|
|
codexComparisonSuite,
|
|
builtInSuites,
|
|
} from "../../src/lib/evals/evalRunner/builtinSuites.ts";
|
|
|
|
import { getSuite, resetSuites } from "../../src/lib/evals/evalRunner.ts";
|
|
|
|
describe("evalRunner/builtinSuites split-guard", () => {
|
|
after(() => {
|
|
// Restore built-ins in case a sibling test mutated the registry.
|
|
resetSuites();
|
|
});
|
|
|
|
it("leaf exports the 7 built-in suites plus the aggregate array", () => {
|
|
const named = [
|
|
goldenSet,
|
|
codingSuite,
|
|
reasoningSuite,
|
|
multilingualSuite,
|
|
safetySuite,
|
|
instructionSuite,
|
|
codexComparisonSuite,
|
|
];
|
|
for (const s of named) {
|
|
assert.ok(s, "suite must be defined");
|
|
assert.equal(typeof s.id, "string");
|
|
assert.equal(typeof s.name, "string");
|
|
assert.ok(Array.isArray(s.cases));
|
|
}
|
|
// Same references, same order — the host relies on this ordering for load-time registration.
|
|
assert.equal(builtInSuites.length, 7);
|
|
assert.deepEqual(builtInSuites, named);
|
|
});
|
|
|
|
it("built-in suite ids and case counts are stable (data pin)", () => {
|
|
assert.deepEqual(
|
|
builtInSuites.map((s) => [s.id, s.cases.length]),
|
|
[
|
|
["golden-set", 10],
|
|
["coding-proficiency", 5],
|
|
["reasoning-logic", 5],
|
|
["multilingual", 5],
|
|
["safety-guardrails", 6],
|
|
["instruction-following", 5],
|
|
["codex-comparison", 8],
|
|
]
|
|
);
|
|
});
|
|
|
|
it("golden-set first case is the contains-hello greeting", () => {
|
|
const gs01 = goldenSet.cases[0];
|
|
assert.equal(gs01.id, "gs-01");
|
|
assert.equal(gs01.name, "Simple greeting");
|
|
assert.equal(gs01.model, "gpt-4o");
|
|
assert.deepEqual(gs01.expected, { strategy: "contains", value: "hello" });
|
|
});
|
|
|
|
it("every codex-comparison case targets the codex model", () => {
|
|
assert.equal(codexComparisonSuite.id, "codex-comparison");
|
|
assert.equal(codexComparisonSuite.cases.length, 8);
|
|
for (const c of codexComparisonSuite.cases) {
|
|
assert.equal(c.model, "codex");
|
|
}
|
|
});
|
|
|
|
it("every safety-guardrails case carries the safety tag", () => {
|
|
assert.equal(safetySuite.cases.length, 6);
|
|
for (const c of safetySuite.cases) {
|
|
assert.ok(
|
|
Array.isArray(c.tags) && c.tags.includes("safety"),
|
|
`case ${c.id} needs safety tag`
|
|
);
|
|
}
|
|
});
|
|
|
|
it("host registers every leaf suite at module load", () => {
|
|
resetSuites();
|
|
for (const s of builtInSuites) {
|
|
const registered = getSuite(s.id);
|
|
assert.ok(registered, `suite ${s.id} should be registered on the host`);
|
|
assert.equal(registered.id, s.id);
|
|
assert.equal(registered.name, s.name);
|
|
assert.equal(registered.cases.length, s.cases.length);
|
|
}
|
|
});
|
|
});
|