diff --git a/changelog.d/maintenance/router-eval-test-assert-strengthen.md b/changelog.d/maintenance/router-eval-test-assert-strengthen.md new file mode 100644 index 0000000000..c6f6e86046 --- /dev/null +++ b/changelog.d/maintenance/router-eval-test-assert-strengthen.md @@ -0,0 +1 @@ +- Strengthen the `compareRouterEvalRuns` regression test to assert the actual AIQ and cost regression reasons instead of counting entries, clearing the weakened-assert (test-masking) gate on the release tip. diff --git a/tests/unit/router-eval.test.ts b/tests/unit/router-eval.test.ts index ba520a6c86..4d6527704f 100644 --- a/tests/unit/router-eval.test.ts +++ b/tests/unit/router-eval.test.ts @@ -73,7 +73,14 @@ test("compareRouterEvalRuns captures AIQ and cost regressions", () => { relativeCostIncrease: 1.2, }); - assert.equal(comparison.regressions.length > 0, true); + assert.ok( + comparison.regressions.some((reason) => reason.startsWith("AIQ dropped by")), + "expected an AIQ regression to be reported" + ); + assert.ok( + comparison.regressions.some((reason) => reason.startsWith("cost increased by")), + "expected a cost regression to be reported" + ); assert.equal(comparison.delta.aiq <= 0, true); assert.equal(comparison.delta.costUsd > 0, true); });