fix(ci): stabilize nightly-mutation — guard tap.testFiles drift + anti-flake eps (#5682)

Root cause (NOT a timeout): the nightly-mutation run fails on cold-cache nights
because the blocking mutation-ratchet job measures modules below baseline, while
warm-cache nights pass — the verdict tracked GitHub Actions cache state, not code
quality. Proven via a local Stryker probe on headers.ts: covering unit tests
(no-memory-header, strip-reasoning) had drifted OUT of stryker.conf.json
tap.testFiles, so their mutants went covered-but-unkilled = Survived on a cold
full run (COVERED score 61.73 vs 94.29 baseline); adding them restores the kills.

- Add scripts/check/check-mutation-test-coverage.mjs: guards that every UNIT test
  importing a Stryker-mutated module is listed in tap.testFiles. Advisory by
  default, --strict in CI (wired in quality.yml fast-gates). Prevents recurrence.
- Add the 38 drifted covering unit tests to stryker.conf.json tap.testFiles
  (138 -> 176). Monotonically safe: more covering tests only raise/hold the score.
- Add MUTATION_RATCHET_EPS (1.0pt) anti-flake tolerance to check-mutation-ratchet
  so sub-point tap-runner jitter no longer false-fails the gate. Lowers no baseline.
- Tests: check-mutation-test-coverage (3) + eps cases in check-mutation-ratchet.

Residual: a clean post-merge nightly confirms scores return to/above baseline;
any marginal residual gets a baseline re-seed (operator).

.github/workflows/quality.yml

@@ -52,6 +52,10 @@ jobs:
       - run: npm run check:route-guard-membership
       - run: npm run check:test-discovery
       - run: npm run check:test-runner-api
+      # Guards tap.testFiles drift: a covering unit test absent from stryker.conf.json
+      # tap.testFiles makes its module's mutants survive on a cold nightly-mutation run,
+      # false-failing the blocking mutationScore ratchet. See check-mutation-test-coverage.mjs.
+      - run: npm run check:mutation-test-coverage
       - run: npm run check:any-budget:t11
       # Build-scope guard: fails if worktrees/cruft leak into the tsconfig include
       # scope (would OOM `next build`). Instant. See incident 2026-06-25 / #5031.

package.json

@@ -150,6 +150,7 @@
     "check:known-symbols": "node --import tsx scripts/check/check-known-symbols.ts",
     "check:route-guard-membership": "node --import tsx scripts/check/check-route-guard-membership.ts",
     "check:test-discovery": "node scripts/check/check-test-discovery.mjs",
+    "check:mutation-test-coverage": "node scripts/check/check-mutation-test-coverage.mjs --strict",
     "check:complexity": "node scripts/check/check-complexity.mjs",
     "check:dead-code": "node scripts/check/check-dead-code.mjs",
     "check:cognitive-complexity": "node scripts/check/check-cognitive-complexity.mjs",

scripts/check/check-mutation-ratchet.mjs

@@ -31,19 +31,30 @@ const DEFAULT_REPORT = path.join(ROOT, "reports/mutation/mutation.json");
 const BASELINE_PREFIX = "mutationScore.";
 const RATCHET = process.argv.includes("--ratchet");
 
+// Anti-flake tolerance (percentage points). The tap-runner runs at concurrency 4 and a
+// mutant whose tests sit near the `timeoutMS` boundary can flip Killed/Timeout/Survived
+// between runs, jittering a module's covered score by a fraction of a point. Only a drop
+// LARGER than this counts as a regression — same idea as the coverage ratchet's `eps`. It
+// does NOT lower the baseline; it absorbs run-to-run noise so the blocking gate stays
+// deterministic. Real test-quality regressions (the headers.ts/telemetry-scale drops that
+// motivated the tap.testFiles coverage fix) are far larger than EPS and still fire.
+export const MUTATION_RATCHET_EPS = 1.0;
+
 const DETECTED = new Set(["Killed", "Timeout"]);
 const SURVIVED = new Set(["Survived"]);
 
 /**
  * Avalia o score MEDIDO de um módulo contra o baseline. Direction: UP (o score só
- * pode subir — menor = regressão).
+ * pode subir — menor = regressão), com tolerância anti-flake `eps`: só conta como
+ * regressão uma queda MAIOR que `eps` pontos.
  * @param {number} current
  * @param {number} baseline
+ * @param {number} [eps] tolerância anti-flake em pontos percentuais
  * @returns {{ regressed: boolean, improved: boolean }}
  */
-export function evaluateMutationRatchet(current, baseline) {
+export function evaluateMutationRatchet(current, baseline, eps = MUTATION_RATCHET_EPS) {
   return {
-    regressed: current < baseline,
+    regressed: current < baseline - eps,
     improved: current > baseline,
   };
 }

scripts/check/check-mutation-test-coverage.mjs

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+// check-mutation-test-coverage — guards against tap.testFiles drift.
+//
+// WHY: Stryker (nightly-mutation) only runs the test files listed in
+// stryker.conf.json `tap.testFiles` against each mutant. When a NEW unit test
+// that covers a mutated module is added (or an existing one is split/renamed)
+// but NOT added to tap.testFiles, that test's kills stop counting. The mutants
+// it would kill then go COVERED-but-unkilled = SURVIVED on a cold run, so the
+// module's COVERED mutation score collapses and the blocking mutationScore
+// ratchet (nightly-mutation.yml) false-fails — but only on cold-cache nights,
+// because the warm incremental run reuses the older (passing) verdicts. The
+// pass/fail then tracks GitHub cache state, not code quality. Root cause:
+// tap.testFiles is a hand-maintained list with no drift guard. This is it.
+//
+// INVARIANT: every UNIT test (tests/unit/**) that imports a mutated module
+// (stryker.conf.json `mutate`) MUST be in `tap.testFiles`. Integration/e2e
+// tests are intentionally excluded (the tap-runner runs node:test units only).
+//
+// MODE: advisory by default (prints drift, exit 0). `--strict` exits 1 on drift
+// so CI can block. Skip-graceful (exit 0) if stryker.conf.json is absent.
+//
+// USAGE:
+//   node scripts/check/check-mutation-test-coverage.mjs            # advisory
+//   node scripts/check/check-mutation-test-coverage.mjs --strict   # blocking
+
+import fs from "node:fs";
+import { execFileSync } from "node:child_process";
+
+/** 3-segment path suffix without extension (unique enough; matches relative + alias imports). */
+export function moduleFragment(modulePath) {
+  return modulePath.replace(/\.ts$/, "").split("/").slice(-3).join("/");
+}
+
+/**
+ * True if `content` imports the module identified by `fragment`, in any form:
+ * static `... from "…fragment…"`, dynamic `import("…fragment…")` (incl. split
+ * across lines), or `require("…fragment…")`. The fragment must appear INSIDE the
+ * import string literal — a bare mention in a comment does not match.
+ */
+export function testImportsModule(content, fragment) {
+  const esc = fragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const re = new RegExp(
+    "(?:from\\s+|\\bimport\\s*\\(\\s*|\\brequire\\s*\\(\\s*)[\"'][^\"']*" + esc
+  );
+  return re.test(content);
+}
+
+/**
+ * @param {{mutate: string[], tapTestFiles: string[], unitTests: {path:string, content:string}[]}} input
+ * @returns {Record<string,string[]>} module -> covering unit tests NOT in tap.testFiles (drift)
+ */
+export function findCoverageDrift({ mutate, tapTestFiles, unitTests }) {
+  const tap = new Set(tapTestFiles);
+  const drift = {};
+  for (const mod of mutate) {
+    if (mod.startsWith("_") || !mod.endsWith(".ts")) continue; // skip comment/non-ts entries
+    const frag = moduleFragment(mod);
+    const missing = unitTests
+      .filter((t) => testImportsModule(t.content, frag) && !tap.has(t.path))
+      .map((t) => t.path)
+      .sort();
+    if (missing.length > 0) drift[mod] = missing;
+  }
+  return drift;
+}
+
+function listUnitTests() {
+  // Static argv — no shell, no interpolation.
+  const out = execFileSync("git", ["ls-files", "tests/unit"], { encoding: "utf8" });
+  return out
+    .split("\n")
+    .filter((f) => /\.test\.ts$/.test(f))
+    // Exclude tests/unit/build/: these test the build TOOLING (scripts/), not the
+    // mutated runtime modules. They legitimately embed module paths as fixture
+    // strings (e.g. this gate's own test), which would otherwise false-match.
+    .filter((f) => !f.startsWith("tests/unit/build/"))
+    .map((path) => ({ path, content: fs.readFileSync(path, "utf8") }));
+}
+
+function main() {
+  const STRICT = process.argv.includes("--strict");
+  let conf;
+  try {
+    conf = JSON.parse(fs.readFileSync("stryker.conf.json", "utf8"));
+  } catch {
+    console.warn("[mutation-test-coverage] stryker.conf.json not found — skipping (advisory).");
+    process.exit(0);
+  }
+  const mutate = (conf.mutate || []).filter((m) => typeof m === "string");
+  const tapTestFiles = conf.tap?.testFiles || [];
+  const unitTests = listUnitTests();
+
+  const drift = findCoverageDrift({ mutate, tapTestFiles, unitTests });
+  const modules = Object.keys(drift);
+  const total = modules.reduce((n, m) => n + drift[m].length, 0);
+
+  console.log("Mutation test-coverage gate — tap.testFiles drift detection");
+  console.log("===========================================================");
+  console.log(
+    `Scanned ${unitTests.length} unit test file(s) against ${mutate.filter((m) => m.endsWith(".ts")).length} mutated module(s).`
+  );
+
+  if (total === 0) {
+    console.log("✓ No drift — every covering unit test is listed in tap.testFiles.");
+    process.exit(0);
+  }
+
+  for (const m of modules) {
+    console.log(`\n  ${m}`);
+    for (const t of drift[m]) console.log(`    + ${t}`);
+  }
+  const msg = `${total} covering unit test(s) across ${modules.length} module(s) are missing from stryker.conf.json tap.testFiles.`;
+  if (STRICT) {
+    console.error(`\n✗ ${msg} Add them so their mutant kills count (--strict).`);
+    process.exit(1);
+  }
+  console.warn(`\n⚠ ${msg} Re-run with --strict to fail. Add them to tap.testFiles.`);
+  process.exit(0);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) main();

stryker.conf.json

@@ -179,7 +179,45 @@
       "tests/unit/token-refresh-service.test.ts",
       "tests/unit/tools-filter-anthropic-format.test.ts",
       "tests/unit/usage-service-hardening.test.ts",
-      "tests/unit/validate-response-quality.test.ts"
+      "tests/unit/validate-response-quality.test.ts",
+      "tests/unit/accountfallback-ratelimit-400-4976.test.ts",
+      "tests/unit/auth-antigravity-account-retry-v2.test.ts",
+      "tests/unit/authz/route-guard-local-prefix.test.ts",
+      "tests/unit/authz/route-guard-version-get-exemption.test.ts",
+      "tests/unit/chat-combo-live-test.test.ts",
+      "tests/unit/chatcore-executor-proxy.test.ts",
+      "tests/unit/chatcore-request-format.test.ts",
+      "tests/unit/chatcore-semantic-cache-store.test.ts",
+      "tests/unit/chatcore-upstream-timeouts.test.ts",
+      "tests/unit/circuit-breaker-registry-cap.test.ts",
+      "tests/unit/circuit-breaker-stream-controller-4602.test.ts",
+      "tests/unit/combo-account-allowlist-3266.test.ts",
+      "tests/unit/combo-headroom-strategy.test.ts",
+      "tests/unit/combo-model-lockout-honors-reset-1308.test.ts",
+      "tests/unit/combo-param-validation-fallback-4519.test.ts",
+      "tests/unit/combo-quota-share-cooldown-wait.test.ts",
+      "tests/unit/combo-selected-connection-success.test.ts",
+      "tests/unit/combo-stream-readiness-fallback.test.ts",
+      "tests/unit/combo-strict-random-distribution-3959.test.ts",
+      "tests/unit/combo/auto-quota-cutoff.test.ts",
+      "tests/unit/combo/auto-status-penalty-4540.test.ts",
+      "tests/unit/combo/combo-exhausted-skip.test.ts",
+      "tests/unit/combo/effective-max-concurrency.test.ts",
+      "tests/unit/cooldown-epoch-string-3954.test.ts",
+      "tests/unit/format-provider-error-cause.test.ts",
+      "tests/unit/grok-cli-oauth.test.ts",
+      "tests/unit/headroom-proxy-lifecycle.test.ts",
+      "tests/unit/livews-forward-backoff-4604.test.ts",
+      "tests/unit/management-auth-hardening.test.ts",
+      "tests/unit/model-lockout-max-cooldown.test.ts",
+      "tests/unit/no-memory-header.test.ts",
+      "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts",
+      "tests/unit/openapi-security-tiers.test.ts",
+      "tests/unit/rate-limit-queue-timeout-lockout.test.ts",
+      "tests/unit/strip-reasoning-header.test.ts",
+      "tests/unit/tproxy-route.test.ts",
+      "tests/unit/types-barrel-model-cooldown.test.ts",
+      "tests/unit/upstream-retry-hints-toggle.test.ts"
     ],
     "nodeArgs": [
       "--import",

tests/unit/build/check-mutation-ratchet.test.ts

@@ -8,6 +8,7 @@ import {
   mutationScoreForFile,
   measureMutationScores,
   readBaselineMutationScores,
+  MUTATION_RATCHET_EPS,
 } from "../../../scripts/check/check-mutation-ratchet.mjs";
 
 // ── evaluateMutationRatchet: direction UP (score can only improve) ───────────
@@ -19,6 +20,17 @@ test("mutation ratchet flags a drop (direction up)", () => {
   assert.equal(evaluateMutationRatchet(75.0, 75.0).improved, false);
 });
 
+// ── eps anti-flake: a drop WITHIN eps holds; a drop BEYOND eps regresses ──────
+test("mutation ratchet tolerates sub-eps jitter but catches real drops", () => {
+  assert.ok(MUTATION_RATCHET_EPS > 0);
+  // drop of 0.5pt with default eps 1.0 -> within tolerance, not a regression
+  assert.equal(evaluateMutationRatchet(74.5, 75.0).regressed, false);
+  // drop just past eps -> regression
+  assert.equal(evaluateMutationRatchet(75.0 - MUTATION_RATCHET_EPS - 0.01, 75.0).regressed, true);
+  // explicit eps argument is honored (0 eps = strict)
+  assert.equal(evaluateMutationRatchet(74.99, 75.0, 0).regressed, true);
+});
+
 // ── mutationScoreForFile: covered score = detected/(detected+survived),
 // NoCoverage EXCLUDED (it is a coverage gap, not a test-quality signal). ─────
 test("mutationScoreForFile computes the covered score and excludes NoCoverage", () => {

tests/unit/build/check-mutation-test-coverage.test.ts

@@ -0,0 +1,74 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+  moduleFragment,
+  testImportsModule,
+  findCoverageDrift,
+} from "../../../scripts/check/check-mutation-test-coverage.mjs";
+
+test("moduleFragment returns the 3-segment suffix without extension", () => {
+  assert.equal(
+    moduleFragment("open-sse/handlers/chatCore/headers.ts"),
+    "handlers/chatCore/headers"
+  );
+  assert.equal(moduleFragment("src/sse/services/auth.ts"), "sse/services/auth");
+  // shallow paths just use what is available
+  assert.equal(moduleFragment("a/b.ts"), "a/b");
+});
+
+test("testImportsModule matches static, dynamic and require imports of the module path", () => {
+  const frag = "handlers/chatCore/headers";
+  // static import-from
+  assert.equal(
+    testImportsModule(`import { x } from "@omniroute/open-sse/handlers/chatCore/headers";`, frag),
+    true
+  );
+  // dynamic await import, even split across lines
+  assert.equal(
+    testImportsModule(`const { y } = await import(\n  "../../open-sse/handlers/chatCore/headers.ts"\n);`, frag),
+    true
+  );
+  // require()
+  assert.equal(
+    testImportsModule(`const z = require("../../open-sse/handlers/chatCore/headers.ts");`, frag),
+    true
+  );
+  // unrelated module is not matched
+  assert.equal(
+    testImportsModule(`import { a } from "@omniroute/open-sse/handlers/chatCore/idempotency";`, frag),
+    false
+  );
+  // the fragment appearing only in a comment (not an import string) is NOT a match
+  assert.equal(
+    testImportsModule(`// see handlers/chatCore/headers for details\nconst a = 1;`, frag),
+    false
+  );
+});
+
+test("findCoverageDrift flags covering unit tests absent from tap.testFiles", () => {
+  const mutate = [
+    "open-sse/handlers/chatCore/headers.ts",
+    "open-sse/handlers/chatCore/idempotency.ts",
+    "_a_comment_entry",
+  ];
+  const tapTestFiles = ["tests/unit/chatcore-headers.test.ts"];
+  const unitTests = [
+    // covers headers, already in tap -> not drift
+    { path: "tests/unit/chatcore-headers.test.ts", content: `await import("../../open-sse/handlers/chatCore/headers.ts");` },
+    // covers headers, NOT in tap -> drift
+    { path: "tests/unit/no-memory-header.test.ts", content: `const { isNoMemoryRequested } = await import("../../open-sse/handlers/chatCore/headers.ts");` },
+    // covers idempotency, NOT in tap -> drift
+    { path: "tests/unit/idempo.test.ts", content: `import { x } from "@omniroute/open-sse/handlers/chatCore/idempotency";` },
+    // covers nothing mutated -> ignored
+    { path: "tests/unit/unrelated.test.ts", content: `import { z } from "@/lib/foo";` },
+  ];
+  const drift = findCoverageDrift({ mutate, tapTestFiles, unitTests });
+  assert.deepEqual(drift["open-sse/handlers/chatCore/headers.ts"], [
+    "tests/unit/no-memory-header.test.ts",
+  ]);
+  assert.deepEqual(drift["open-sse/handlers/chatCore/idempotency.ts"], [
+    "tests/unit/idempo.test.ts",
+  ]);
+  // comment-only mutate entries are skipped
+  assert.equal("_a_comment_entry" in drift, false);
+});