diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 4242fb6648..bc9a307d3b 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -52,6 +52,10 @@ jobs: - run: npm run check:route-guard-membership - run: npm run check:test-discovery - run: npm run check:test-runner-api + # Guards tap.testFiles drift: a covering unit test absent from stryker.conf.json + # tap.testFiles makes its module's mutants survive on a cold nightly-mutation run, + # false-failing the blocking mutationScore ratchet. See check-mutation-test-coverage.mjs. + - run: npm run check:mutation-test-coverage - run: npm run check:any-budget:t11 # Build-scope guard: fails if worktrees/cruft leak into the tsconfig include # scope (would OOM `next build`). Instant. See incident 2026-06-25 / #5031. diff --git a/package.json b/package.json index 5278ec611a..ef75590cc7 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "check:known-symbols": "node --import tsx scripts/check/check-known-symbols.ts", "check:route-guard-membership": "node --import tsx scripts/check/check-route-guard-membership.ts", "check:test-discovery": "node scripts/check/check-test-discovery.mjs", + "check:mutation-test-coverage": "node scripts/check/check-mutation-test-coverage.mjs --strict", "check:complexity": "node scripts/check/check-complexity.mjs", "check:dead-code": "node scripts/check/check-dead-code.mjs", "check:cognitive-complexity": "node scripts/check/check-cognitive-complexity.mjs", diff --git a/scripts/check/check-mutation-ratchet.mjs b/scripts/check/check-mutation-ratchet.mjs index 990436b7ed..8cd6747d98 100644 --- a/scripts/check/check-mutation-ratchet.mjs +++ b/scripts/check/check-mutation-ratchet.mjs @@ -31,19 +31,30 @@ const DEFAULT_REPORT = path.join(ROOT, "reports/mutation/mutation.json"); const BASELINE_PREFIX = "mutationScore."; const RATCHET = process.argv.includes("--ratchet"); +// Anti-flake tolerance (percentage points). The tap-runner runs at concurrency 4 and a +// mutant whose tests sit near the `timeoutMS` boundary can flip Killed/Timeout/Survived +// between runs, jittering a module's covered score by a fraction of a point. Only a drop +// LARGER than this counts as a regression — same idea as the coverage ratchet's `eps`. It +// does NOT lower the baseline; it absorbs run-to-run noise so the blocking gate stays +// deterministic. Real test-quality regressions (the headers.ts/telemetry-scale drops that +// motivated the tap.testFiles coverage fix) are far larger than EPS and still fire. +export const MUTATION_RATCHET_EPS = 1.0; + const DETECTED = new Set(["Killed", "Timeout"]); const SURVIVED = new Set(["Survived"]); /** * Avalia o score MEDIDO de um módulo contra o baseline. Direction: UP (o score só - * pode subir — menor = regressão). + * pode subir — menor = regressão), com tolerância anti-flake `eps`: só conta como + * regressão uma queda MAIOR que `eps` pontos. * @param {number} current * @param {number} baseline + * @param {number} [eps] tolerância anti-flake em pontos percentuais * @returns {{ regressed: boolean, improved: boolean }} */ -export function evaluateMutationRatchet(current, baseline) { +export function evaluateMutationRatchet(current, baseline, eps = MUTATION_RATCHET_EPS) { return { - regressed: current < baseline, + regressed: current < baseline - eps, improved: current > baseline, }; } diff --git a/scripts/check/check-mutation-test-coverage.mjs b/scripts/check/check-mutation-test-coverage.mjs new file mode 100644 index 0000000000..e256521c81 --- /dev/null +++ b/scripts/check/check-mutation-test-coverage.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node +// check-mutation-test-coverage — guards against tap.testFiles drift. +// +// WHY: Stryker (nightly-mutation) only runs the test files listed in +// stryker.conf.json `tap.testFiles` against each mutant. When a NEW unit test +// that covers a mutated module is added (or an existing one is split/renamed) +// but NOT added to tap.testFiles, that test's kills stop counting. The mutants +// it would kill then go COVERED-but-unkilled = SURVIVED on a cold run, so the +// module's COVERED mutation score collapses and the blocking mutationScore +// ratchet (nightly-mutation.yml) false-fails — but only on cold-cache nights, +// because the warm incremental run reuses the older (passing) verdicts. The +// pass/fail then tracks GitHub cache state, not code quality. Root cause: +// tap.testFiles is a hand-maintained list with no drift guard. This is it. +// +// INVARIANT: every UNIT test (tests/unit/**) that imports a mutated module +// (stryker.conf.json `mutate`) MUST be in `tap.testFiles`. Integration/e2e +// tests are intentionally excluded (the tap-runner runs node:test units only). +// +// MODE: advisory by default (prints drift, exit 0). `--strict` exits 1 on drift +// so CI can block. Skip-graceful (exit 0) if stryker.conf.json is absent. +// +// USAGE: +// node scripts/check/check-mutation-test-coverage.mjs # advisory +// node scripts/check/check-mutation-test-coverage.mjs --strict # blocking + +import fs from "node:fs"; +import { execFileSync } from "node:child_process"; + +/** 3-segment path suffix without extension (unique enough; matches relative + alias imports). */ +export function moduleFragment(modulePath) { + return modulePath.replace(/\.ts$/, "").split("/").slice(-3).join("/"); +} + +/** + * True if `content` imports the module identified by `fragment`, in any form: + * static `... from "…fragment…"`, dynamic `import("…fragment…")` (incl. split + * across lines), or `require("…fragment…")`. The fragment must appear INSIDE the + * import string literal — a bare mention in a comment does not match. + */ +export function testImportsModule(content, fragment) { + const esc = fragment.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const re = new RegExp( + "(?:from\\s+|\\bimport\\s*\\(\\s*|\\brequire\\s*\\(\\s*)[\"'][^\"']*" + esc + ); + return re.test(content); +} + +/** + * @param {{mutate: string[], tapTestFiles: string[], unitTests: {path:string, content:string}[]}} input + * @returns {Record} module -> covering unit tests NOT in tap.testFiles (drift) + */ +export function findCoverageDrift({ mutate, tapTestFiles, unitTests }) { + const tap = new Set(tapTestFiles); + const drift = {}; + for (const mod of mutate) { + if (mod.startsWith("_") || !mod.endsWith(".ts")) continue; // skip comment/non-ts entries + const frag = moduleFragment(mod); + const missing = unitTests + .filter((t) => testImportsModule(t.content, frag) && !tap.has(t.path)) + .map((t) => t.path) + .sort(); + if (missing.length > 0) drift[mod] = missing; + } + return drift; +} + +function listUnitTests() { + // Static argv — no shell, no interpolation. + const out = execFileSync("git", ["ls-files", "tests/unit"], { encoding: "utf8" }); + return out + .split("\n") + .filter((f) => /\.test\.ts$/.test(f)) + // Exclude tests/unit/build/: these test the build TOOLING (scripts/), not the + // mutated runtime modules. They legitimately embed module paths as fixture + // strings (e.g. this gate's own test), which would otherwise false-match. + .filter((f) => !f.startsWith("tests/unit/build/")) + .map((path) => ({ path, content: fs.readFileSync(path, "utf8") })); +} + +function main() { + const STRICT = process.argv.includes("--strict"); + let conf; + try { + conf = JSON.parse(fs.readFileSync("stryker.conf.json", "utf8")); + } catch { + console.warn("[mutation-test-coverage] stryker.conf.json not found — skipping (advisory)."); + process.exit(0); + } + const mutate = (conf.mutate || []).filter((m) => typeof m === "string"); + const tapTestFiles = conf.tap?.testFiles || []; + const unitTests = listUnitTests(); + + const drift = findCoverageDrift({ mutate, tapTestFiles, unitTests }); + const modules = Object.keys(drift); + const total = modules.reduce((n, m) => n + drift[m].length, 0); + + console.log("Mutation test-coverage gate — tap.testFiles drift detection"); + console.log("==========================================================="); + console.log( + `Scanned ${unitTests.length} unit test file(s) against ${mutate.filter((m) => m.endsWith(".ts")).length} mutated module(s).` + ); + + if (total === 0) { + console.log("✓ No drift — every covering unit test is listed in tap.testFiles."); + process.exit(0); + } + + for (const m of modules) { + console.log(`\n ${m}`); + for (const t of drift[m]) console.log(` + ${t}`); + } + const msg = `${total} covering unit test(s) across ${modules.length} module(s) are missing from stryker.conf.json tap.testFiles.`; + if (STRICT) { + console.error(`\n✗ ${msg} Add them so their mutant kills count (--strict).`); + process.exit(1); + } + console.warn(`\n⚠ ${msg} Re-run with --strict to fail. Add them to tap.testFiles.`); + process.exit(0); +} + +if (import.meta.url === `file://${process.argv[1]}`) main(); diff --git a/stryker.conf.json b/stryker.conf.json index e10ede3d08..36a7bc3eab 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -179,7 +179,45 @@ "tests/unit/token-refresh-service.test.ts", "tests/unit/tools-filter-anthropic-format.test.ts", "tests/unit/usage-service-hardening.test.ts", - "tests/unit/validate-response-quality.test.ts" + "tests/unit/validate-response-quality.test.ts", + "tests/unit/accountfallback-ratelimit-400-4976.test.ts", + "tests/unit/auth-antigravity-account-retry-v2.test.ts", + "tests/unit/authz/route-guard-local-prefix.test.ts", + "tests/unit/authz/route-guard-version-get-exemption.test.ts", + "tests/unit/chat-combo-live-test.test.ts", + "tests/unit/chatcore-executor-proxy.test.ts", + "tests/unit/chatcore-request-format.test.ts", + "tests/unit/chatcore-semantic-cache-store.test.ts", + "tests/unit/chatcore-upstream-timeouts.test.ts", + "tests/unit/circuit-breaker-registry-cap.test.ts", + "tests/unit/circuit-breaker-stream-controller-4602.test.ts", + "tests/unit/combo-account-allowlist-3266.test.ts", + "tests/unit/combo-headroom-strategy.test.ts", + "tests/unit/combo-model-lockout-honors-reset-1308.test.ts", + "tests/unit/combo-param-validation-fallback-4519.test.ts", + "tests/unit/combo-quota-share-cooldown-wait.test.ts", + "tests/unit/combo-selected-connection-success.test.ts", + "tests/unit/combo-stream-readiness-fallback.test.ts", + "tests/unit/combo-strict-random-distribution-3959.test.ts", + "tests/unit/combo/auto-quota-cutoff.test.ts", + "tests/unit/combo/auto-status-penalty-4540.test.ts", + "tests/unit/combo/combo-exhausted-skip.test.ts", + "tests/unit/combo/effective-max-concurrency.test.ts", + "tests/unit/cooldown-epoch-string-3954.test.ts", + "tests/unit/format-provider-error-cause.test.ts", + "tests/unit/grok-cli-oauth.test.ts", + "tests/unit/headroom-proxy-lifecycle.test.ts", + "tests/unit/livews-forward-backoff-4604.test.ts", + "tests/unit/management-auth-hardening.test.ts", + "tests/unit/model-lockout-max-cooldown.test.ts", + "tests/unit/no-memory-header.test.ts", + "tests/unit/non-streaming-sse-terminal-typescan-4459.test.ts", + "tests/unit/openapi-security-tiers.test.ts", + "tests/unit/rate-limit-queue-timeout-lockout.test.ts", + "tests/unit/strip-reasoning-header.test.ts", + "tests/unit/tproxy-route.test.ts", + "tests/unit/types-barrel-model-cooldown.test.ts", + "tests/unit/upstream-retry-hints-toggle.test.ts" ], "nodeArgs": [ "--import", diff --git a/tests/unit/build/check-mutation-ratchet.test.ts b/tests/unit/build/check-mutation-ratchet.test.ts index 7aaa4ad095..e209e804eb 100644 --- a/tests/unit/build/check-mutation-ratchet.test.ts +++ b/tests/unit/build/check-mutation-ratchet.test.ts @@ -8,6 +8,7 @@ import { mutationScoreForFile, measureMutationScores, readBaselineMutationScores, + MUTATION_RATCHET_EPS, } from "../../../scripts/check/check-mutation-ratchet.mjs"; // ── evaluateMutationRatchet: direction UP (score can only improve) ─────────── @@ -19,6 +20,17 @@ test("mutation ratchet flags a drop (direction up)", () => { assert.equal(evaluateMutationRatchet(75.0, 75.0).improved, false); }); +// ── eps anti-flake: a drop WITHIN eps holds; a drop BEYOND eps regresses ────── +test("mutation ratchet tolerates sub-eps jitter but catches real drops", () => { + assert.ok(MUTATION_RATCHET_EPS > 0); + // drop of 0.5pt with default eps 1.0 -> within tolerance, not a regression + assert.equal(evaluateMutationRatchet(74.5, 75.0).regressed, false); + // drop just past eps -> regression + assert.equal(evaluateMutationRatchet(75.0 - MUTATION_RATCHET_EPS - 0.01, 75.0).regressed, true); + // explicit eps argument is honored (0 eps = strict) + assert.equal(evaluateMutationRatchet(74.99, 75.0, 0).regressed, true); +}); + // ── mutationScoreForFile: covered score = detected/(detected+survived), // NoCoverage EXCLUDED (it is a coverage gap, not a test-quality signal). ───── test("mutationScoreForFile computes the covered score and excludes NoCoverage", () => { diff --git a/tests/unit/build/check-mutation-test-coverage.test.ts b/tests/unit/build/check-mutation-test-coverage.test.ts new file mode 100644 index 0000000000..acaf7f5736 --- /dev/null +++ b/tests/unit/build/check-mutation-test-coverage.test.ts @@ -0,0 +1,74 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + moduleFragment, + testImportsModule, + findCoverageDrift, +} from "../../../scripts/check/check-mutation-test-coverage.mjs"; + +test("moduleFragment returns the 3-segment suffix without extension", () => { + assert.equal( + moduleFragment("open-sse/handlers/chatCore/headers.ts"), + "handlers/chatCore/headers" + ); + assert.equal(moduleFragment("src/sse/services/auth.ts"), "sse/services/auth"); + // shallow paths just use what is available + assert.equal(moduleFragment("a/b.ts"), "a/b"); +}); + +test("testImportsModule matches static, dynamic and require imports of the module path", () => { + const frag = "handlers/chatCore/headers"; + // static import-from + assert.equal( + testImportsModule(`import { x } from "@omniroute/open-sse/handlers/chatCore/headers";`, frag), + true + ); + // dynamic await import, even split across lines + assert.equal( + testImportsModule(`const { y } = await import(\n "../../open-sse/handlers/chatCore/headers.ts"\n);`, frag), + true + ); + // require() + assert.equal( + testImportsModule(`const z = require("../../open-sse/handlers/chatCore/headers.ts");`, frag), + true + ); + // unrelated module is not matched + assert.equal( + testImportsModule(`import { a } from "@omniroute/open-sse/handlers/chatCore/idempotency";`, frag), + false + ); + // the fragment appearing only in a comment (not an import string) is NOT a match + assert.equal( + testImportsModule(`// see handlers/chatCore/headers for details\nconst a = 1;`, frag), + false + ); +}); + +test("findCoverageDrift flags covering unit tests absent from tap.testFiles", () => { + const mutate = [ + "open-sse/handlers/chatCore/headers.ts", + "open-sse/handlers/chatCore/idempotency.ts", + "_a_comment_entry", + ]; + const tapTestFiles = ["tests/unit/chatcore-headers.test.ts"]; + const unitTests = [ + // covers headers, already in tap -> not drift + { path: "tests/unit/chatcore-headers.test.ts", content: `await import("../../open-sse/handlers/chatCore/headers.ts");` }, + // covers headers, NOT in tap -> drift + { path: "tests/unit/no-memory-header.test.ts", content: `const { isNoMemoryRequested } = await import("../../open-sse/handlers/chatCore/headers.ts");` }, + // covers idempotency, NOT in tap -> drift + { path: "tests/unit/idempo.test.ts", content: `import { x } from "@omniroute/open-sse/handlers/chatCore/idempotency";` }, + // covers nothing mutated -> ignored + { path: "tests/unit/unrelated.test.ts", content: `import { z } from "@/lib/foo";` }, + ]; + const drift = findCoverageDrift({ mutate, tapTestFiles, unitTests }); + assert.deepEqual(drift["open-sse/handlers/chatCore/headers.ts"], [ + "tests/unit/no-memory-header.test.ts", + ]); + assert.deepEqual(drift["open-sse/handlers/chatCore/idempotency.ts"], [ + "tests/unit/idempo.test.ts", + ]); + // comment-only mutate entries are skipped + assert.equal("_a_comment_entry" in drift, false); +});