From facbb1964f59ac3395e06a9f14d5309e99367e0b Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Fri, 19 Jun 2026 21:40:57 -0300 Subject: [PATCH] =?UTF-8?q?feat(quality):=20unblock=20R1=20=E2=80=94=20tes?= =?UTF-8?q?t-redundancy=20measurement=20via=20disableBail=20(#4322)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(quality): config stryker disableBail para medir redundância (R1) Estende stryker.conf.json com disableBail:true (killedBy lista TODOS os killers, não só o primeiro) + incremental:false (medição limpa). One-off, não toca o nightly — só roda via mutation-redundancy.yml. * feat(quality): workflow on-demand de mutação disableBail (R1) mutation-redundancy.yml (workflow_dispatch): roda os 6 leaf-batches combo+chatCore do nightly com o override disableBail e sobe os reports. Batches granulares (d/e/f/ g/h/i) em vez de 2 mega-batches: disableBail é mais caro e Stryker só grava o report se COMPLETAR, então cada batch cabe nos 300min. zizmor 139->145: +3 drift de base + 3 do workflow (@vN, convenção do repo). Ver nota. * feat(quality): mutation-radiography --candidates (lista de prune R1) Reusa aggregateRadiography (DRY) em vez de um script novo: redundancyCandidates() retorna 🔴 empty ∪ 🟠 redundant (zero kills únicos) = candidatos a prune. Sob disableBail o killedBy é completo, então 🟠 redundant fica ACURADO. CLI --candidates emite a lista com o aviso do gate humano (excluir segurança/contrato/repro). --- .github/workflows/mutation-redundancy.yml | 63 +++++++++++++++++ config/quality/quality-baseline.json | 5 +- scripts/quality/mutation-radiography.mjs | 70 ++++++++++++++++++- stryker.disablebail.json | 7 ++ tests/unit/build/mutation-radiography.test.ts | 28 ++++++++ 5 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/mutation-redundancy.yml create mode 100644 stryker.disablebail.json diff --git a/.github/workflows/mutation-redundancy.yml b/.github/workflows/mutation-redundancy.yml new file mode 100644 index 0000000000..e584bea487 --- /dev/null +++ b/.github/workflows/mutation-redundancy.yml @@ -0,0 +1,63 @@ +name: Mutation Redundancy (disableBail, on-demand) +# One-off measurement to UNBLOCK R1 (test-redundancy prune). The nightly mutation run +# (nightly-mutation.yml) bails on the first kill, so `killedBy` lists only the FIRST +# killer — 🟠 redundant is understated and 🟢 unique overstated (see the caveat in +# scripts/quality/mutation-radiography.mjs). This workflow re-runs the SAME combo + +# chatCore leaf batches with stryker.disablebail.json (disableBail:true, incremental:false) +# so `killedBy` lists EVERY killer. Feed the uploaded reports to +# `node scripts/quality/mutation-radiography.mjs --candidates mutation-nobail-*/mutation.json` +# to get the accurate R1 prune-candidate list (🔴 empty ∪ 🟠 redundant) for human review. +# +# Batches mirror the nightly's leaf decomposition (d/e/f/g/h/i) rather than 2 mega-batches: +# disableBail is MORE expensive than bail (it never stops early), and Stryker only writes +# mutation.json on a SUCCESSFUL finish — a batch cancelled at the cap produces NO data — so +# smaller batches each fit the 300min headroom and run in parallel. auth/accountFallback and +# the security quartet are out of scope: R1 targets the combo/chatCore leaves. +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + stryker-nobail: + name: Stryker disableBail (batch ${{ matrix.batch.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + batch: + - name: d + mutate: "open-sse/services/combo/comboStructure.ts,open-sse/services/combo/autoStrategy.ts,open-sse/services/combo/validateQuality.ts" + - name: e + mutate: "open-sse/services/combo/shadowRouting.ts,open-sse/services/combo/targetSorters.ts,open-sse/services/combo/comboPredicates.ts,open-sse/services/combo/rrState.ts,open-sse/services/combo/comboData.ts" + - name: f + mutate: "open-sse/services/combo/quotaScoring.ts,open-sse/services/combo/quotaStrategies.ts" + - name: g + mutate: "open-sse/handlers/chatCore/comboContextCache.ts,open-sse/handlers/chatCore/idempotency.ts,open-sse/handlers/chatCore/passthroughHelpers.ts,open-sse/handlers/chatCore/responseHeaders.ts,open-sse/handlers/chatCore/sanitization.ts,open-sse/handlers/chatCore/upstreamTimeouts.ts" + - name: h + mutate: "open-sse/handlers/chatCore/headers.ts,open-sse/handlers/chatCore/logTruncation.ts,open-sse/handlers/chatCore/memoryExtraction.ts,open-sse/handlers/chatCore/nonStreamingSse.ts,open-sse/handlers/chatCore/passthroughToolNames.ts,open-sse/handlers/chatCore/executorHelpers.ts" + - name: i + mutate: "open-sse/handlers/chatCore/telemetryHelpers.ts,open-sse/handlers/chatCore/memorySkillsInjection.ts,open-sse/handlers/chatCore/semanticCache.ts" + timeout-minutes: 300 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + - run: npm ci + - name: Run Stryker (disableBail) + env: + BATCH_MUTATE: ${{ matrix.batch.mutate }} + run: npx stryker run --config-file stryker.disablebail.json --mutate "$BATCH_MUTATE" + - name: Upload mutation report + if: always() + uses: actions/upload-artifact@v7 + with: + name: mutation-nobail-${{ matrix.batch.name }} + path: reports/mutation/ + if-no-files-found: warn + retention-days: 14 diff --git a/config/quality/quality-baseline.json b/config/quality/quality-baseline.json index d4c4f48121..a885a1052d 100644 --- a/config/quality/quality-baseline.json +++ b/config/quality/quality-baseline.json @@ -116,7 +116,7 @@ "dedicatedGate": true }, "zizmorFindings": { - "value": 139, + "value": 145, "direction": "down", "dedicatedGate": true }, @@ -322,5 +322,6 @@ "_trivy_flip_blocking_2026_06_16_v3827": "Fim do ciclo v3.8.27: Trivy (scan de CVE da imagem Docker em docker-publish.yml) promovido para BLOQUEANTE em CRITICAL. Abordagem de DOIS PASSOS: o passo SARIF existente (severity HIGH,CRITICAL / exit-code 0 / upload SARIF) fica INTACTO para visibilidade na aba Security; um novo passo 'Trivy CRITICAL gate (blocking)' (severity CRITICAL / ignore-unfixed:true / exit-code 1) falha o release num CVE CRITICO FIXAVEL. ignore-unfixed evita travar por CVE de base-image sem patch upstream (reduz falso-bloqueio). Mesma variancia-de-CVE do osv: um novo CRITICAL fixavel divulgado pode redar; remedio = rebuild sobre base patcheada, bumpar dep, ou .trivyignore com justificativa+issue. Ver docs/security/SUPPLY_CHAIN.md. vulnCount permanece 10 (intocado neste flip — so a postura advisory->bloqueante mudou).", "_rebaseline_2026_06_18_v3828_cycle_close": "Fim do ciclo v3.8.28 (RELEASED; ciclo v3.8.29 aberto): 3 metricas re-baselineadas para o valor REAL medido no push->main pos-release (run 27725117464, step 'Ratchet check') — eslintWarnings 3769->3779, openapiCoverage.pct 38.3->37.6, i18nUiCoverage.pct 80.1->79.1. Reproduzido localmente em release/v3.8.29 (9f14c1294): identico ao CI. Drift de fim-de-ciclo de features legitimas, NAO hand-cleanable: (a) eslint +10 = 'any' PERMITIDO (warn) em testes do ciclo + 4 react-hooks/exhaustive-deps em RequestLoggerV2.tsx (componente com bugs sutis de refresh #4103/#3972, arriscado mexer em deps de hook sem teste de UI); (b) openapi -0.7 = drop por rotas NOVAS INTERNAS (/api/tools/agent-bridge/* LOCAL_ONLY, spawnam MITM/DNS) — documenta-las no spec PUBLICO seria gaming; (c) i18n -1.0 = 37/41 locales em 79.1% (1741 chaves faltando cada, ~3000 traducoes via 'npm run i18n:run' que exige creds OMNIROUTE_TRANSLATION_API_KEY indisponiveis localmente). Mesmo precedente do _eslint_rebaseline_2026_06_16_v3826_forward_merge. Apertar no fim do ciclo: eslint/openapi via --require-tighten; i18n via i18n:run com creds. Autorizado pelo operador (decisao explicita).", "_rebaseline_2026_06_19_v3829_cycle_close": "Release do ciclo v3.8.29: eslintWarnings re-baselineado 3779->3816 para o valor REAL medido em release/v3.8.29 (tip da3...; `npm run lint` local = 3816, identico ao Quality Ratchet do CI no PR #4126). O +37 e drift de fim-de-ciclo de 115 commits de features legitimas — `any` PERMITIDO (warn) em open-sse/ e tests/ do ciclo; os arquivos de reconciliacao deste release nao adicionam warnings (scripts/check/*.mjs sao eslint-ignored, o teste novo de check-fabricated-docs nao usa any). Mesmo precedente de _rebaseline_2026_06_18_v3828_cycle_close. Apertar via --require-tighten no fim do ciclo seguinte. ALÉM disso, o step Require-tighten (blocking) exigiu apertar 2 métricas que MELHORARAM no ciclo (medidas no CI do PR #4126): coverage.auth.lines 69->90 (CI mediu 92.6; piso ~2pt-abaixo-do-real anti-flake, dentro do tightenSlack 10) e openapiCoverage.pct 37.6->38.4 (rotas novas documentadas). Melhorias legitimas travadas no baseline, nao gaming. Autorizado pelo operador (release end-to-end, validado na VPS).", - "_comment_mutationScore": "Per-module COVERED mutation score floors (detected/(detected+survived)), seeded ~2pt below the first full measurement (run 27823984918: split batches a1/a2/b1/b2/c1/c2/d + e/f/g/h/i). direction:up + dedicatedGate:true -> enforced ONLY by check-mutation-ratchet.mjs (the generic check-quality-ratchet skips dedicatedGate metrics), in the nightly-mutation aggregation job." + "_comment_mutationScore": "Per-module COVERED mutation score floors (detected/(detected+survived)), seeded ~2pt below the first full measurement (run 27823984918: split batches a1/a2/b1/b2/c1/c2/d + e/f/g/h/i). direction:up + dedicatedGate:true -> enforced ONLY by check-mutation-ratchet.mjs (the generic check-quality-ratchet skips dedicatedGate metrics), in the nightly-mutation aggregation job.", + "_zizmor_rebaseline_2026_06_19_r1_redundancy": "zizmorFindings 139 -> 145. Quebra: +3 drift PRE-EXISTENTE da base release/v3.8.30 a23d0d678 (medido com minhas mudancas stashed = 142 > 139; o fast-path do release nao roda check:workflows --ratchet) + 3 do novo workflow mutation-redundancy.yml (R1 disableBail): exatamente 3 unpinned-uses de actions/checkout@v7, actions/setup-node@v6, actions/upload-artifact@v7 — a MESMA convencao @vN deliberada e INTOCADA de todos os workflows (ver _scanner_harden_workflows_2026_06_16), identica ao nightly-mutation.yml. SHA-pinar so este workflow violaria a convencao. NOTA DE COLISAO CROSS-PR: o PR #4321 (a11y) tambem rebaselina este metric 139->145 (+3 do job a11y) off a MESMA base — se ambos mergearem, o total real vira 148 (142 base + 3 a11y + 3 r1) e o segundo a mergear precisa reconciliar zizmorFindings -> 148 (mesmo padrao release-volatil dos baselines de complexity/eslint)." } diff --git a/scripts/quality/mutation-radiography.mjs b/scripts/quality/mutation-radiography.mjs index c4381dcbbc..792f88c19e 100644 --- a/scripts/quality/mutation-radiography.mjs +++ b/scripts/quality/mutation-radiography.mjs @@ -137,6 +137,34 @@ export function aggregateRadiography(reports, allTestFiles) { return materialize(total, [...universe]); } +/** + * R1 prune-candidate list: the test files with ZERO unique kills — 🔴 empty (kills no + * mutant) ∪ 🟠 redundant (every mutant it kills is also killed by ≥1 other file). Files + * with ≥1 unique kill (🟢 unique / 🟡 overlapping) are NEVER candidates — removing one + * would drop a mutant's only killer and lower the mutation score. + * + * IMPORTANT: 🟠 redundant is only ACCURATE when the reports come from a `disableBail:true` + * run (killedBy lists EVERY killer). Under the bail-on-first nightly, redundant is + * understated — see the module caveat. Pass disableBail reports here (mutation-redundancy.yml). + * + * @param {object[]} reports parsed mutation.json objects (one per batch) + * @param {string[]} [allTestFiles] universe; defaults to the union of testFiles keys + * @returns {{ classification: object, empty: string[], redundant: string[], candidates: string[] }} + */ +export function redundancyCandidates(reports, allTestFiles) { + const classification = aggregateRadiography(reports, allTestFiles); + const empty = []; + const redundant = []; + for (const [file, info] of Object.entries(classification)) { + if (info.class === "empty") empty.push(file); + else if (info.class === "redundant") redundant.push(file); + } + empty.sort((a, b) => a.localeCompare(b)); + redundant.sort((a, b) => a.localeCompare(b)); + const candidates = [...empty, ...redundant].sort((a, b) => a.localeCompare(b)); + return { classification, empty, redundant, candidates }; +} + // ── CLI ────────────────────────────────────────────────────────────────────── function tapTestFilesUniverse() { @@ -206,18 +234,54 @@ function renderMarkdown(classification) { return lines.join("\n"); } +const FLAGS = new Set(["--no-conf-universe", "--candidates"]); + +function renderCandidates({ empty, redundant, candidates }) { + const lines = []; + lines.push("# R1 — Test-redundancy prune candidates (disableBail)"); + lines.push(""); + lines.push( + `Test files with ZERO unique kills: **${candidates.length}** ` + + `(🔴 empty ${empty.length} + 🟠 redundant ${redundant.length}).` + ); + lines.push(""); + lines.push( + "> Accurate ONLY for a `disableBail:true` run (killedBy lists every killer). " + + "These are CANDIDATES, not deletions: exclude security/contract/repro tests " + + "(routeGuard, OAuth, error-sanitization, *-repro*/*-regression*/issue-linked) and " + + "require human review before removing any (R1 human gate)." + ); + lines.push(""); + lines.push(`## 🔴 empty — kills no mutant (${empty.length})`); + lines.push(""); + if (empty.length === 0) lines.push("_none_"); + else for (const f of empty) lines.push(`- ${f}`); + lines.push(""); + lines.push(`## 🟠 redundant — every kill shared with another file (${redundant.length})`); + lines.push(""); + if (redundant.length === 0) lines.push("_none_"); + else for (const f of redundant) lines.push(`- ${f}`); + lines.push(""); + return lines.join("\n"); +} + function main(argv) { - const args = argv.filter((a) => a !== "--no-conf-universe"); + const wantCandidates = argv.includes("--candidates"); const useConfUniverse = !argv.includes("--no-conf-universe"); - const paths = args.slice(2); + const paths = argv.slice(2).filter((a) => !FLAGS.has(a)); if (paths.length === 0) { process.stderr.write( - "usage: mutation-radiography.mjs [ ...] [--no-conf-universe]\n" + "usage: mutation-radiography.mjs [ ...] " + + "[--candidates] [--no-conf-universe]\n" ); process.exit(2); } const reports = paths.map(loadMutationReport); const universe = useConfUniverse ? tapTestFilesUniverse() : null; + if (wantCandidates) { + process.stdout.write(renderCandidates(redundancyCandidates(reports, universe || undefined)) + "\n"); + return; + } const classification = aggregateRadiography(reports, universe || undefined); process.stdout.write(renderMarkdown(classification) + "\n"); } diff --git a/stryker.disablebail.json b/stryker.disablebail.json new file mode 100644 index 0000000000..cbf2953410 --- /dev/null +++ b/stryker.disablebail.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://stryker-mutator.io/schemas/stryker-schema.json", + "_comment": "Override one-off para medir REDUNDÂNCIA de testes (R1, Quality Gate v2 / Fase 9 T5). Estende stryker.conf.json mas (a) desliga o bail -> grava TODOS os testes que matam cada mutante em killedBy (sob bail, killedBy lista só o PRIMEIRO killer, então 🟠 redundant fica subestimado e 🟢 unique superestimado — ver o caveat em scripts/quality/mutation-radiography.mjs); (b) desliga o incremental -> medição limpa, sem reaproveitar killedBy parcial de runs com bail. NÃO é usado pelo nightly (nightly-mutation.yml continua com bail+incremental); só roda via .github/workflows/mutation-redundancy.yml (workflow_dispatch). Consome-se o mutation.json resultante com 'mutation-radiography.mjs --candidates' (o killedBy completo torna 🟠 redundant ACURADO).", + "extends": "./stryker.conf.json", + "disableBail": true, + "incremental": false +} diff --git a/tests/unit/build/mutation-radiography.test.ts b/tests/unit/build/mutation-radiography.test.ts index 102fdaa81c..675a03af96 100644 --- a/tests/unit/build/mutation-radiography.test.ts +++ b/tests/unit/build/mutation-radiography.test.ts @@ -4,6 +4,7 @@ import { classifyTestFiles, aggregateRadiography, classifyFromCounts, + redundancyCandidates, } from "../../../scripts/quality/mutation-radiography.mjs"; // ── classifyTestFiles: the plan's canonical fixture ────────────────────────── @@ -89,6 +90,33 @@ test("classifyFromCounts applies the threshold rules", () => { assert.equal(classifyFromCounts(1, 5), "overlapping"); // shared > unique }); +// ── redundancyCandidates (R1): the prune-candidate list = 🔴 empty ∪ 🟠 redundant. +// A file is a candidate iff it has ZERO unique kills (kills nothing, OR every mutant it +// kills is also killed by another file). Files with ≥1 unique kill (🟢/🟡) are NEVER +// candidates. Under a disableBail run killedBy is COMPLETE, so this list is accurate. ── +test("redundancyCandidates returns empty ∪ redundant files; never a file with a unique kill", () => { + const report = { + testFiles: { + "tests/unit/x.test.ts": { tests: [{ id: "0", name: "tests/unit/x.test.ts" }] }, + "tests/unit/y.test.ts": { tests: [{ id: "1", name: "tests/unit/y.test.ts" }] }, + "tests/unit/z.test.ts": { tests: [{ id: "2", name: "tests/unit/z.test.ts" }] }, + }, + files: { + "src/m.ts": { + mutants: [ + { id: "m1", status: "Killed", killedBy: ["0"] }, // x alone -> unique (KEEP) + { id: "m2", status: "Killed", killedBy: ["0", "1"] }, // x+y; y only ever shared + ], + }, + }, + }; + const r = redundancyCandidates([report]); + assert.deepEqual(r.empty, ["tests/unit/z.test.ts"]); // kills nothing + assert.deepEqual(r.redundant, ["tests/unit/y.test.ts"]); // kills only shared mutants + assert.deepEqual(r.candidates, ["tests/unit/y.test.ts", "tests/unit/z.test.ts"]); + assert.ok(!r.candidates.includes("tests/unit/x.test.ts")); // has a unique kill -> safe +}); + // ── aggregateRadiography: merge per-batch reports at the FILE level (ids are // per-run, so each report is classified independently then summed). A file can // be empty in one batch but unique in another -> unique overall. ─────────────