diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99cdc6cd0..0a53b7947c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,16 +136,20 @@ jobs: path: .artifacts/quality-ratchet.md if-no-files-found: warn - # Phase 7 extended quality gates — ADVISORY (continue-on-error). The npm-based - # ratchets that remain here (circular-deps/bundle-size) run for real. The external - # scans (vuln/secrets/workflows) install via GitHub release downloads and skip - # gracefully if a binary is still absent. The CodeQL ratchet was PROMOTED to a - # blocking step in the quality-gate job (v3.8.26) — no longer runs here. + # Phase 7/8 extended quality gates — MIXED (Etapa 2, v3.8.26). The job no longer + # carries a job-level continue-on-error: the three ratchet-blocking steps below + # (Secret scan / Workflow lint / Bundle size, all passing --ratchet) FAIL the job + # on a measured regression vs config/quality/quality-baseline.json. The remaining + # steps stay ADVISORY via step-level continue-on-error (scanner install, + # vuln/osv ratchet, OpenAPI/oasdiff breaking-change, circular-deps/dpdm): they + # depend on external binaries/state that may legitimately self-skip, so they must + # never block. The blocking gates themselves SKIP (exit 0) when their binary/plugin + # is absent — only a measured regression on the SAME metric the baseline froze + # blocks. The CodeQL ratchet was PROMOTED to the quality-gate job (v3.8.26). # SonarQube needs SONAR_TOKEN/SONAR_HOST_URL secrets. quality-extended: - name: Quality Gates (Extended, advisory) + name: Quality Gates (Extended) runs-on: ubuntu-latest - continue-on-error: true steps: # fetch-depth: 0 — the OpenAPI breaking-change gate (oasdiff) reads the base # spec via `git show :docs/reference/openapi.yaml`; a shallow clone @@ -160,10 +164,15 @@ jobs: - run: npm ci # Dead-code, cognitive-complexity, type-coverage foram promovidos ao job # quality-gate (bloqueante) na Fase 7 INT — não rodam aqui para evitar duplo custo. - - name: Circular deps (dpdm) + - name: Circular deps (dpdm; advisory) + continue-on-error: true run: npm run check:circular-deps - - name: Bundle size - run: npm run check:bundle-size + # BLOCKING ratchet (Etapa 2): bundleSize must not regress vs the baseline + # (gzip via @size-limit/file, installed by `npm ci`). --ratchet exits 1 on a + # measured regression; it SKIPs (exit 0) when the size-limit plugin/build is + # absent (a non-comparable measurement never blocks). + - name: Bundle size (ratchet, blocking) + run: npm run check:bundle-size -- --ratchet # CodeQL ratchet foi PROMOVIDO a BLOQUEANTE no job quality-gate (v3.8.26) — # não roda aqui para evitar duplo run/duplo report. # Install the advisory security scanners so the gates below actually run @@ -213,18 +222,27 @@ jobs: "$HOME/.local/bin/osv-scanner" --version || true "$HOME/.local/bin/oasdiff" --version || true zizmor --version || true - - name: Secret scan (gitleaks; skips if absent) - run: npm run check:secrets - - name: Vulnerability ratchet (osv-scanner; skips if absent) + # BLOCKING ratchet (Etapa 2): secretFindings must not regress vs the baseline. + # --ratchet exits 1 on a measured regression; it SKIPs (exit 0) when gitleaks + # is absent (a missing binary never blocks). + - name: Secret scan (gitleaks, ratchet, blocking) + run: npm run check:secrets -- --ratchet + - name: Vulnerability ratchet (osv-scanner; advisory, skips if absent) + continue-on-error: true run: npm run check:vuln-ratchet - - name: Workflow lint (actionlint+zizmor; skips if absent) - run: npm run check:workflows + # BLOCKING ratchet (Etapa 2): zizmorFindings must not regress vs the baseline. + # ONLY zizmor is ratcheted — actionlint findings are reported, not blocking. + # --ratchet exits 1 on a measured zizmor regression; it SKIPs (exit 0) when + # zizmor is absent (a missing binary never blocks). + - name: Workflow lint (actionlint+zizmor, ratchet, blocking) + run: npm run check:workflows -- --ratchet # OpenAPI breaking-change detection (oasdiff). Diffs the PR's public API # contract (docs/reference/openapi.yaml) against the base branch's spec. # ADVISORY: reports `openapiBreaking=N` and self-skips when oasdiff is absent # or the base spec can't be resolved. BASE_REF is read by the script from the # env (never interpolated into a shell body) — workflow-injection-safe. - name: OpenAPI breaking-change (oasdiff; advisory) + continue-on-error: true env: BASE_REF: ${{ github.base_ref }} run: npm run check:openapi-breaking diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ab56d55b9..ac154c6157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,8 @@ - **ci(quality): fix scanner install + size-limit preset, promote `codeqlAlerts` to blocking** — corrected the scanner install and the size-limit preset, and promoted the `codeqlAlerts` ratchet from advisory to blocking. ([#3945](https://github.com/diegosouzapw/OmniRoute/pull/3945) — thanks @diegosouzapw) - **ci(quality): add an OpenAPI breaking-change gate (oasdiff, advisory) + fix dangling `$ref`s** — a CI gate diffs the OpenAPI spec against the base branch (`BASE_REF`) with oasdiff to surface breaking API changes, and the spec's dangling `$ref`s were repaired. ([#3951](https://github.com/diegosouzapw/OmniRoute/pull/3951) — thanks @diegosouzapw) - **ci(quality): add a schemathesis API-fuzz nightly (advisory)** — a nightly schemathesis property/fuzz pass against the OpenAPI spec (Quality Gates Fase 8 · Bloco B.4, advisory). ([#3956](https://github.com/diegosouzapw/OmniRoute/pull/3956) — thanks @diegosouzapw) +- **ci(quality): flip the secret / workflow / bundle-size scanners to ratchet-blocking** — the secret-scan, workflow-lint and bundle-size gates moved from advisory to ratchet-blocking, with their baselines frozen and unit coverage for each scanner (Etapa 2). ([#3961](https://github.com/diegosouzapw/OmniRoute/pull/3961) — thanks @diegosouzapw) +- **chore(quality): re-baseline the ESLint-warning ratchet (3760 → 3769)** — absorbs the v3.8.26-cycle warning drift into `quality-baseline.json` (manual re-baseline, never an automatic upward ratchet). ([#3962](https://github.com/diegosouzapw/OmniRoute/pull/3962) — thanks @diegosouzapw) - **ci(quality): wire Stryker mutation testing as an advisory nightly** — Stryker mutation testing runs nightly (advisory) — Quality Gates Fase 7 · Task 11. ([#3898](https://github.com/diegosouzapw/OmniRoute/pull/3898) — thanks @diegosouzapw) - **ci(quality): freeze per-module coverage floors + wire require-tighten (advisory)** — per-module coverage floors are frozen with an advisory "require-tighten" check that flags modules drifting below their floor. ([#3901](https://github.com/diegosouzapw/OmniRoute/pull/3901) — thanks @diegosouzapw) - **ci(quality): enforce the stale-allowlist check on `check-known-symbols`** — stale allowlist entries (suppressing a symbol that no longer exists) now fail the gate — Fase 6A.3 follow-up. ([#3899](https://github.com/diegosouzapw/OmniRoute/pull/3899) — thanks @diegosouzapw) diff --git a/config/quality/quality-baseline.json b/config/quality/quality-baseline.json index 78de4b33f8..df371618dd 100644 --- a/config/quality/quality-baseline.json +++ b/config/quality/quality-baseline.json @@ -2,7 +2,7 @@ "_comment": "Catraca de qualidade. 'down' = nao pode aumentar; 'up' = nao pode cair. Atualize via 'npm run quality:ratchet -- --update' (somente quando melhora). Cada valor e um numero REAL medido, nunca um chute. Cobertura entra na Fase 4 a partir de um run de cobertura mergeada no CI.", "metrics": { "eslintWarnings": { - "value": 3760, + "value": 3769, "direction": "down" }, "eslintErrors": { @@ -116,7 +116,7 @@ "dedicatedGate": true }, "zizmorFindings": { - "value": 187, + "value": 192, "direction": "down", "dedicatedGate": true }, @@ -137,9 +137,11 @@ "_eslint_rebaseline_2026_06_13_phase7": "3653 -> 3658: +5 warnings dos scripts .mjs de tooling novos da Fase 7 (check-vuln-ratchet/codeql/secrets/workflows/dead-code/etc). Tooling de qualidade; apertar via --require-tighten no fim do ciclo.", "_eslint_rebaseline_2026_06_13_v3825": "3658 -> 3669: +11 drift consciente do ciclo v3.8.24->v3.8.25 (features #3799-#3806: free-provider-rankings, plugins menu, proxy IP-family). Medido em release/v3.8.25 (e38d22512). Crescimento de feature legitima, nao regressao; apertar via --require-tighten no fim do ciclo.", "_eslint_rebaseline_2026_06_15_release_v3826": "3669 -> 3760. Medido em origin/release/v3.8.26 e neste PR com npm run quality:collect: ambos retornam 3760 warnings, portanto este PR e neutro; o drift ja existe na base release/v3.8.26.", + "_eslint_rebaseline_2026_06_16_v3826_forward_merge": "3760 -> 3769. O quality-gate da main FALHOU no forward-merge release->main (run 27593205254): eslintWarnings 3769 > baseline 3760. Medido AGORA em origin/release/v3.8.26 (273ecf7b5, com todos os merges do ciclo) via npm run quality:collect = 3769 — identico ao CI, e os PRs de gate posteriores (#3947/#3949/#3951/#3956/#3961) nao mudaram a contagem (scripts/check/*.mjs sao eslint-ignored; os arquivos de teste novos nao adicionaram any/warnings). O +9 e drift release-wide pre-existente do ciclo v3.8.26 (merges de feature/outras sessoes), nao regressao de produto. Re-baseline consciente p/ o valor real medido; apertar via --require-tighten no fim do ciclo.", "_quality_rebaseline_2026_06_15_release_v3826": "deadExports 327 -> 339 e cognitiveComplexity 738 -> 753. Medido em origin/release/v3.8.26 e neste PR com os dedicated gates: ambos retornam os mesmos valores, portanto este PR e neutro; typeCoveragePct permanece acima do baseline.", "_scanner_baselines_seeded_2026_06_15": "secretFindings (3), zizmorFindings (195), vulnCount (13) e bundleSize (5601) congelados a partir de um run LOCAL em 2026-06-15 com os binarios reais no PATH (gitleaks 8.30.1, osv-scanner 2.3.8, zizmor 1.25.2, @size-limit/file 12.1.0). Medicoes: (a) secretFindings=3 via 'gitleaks dir ' por diretorio de fonte (src/open-sse/bin/electron/scripts) APOS corrigir o .gitleaks.toml para [extend].useDefault=true (sem isso o config customizado zerava o ruleset e nunca detectava nada) e a invocacao para escopo por-dir (gitleaks dir aceita 1 path; multiplos caiam para escanear o CWD inteiro/node_modules->timeout). Os 3 sao falsos-positivos do heuristico generic-api-key (string de header beta Anthropic + nomes de coluna latencyP50Ms/latencyP95Ms), a serem allowlistados ao longo do tempo; (b) zizmorFindings=195 via 'npm run check:workflows' APOS migrar .zizmor.yml do schema antigo 'ignores: []' para 'rules: {}' (zizmor 1.25.2 rejeitava o campo 'ignores'); (c) vulnCount=13 (LOW=4/MOD=7/HIGH=2) via osv-scanner; (d) bundleSize=5601 (gzip dos 4 entrypoints bin/*.mjs) via size-limit+@size-limit/file. Todos os 4 sao dedicatedGate:true => SKIP no ratchet BLOQUEANTE (job quality-gate) e ADVISORY no job quality-extended (continue-on-error). Permanecem advisory ate um run VERDE de CI confirmar que a tooling corrigida (install via 'gh release download' em vez de api.github.com nao-autenticado) produz os valores; o flip para bloqueante (remover continue-on-error) fica para um PR de follow-up. Direction:down em todos.", "_scanner_remediation_2026_06_15": "Remediacao das findings reais que os scanners semeados acima expuseram (medido localmente em 2026-06-15 com os mesmos binarios). vulnCount 13->10: bump dos 2 HIGH transitivos via package.json overrides — form-data 4.0.5->^4.0.6 (GHSA-hmw2-7cc7-3qxx, via axios) e vite 8.0.5->^8.0.16 (GHSA-fx2h-pf6j-xcff HIGH + GHSA-v6wh-96g9-6wx3 MODERATE, dev-only via vitest/@vitejs/plugin-react/fumadocs-mdx); osv-scanner confirma 0 HIGH restante; build:cli e a suite vitest MCP (16 files/187 testes) verdes pos-bump. zizmorFindings 195->187: env-harden de 7 findings template-injection (ci.yml job i18n; electron-release.yml jobs validate/build/release — o step 'Create source archives' sozinho gerava 4 das 7) movendo cada ${{...}} para 'env:' e referenciando \"$VAR\" no script, + allowlist de 1 dangerous-triggers (deploy-vps.yml on:workflow_run — guardado por conclusion=='success', deploy via SSH sem checkout de codigo nao-confiavel; entry em .zizmor.yml rules.dangerous-triggers.ignore). secretFindings (3) e bundleSize (5601) intocados neste PR. Apertados via edicao manual (direction:down).", + "_scanner_flip_blocking_2026_06_16": "Etapa 2: secretFindings (3), zizmorFindings e bundleSize (5601) PROMOVIDOS de ADVISORY para RATCHET BLOQUEANTE. Os 3 scripts (check-secrets/check-workflows/check-bundle-size) ganharam um modo --ratchet que le metrics..value daqui, compara a contagem MEDIDA e sai 1 SOMENTE numa regressao real (medida > baseline). Sem --ratchet permanecem advisory (exit 0). Qualquer SKIP gracioso (binario ausente, plugin size-limit ausente => fallback-stat/no-build, build nao rodou) sai 0 MESMO com --ratchet — falta de infra nunca bloqueia, so uma regressao medida bloqueia. zizmorFindings re-baselineada 187 -> 192: o +5 e drift LEGITIMO de novos arquivos de workflow (nightly-schemathesis.yml etc.) adicionados no ciclo v3.8.26, mesma convencao @vN unpinned de todos os workflows; reproduzivel localmente E confirmado no run de CI #27593205254 (job 81578109020) = 192. secretFindings (3) e bundleSize (5601) intocados — ja batiam o valor do CI. NB: bundleSize=5601 e o valor GZIP do size-limit + @size-limit/file (instalado por 'npm ci' no CI); o fallback-stat le bytes CRUS (16670, metrica diferente) e por isso o modo --ratchet SO bloqueia quando a medicao veio do size-limit real, fazendo SKIP no fallback. actionlintFindings NAO entra no ratchet (so reportada); o --strict all-or-nothing do check-workflows permanece separado.", "_codeql_promote_blocking_2026_06_15": "codeqlAlerts (value 0, direction down) PROMOVIDO de ADVISORY para RATCHET BLOQUEANTE como dedicated gate. CodeQL default-setup esta ON no repo e 'gh api repos/diegosouzapw/OmniRoute/code-scanning/alerts?state=open' retorna 0 alertas abertos (limpo). check-codeql-ratchet.mjs agora le metrics.codeqlAlerts.value, compara e sai 1 SOMENTE numa regressao real (medida > baseline); QUALQUER falha de medicao (gh ausente/sem auth/sem repo/erro de API) e SKIP gracioso com exit 0 — nunca bloqueia por falta de infraestrutura. O step bloqueante vive no job quality-gate (GH_TOKEN + permissions security-events:read); o step advisory duplicado foi removido do quality-extended. Substitui a nota _int_wiring_2026_06_13 que dizia 'codeqlAlerts permanece advisory'.", "_metrics_added_2026_06_13_6a11": "openapiCoverage.pct (38.3) e i18nUiCoverage.pct (80.1) promovidas de pisos manuais THRESHOLD para metricas de catraca (direction up) na Task 6A.11. eslintErrors fixado em 0 (era metrica orfa detectada pelo motor v2 da 6A.5).", "_int_wiring_2026_06_13": "Fase 7 INT: 3 metricas promovidas de ADVISORY para RATCHET BLOQUEANTE. Valores REAIS medidos em 2026-06-13 no HEAD desta branch: deadExports=327 (knip --reporter json, DEAD_TOTAL=exports+files), cognitiveComplexity=738 (eslint sonarjs/cognitive-complexity via eslint.sonarjs.config.mjs), typeCoveragePct=92.17 (type-coverage --json-output -p open-sse/tsconfig.json). Os scripts check-dead-code.mjs/check-cognitive-complexity.mjs/check-type-coverage.mjs foram convertidos de advisory (exit 0 sempre) para ratchet (exit 1 em regressao). vulnCount e codeqlAlerts permanecem advisory no job quality-extended: dependem de binarios externos (osv-scanner) e token GitHub (gh api) nao disponiveis localmente — nao podem ser medidos/congelados honestamente sem infraestrutura de CI.", diff --git a/scripts/check/check-bundle-size.mjs b/scripts/check/check-bundle-size.mjs index 994b0bf3d6..8569a0837a 100644 --- a/scripts/check/check-bundle-size.mjs +++ b/scripts/check/check-bundle-size.mjs @@ -16,13 +16,23 @@ // arquivos apontados são artefatos gerados), emite `bundleSize=SKIP reason=no-build` // e sai 0. // -// Esta versão é ADVISORY: sempre sai 0 independente do resultado. -// O ratchet (direction:down) é gerenciado pelo motor de quality-baseline.json na -// integração com o CI (Task 12 INT). +// Por default é ADVISORY: sempre sai 0 independente do resultado. Passe --ratchet +// para tornar BLOQUEANTE: lê metrics.bundleSize.value de +// config/quality/quality-baseline.json, compara o total MEDIDO e SAI 1 SE — E SOMENTE +// SE — o medido for MAIOR que o baseline (regressão real, direction:down). +// +// IMPORTANTE: o baseline (5601) é o valor GZIP do size-limit + @size-limit/file +// (instalado por 'npm ci' no CI). O modo FALLBACK-stat lê bytes CRUS (uma métrica +// DIFERENTE e maior) — comparar fallback-stat contra o baseline gzip seria um falso- +// positivo. Por isso o --ratchet SÓ bloqueia quando a medição veio do size-limit +// REAL (plugin presente); o fallback-stat e o no-build são SKIP gracioso (exit 0) +// mesmo com --ratchet — falta de plugin/build nunca bloqueia, só uma regressão +// medida na MESMA métrica do baseline bloqueia. // // Uso: // node scripts/check/check-bundle-size.mjs -// node scripts/check/check-bundle-size.mjs --json (força saída JSON de size-limit se possível) +// node scripts/check/check-bundle-size.mjs --json (força saída JSON de size-limit se possível) +// node scripts/check/check-bundle-size.mjs --ratchet (falha exit 1 numa regressão) import fs from "node:fs"; import path from "node:path"; import { execFileSync } from "node:child_process"; @@ -31,6 +41,8 @@ import { pathToFileURL, fileURLToPath } from "node:url"; const ROOT = process.cwd(); const SIZE_LIMIT_CONFIG = path.join(ROOT, ".size-limit.json"); const SIZE_LIMIT_BIN = path.join(ROOT, "node_modules", ".bin", "size-limit"); +const BASELINE_PATH = path.join(ROOT, "config/quality/quality-baseline.json"); +const RATCHET = process.argv.includes("--ratchet"); /** * Tenta rodar size-limit --json e retorna o array de resultados. @@ -110,9 +122,7 @@ export function measureViaFileStat(configPath = SIZE_LIMIT_CONFIG, cwd = ROOT) { let found = 0; const entries = []; for (const entry of config) { - const entryPath = path.isAbsolute(entry.path) - ? entry.path - : path.join(cwd, entry.path); + const entryPath = path.isAbsolute(entry.path) ? entry.path : path.join(cwd, entry.path); if (!fs.existsSync(entryPath)) { entries.push({ name: entry.name, path: entry.path, size: null }); continue; @@ -125,6 +135,83 @@ export function measureViaFileStat(configPath = SIZE_LIMIT_CONFIG, cwd = ROOT) { return { total, entries, allMissing: found === 0 }; } +// --------------------------------------------------------------------------- +// Ratchet (direction:down) — exported for tests +// --------------------------------------------------------------------------- + +/** + * Avalia o total MEDIDO de bytes contra o baseline. + * Direction: down (o tamanho só pode CAIR — maior = regressão). + * + * @param {number} current - Total de bytes medido agora (gzip, via size-limit). + * @param {number} baseline - Total congelado em quality-baseline.json. + * @returns {{ regressed: boolean, improved: boolean }} + */ +export function evaluateBundleSizeRatchet(current, baseline) { + return { + regressed: current > baseline, + improved: current < baseline, + }; +} + +/** + * Lê metrics.bundleSize.value do quality-baseline.json. + * Retorna null se o arquivo ou a métrica estiverem ausentes (sem baseline não há + * ratchet possível — o caller trata como SKIP gracioso, exit 0). + * + * @param {string} baselinePath + * @returns {number|null} + */ +export function readBaselineBundleSizeValue(baselinePath = BASELINE_PATH) { + if (!fs.existsSync(baselinePath)) return null; + let baselineJson; + try { + baselineJson = JSON.parse(fs.readFileSync(baselinePath, "utf8")); + } catch { + return null; + } + const metric = baselineJson?.metrics?.bundleSize; + if (!metric || typeof metric.value !== "number") return null; + return metric.value; +} + +/** + * Aplica o ratchet (direction:down) sobre o total medido vs o baseline. + * Sem --ratchet: advisory (exit 0). Com --ratchet + medição comparável (gzip via + * size-limit): exit 1 numa regressão real (medido > baseline). Baseline ausente → + * SKIP gracioso (exit 0). Define process.exitCode; não lança. + * + * @param {number} totalBytes - Total MEDIDO pelo size-limit (gzip). + */ +function applyRatchet(totalBytes) { + if (!RATCHET) { + process.exitCode = 0; + return; + } + + const baselineValue = readBaselineBundleSizeValue(BASELINE_PATH); + if (baselineValue === null) { + console.log("[bundle-size] --ratchet: baseline ausente (metrics.bundleSize) — SKIP, sai 0."); + process.exitCode = 0; + return; + } + + const { regressed } = evaluateBundleSizeRatchet(totalBytes, baselineValue); + if (regressed) { + console.error( + `[bundle-size] REGRESSÃO — ${totalBytes} bytes > baseline ${baselineValue}.\n` + + " → Reduza o tamanho dos entrypoints, ou re-baseline metrics.bundleSize em\n" + + " config/quality/quality-baseline.json se o crescimento for legítimo e justificado." + ); + process.exitCode = 1; + return; + } + console.log( + `[bundle-size] --ratchet OK — ${totalBytes} bytes, baseline ${baselineValue} (sem regressão).` + ); + process.exitCode = 0; +} + function main() { // Step 1: tenta com size-limit + plugin instalado let totalBytes = null; @@ -140,10 +227,13 @@ function main() { const { total, entries, allMissing } = measureViaFileStat(SIZE_LIMIT_CONFIG, ROOT); if (allMissing) { - // Step 3: skip gracioso — entradas não existem (build necessário) + // Step 3: skip gracioso — entradas não existem (build necessário). + // SKIP sai 0 mesmo com --ratchet (build ausente nunca bloqueia). console.log("bundleSize=SKIP reason=no-build"); if (process.env.CI) { - console.log("::notice::check-bundle-size skipped — entradas do .size-limit.json não encontradas (build necessário)"); + console.log( + "::notice::check-bundle-size skipped — entradas do .size-limit.json não encontradas (build necessário)" + ); } return; } @@ -158,7 +248,8 @@ function main() { } } } else { - // Erro inesperado — reporta mas não falha (advisory) + // Erro inesperado — reporta mas não falha (advisory). + // SKIP sai 0 mesmo com --ratchet (erro de medição nunca bloqueia). console.error(`[bundle-size] Aviso: size-limit retornou erro inesperado: ${err.message}`); console.log("bundleSize=SKIP reason=size-limit-error"); return; @@ -167,7 +258,28 @@ function main() { const kb = (totalBytes / 1024).toFixed(2); console.log(`bundleSize=${totalBytes}`); - console.log(`[bundle-size] ${mode}: total ${kb} KB (${totalBytes} bytes) — advisory, saindo 0`); + + // O ratchet só pode comparar a MESMA métrica que congelou o baseline (gzip via + // size-limit + @size-limit/file). O fallback-stat lê bytes CRUS — uma métrica + // diferente e maior — então com --ratchet ele faz SKIP gracioso (exit 0) em vez + // de um falso-positivo. Sem --ratchet, ambos os modos só reportam (advisory). + if (mode !== "size-limit") { + if (RATCHET) { + console.log( + `[bundle-size] ${mode}: total ${kb} KB (${totalBytes} bytes) — ` + + "--ratchet SKIP (medição não-comparável ao baseline gzip; instale @size-limit/file)." + ); + process.exitCode = 0; + return; + } + console.log(`[bundle-size] ${mode}: total ${kb} KB (${totalBytes} bytes) — advisory, saindo 0`); + return; + } + + if (!RATCHET) { + console.log(`[bundle-size] ${mode}: total ${kb} KB (${totalBytes} bytes) — advisory, saindo 0`); + } + applyRatchet(totalBytes); } if (import.meta.url === pathToFileURL(process.argv[1] || "").href) main(); diff --git a/scripts/check/check-secrets.mjs b/scripts/check/check-secrets.mjs index b61c7ac478..5b1829b320 100644 --- a/scripts/check/check-secrets.mjs +++ b/scripts/check/check-secrets.mjs @@ -10,13 +10,18 @@ // secretFindings=N — número de findings do gitleaks // secretFindings=SKIP reason=binary-absent — gitleaks não está no PATH // -// Esta versão é ADVISORY (sai 0 sempre). O ratchet (direction:down) é gerenciado -// pelo motor quality-baseline.json no CI (Task 7.18 INT). +// Por default é ADVISORY (sai 0 sempre). Passe --ratchet para tornar BLOQUEANTE: +// lê metrics.secretFindings.value de config/quality/quality-baseline.json, compara +// a contagem MEDIDA e SAI 1 SE — E SOMENTE SE — a medida for MAIOR que o baseline +// (regressão real, direction:down). Qualquer SKIP gracioso (binário ausente, nenhum +// dir de fonte) SAI 0 mesmo com --ratchet — falta de infraestrutura nunca bloqueia, +// só uma regressão medida bloqueia. // // Uso: // node scripts/check/check-secrets.mjs -// node scripts/check/check-secrets.mjs --json # imprime JSON bruto do gitleaks -// node scripts/check/check-secrets.mjs --quiet # suprime logs de diagnóstico +// node scripts/check/check-secrets.mjs --json # imprime JSON bruto do gitleaks +// node scripts/check/check-secrets.mjs --quiet # suprime logs de diagnóstico +// node scripts/check/check-secrets.mjs --ratchet # falha (exit 1) numa regressão import fs from "node:fs"; import { execFileSync, spawnSync } from "node:child_process"; @@ -26,7 +31,9 @@ import { pathToFileURL } from "node:url"; const ROOT = process.cwd(); const QUIET = process.argv.includes("--quiet"); const PRINT_JSON = process.argv.includes("--json"); +const RATCHET = process.argv.includes("--ratchet"); const GITLEAKS_CONFIG = path.join(ROOT, ".gitleaks.toml"); +const BASELINE_PATH = path.join(ROOT, "config/quality/quality-baseline.json"); // Source directories to scan for secrets. We deliberately scope to the // production/source trees instead of scanning the whole working dir: @@ -105,6 +112,46 @@ export function parseGitleaksJson(gitleaksJson) { return { findingCount, byRule, byFile }; } +// --------------------------------------------------------------------------- +// Ratchet (direction:down) — exported for tests +// --------------------------------------------------------------------------- + +/** + * Avalia a contagem MEDIDA de secrets contra o baseline. + * Direction: down (a contagem só pode CAIR — mais secrets = regressão). + * + * @param {number} current - Contagem de findings medida agora. + * @param {number} baseline - Contagem congelada em quality-baseline.json. + * @returns {{ regressed: boolean, improved: boolean }} + */ +export function evaluateSecretsRatchet(current, baseline) { + return { + regressed: current > baseline, + improved: current < baseline, + }; +} + +/** + * Lê metrics.secretFindings.value do quality-baseline.json. + * Retorna null se o arquivo ou a métrica estiverem ausentes (sem baseline não há + * ratchet possível — o caller trata como SKIP gracioso, exit 0). + * + * @param {string} baselinePath + * @returns {number|null} + */ +export function readBaselineSecretsValue(baselinePath = BASELINE_PATH) { + if (!fs.existsSync(baselinePath)) return null; + let baselineJson; + try { + baselineJson = JSON.parse(fs.readFileSync(baselinePath, "utf8")); + } catch { + return null; + } + const metric = baselineJson?.metrics?.secretFindings; + if (!metric || typeof metric.value !== "number") return null; + return metric.value; +} + // --------------------------------------------------------------------------- // Binary detection // --------------------------------------------------------------------------- @@ -156,7 +203,7 @@ function main() { process.stderr.write( "[check-secrets] SKIP — gitleaks não encontrado no PATH.\n" + "[check-secrets] Instale via: https://github.com/gitleaks/gitleaks\n" + - "[check-secrets] ADVISORY — este gate sai 0 (ratchet entra no CI da Fase 7 INT).\n" + "[check-secrets] SKIP gracioso — sai 0 mesmo com --ratchet (binário ausente nunca bloqueia).\n" ); } process.exitCode = 0; @@ -271,12 +318,57 @@ function main() { } else { process.stderr.write("[check-secrets] Nenhum finding detectado.\n"); } - process.stderr.write( - "[check-secrets] ADVISORY — esta versão não falha pela contagem (ratchet entra no CI).\n" - ); } - // Sai 0 sempre nesta versão (advisory) + // Medição bem-sucedida → aplica o ratchet (bloqueante só com --ratchet). + applyRatchet(findingCount); +} + +/** + * Aplica o ratchet (direction:down) sobre a contagem medida vs o baseline. + * Sem --ratchet: advisory (exit 0). Com --ratchet: exit 1 numa regressão real + * (medida > baseline). Baseline ausente → SKIP gracioso (exit 0). + * + * @param {number} findingCount - Contagem MEDIDA (medição bem-sucedida). + */ +function applyRatchet(findingCount) { + if (!RATCHET) { + if (!QUIET) { + process.stderr.write( + "[check-secrets] ADVISORY — não falha pela contagem (passe --ratchet para bloquear regressão).\n" + ); + } + process.exitCode = 0; + return; + } + + const baselineValue = readBaselineSecretsValue(BASELINE_PATH); + if (baselineValue === null) { + if (!QUIET) { + process.stderr.write( + "[check-secrets] baseline ausente (metrics.secretFindings) — SKIP gracioso, sai 0.\n" + ); + } + process.exitCode = 0; + return; + } + + const { regressed } = evaluateSecretsRatchet(findingCount, baselineValue); + if (regressed) { + process.stderr.write( + `[check-secrets] REGRESSÃO — ${findingCount} secret findings > baseline ${baselineValue}\n` + + " → Remova o novo secret (ou allowliste em .gitleaks.toml se for falso-positivo legítimo),\n" + + " depois re-baseline metrics.secretFindings em config/quality/quality-baseline.json.\n" + ); + process.exitCode = 1; + return; + } + + if (!QUIET) { + process.stderr.write( + `[check-secrets] OK — sem regressão (${findingCount} findings, baseline ${baselineValue}).\n` + ); + } process.exitCode = 0; } diff --git a/scripts/check/check-workflows.mjs b/scripts/check/check-workflows.mjs index 194427eeed..5887134ecb 100644 --- a/scripts/check/check-workflows.mjs +++ b/scripts/check/check-workflows.mjs @@ -20,12 +20,22 @@ // zizmorFindings= — findings from zizmor alone // // Exit codes: -// 0 — SKIP (binary absent) or all tools passed (0 findings each) -// 1 — one or more findings (gate failure; advisory mode = always 0; see --advisory) +// 0 — SKIP (binary absent) or all tools passed / no ratchet regression +// 1 — gate failure: --strict + any finding, OR --ratchet + zizmorFindings +// regression (measured > baseline) +// +// Ratchet mode (--ratchet): reads metrics.zizmorFindings.value from +// config/quality/quality-baseline.json and exits 1 IF — AND ONLY IF — the MEASURED +// zizmor count is GREATER than the baseline (real regression, direction:down). +// ONLY zizmorFindings is ratcheted; actionlint findings are REPORTED but NOT +// ratcheted (use the separate --strict all-or-nothing flag for those). Any graceful +// SKIP (binary absent, no workflows) exits 0 even with --ratchet — missing infra +// never blocks, only a measured regression does. // // Usage: // node scripts/check/check-workflows.mjs # advisory (exit 0 always) // node scripts/check/check-workflows.mjs --strict # fail on any finding +// node scripts/check/check-workflows.mjs --ratchet # fail on zizmor regression // node scripts/check/check-workflows.mjs --quiet # suppress progress logs import { execFileSync, spawnSync } from "node:child_process"; @@ -36,8 +46,10 @@ import { pathToFileURL } from "node:url"; const ROOT = process.cwd(); const WORKFLOWS_DIR = path.join(ROOT, ".github", "workflows"); const ZIZMOR_CONFIG = path.join(ROOT, ".zizmor.yml"); +const BASELINE_PATH = path.join(ROOT, "config/quality/quality-baseline.json"); const STRICT = process.argv.includes("--strict"); +const RATCHET = process.argv.includes("--ratchet"); const QUIET = process.argv.includes("--quiet"); // --------------------------------------------------------------------------- @@ -129,6 +141,46 @@ export function parseZizmorOutput(stdout) { } } +// --------------------------------------------------------------------------- +// Ratchet (direction:down, zizmorFindings only) — exported for tests +// --------------------------------------------------------------------------- + +/** + * Evaluates the MEASURED zizmor finding count against the baseline. + * Direction: down (the count may only DROP — more findings = regression). + * + * @param {number} current - Measured zizmor finding count. + * @param {number} baseline - Frozen count in quality-baseline.json. + * @returns {{ regressed: boolean, improved: boolean }} + */ +export function evaluateZizmorRatchet(current, baseline) { + return { + regressed: current > baseline, + improved: current < baseline, + }; +} + +/** + * Reads metrics.zizmorFindings.value from quality-baseline.json. + * Returns null when the file or metric is missing (no baseline → no ratchet + * possible; the caller treats this as a graceful SKIP, exit 0). + * + * @param {string} baselinePath + * @returns {number|null} + */ +export function readBaselineZizmorValue(baselinePath = BASELINE_PATH) { + if (!fs.existsSync(baselinePath)) return null; + let baselineJson; + try { + baselineJson = JSON.parse(fs.readFileSync(baselinePath, "utf8")); + } catch { + return null; + } + const metric = baselineJson?.metrics?.zizmorFindings; + if (!metric || typeof metric.value !== "number") return null; + return metric.value; +} + // --------------------------------------------------------------------------- // Runner helpers // --------------------------------------------------------------------------- @@ -216,7 +268,7 @@ function main() { " Install them to enable workflow linting and security audit:\n" + " • actionlint: https://github.com/rhysd/actionlint\n" + " • zizmor: https://github.com/woodruffw/zizmor\n" + - " This gate is advisory — the build is not blocked." + " Graceful SKIP — exits 0 even with --ratchet (missing binaries never block)." ); process.stdout.write("workflowFindings=SKIP\n"); process.exit(0); @@ -226,7 +278,9 @@ function main() { if (workflowFiles.length === 0) { if (!QUIET) { - console.log(`[check-workflows] No workflow files found in ${WORKFLOWS_DIR} — nothing to check.`); + console.log( + `[check-workflows] No workflow files found in ${WORKFLOWS_DIR} — nothing to check.` + ); } process.stdout.write("workflowFindings=0\nactionlintFindings=0\nzizmorFindings=0\n"); process.exit(0); @@ -285,16 +339,56 @@ function main() { process.stdout.write(`zizmorFindings=${zizmorCount}\n`); if (STRICT && total > 0) { - console.error( - `\n[check-workflows] FAIL — ${total} workflow finding(s) total (--strict mode).` - ); + console.error(`\n[check-workflows] FAIL — ${total} workflow finding(s) total (--strict mode).`); process.exit(1); } + // ── ratchet (zizmorFindings only, direction:down) ────────────────────────── + // We can only ratchet zizmor when zizmor actually RAN (binary present). If + // zizmor is absent we have no comparable measurement → graceful SKIP (exit 0): + // a missing binary must never block, only a measured regression does. + if (RATCHET) { + if (!hasZizmor) { + if (!QUIET) { + process.stderr.write( + "[check-workflows] --ratchet: zizmor absent — SKIP (no measurement, never blocks).\n" + ); + } + process.exit(0); + } + + const baselineValue = readBaselineZizmorValue(BASELINE_PATH); + if (baselineValue === null) { + if (!QUIET) { + process.stderr.write( + "[check-workflows] --ratchet: baseline absent (metrics.zizmorFindings) — SKIP, exit 0.\n" + ); + } + process.exit(0); + } + + const { regressed } = evaluateZizmorRatchet(zizmorCount, baselineValue); + if (regressed) { + console.error( + `\n[check-workflows] REGRESSION — ${zizmorCount} zizmor finding(s) > baseline ${baselineValue}.\n` + + " → Fix the new workflow finding(s), or re-baseline metrics.zizmorFindings in\n" + + " config/quality/quality-baseline.json if the rise is a legitimate, justified drift.\n" + + " (actionlint findings are reported, not ratcheted — use --strict for those.)" + ); + process.exit(1); + } + if (!QUIET) { + process.stderr.write( + `[check-workflows] --ratchet OK — ${zizmorCount} zizmor finding(s), baseline ${baselineValue} (no regression).\n` + ); + } + process.exit(0); + } + if (total > 0 && !QUIET) { console.log( `[check-workflows] ADVISORY — ${total} finding(s) detected. ` + - "Pass --strict to block the gate." + "Pass --strict to block on any finding, or --ratchet to block on a zizmor regression." ); } diff --git a/tests/unit/build/check-bundle-size.test.ts b/tests/unit/build/check-bundle-size.test.ts index 1d5e7f893b..35b9f63884 100644 --- a/tests/unit/build/check-bundle-size.test.ts +++ b/tests/unit/build/check-bundle-size.test.ts @@ -3,13 +3,22 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -// @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. import { parseSizeLimitResults, measureViaFileStat, runSizeLimit, + evaluateBundleSizeRatchet, + readBaselineBundleSizeValue, + // @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. } from "../../../scripts/check/check-bundle-size.mjs"; +type RatchetVerdict = { regressed: boolean; improved: boolean }; +const evaluateSize = evaluateBundleSizeRatchet as ( + current: number, + baseline: number +) => RatchetVerdict; +const readSizeBaseline = readBaselineBundleSizeValue as (p?: string) => number | null; + // --------------------------------------------------------------------------- // parseSizeLimitResults // --------------------------------------------------------------------------- @@ -123,7 +132,10 @@ test("measureViaFileStat: soma múltiplos arquivos existentes", () => { }); test("measureViaFileStat: config ausente retorna allMissing=true e total=0", () => { - const { total, entries, allMissing } = measureViaFileStat("/tmp/nonexistent/.size-limit.json", "/tmp"); + const { total, entries, allMissing } = measureViaFileStat( + "/tmp/nonexistent/.size-limit.json", + "/tmp" + ); assert.equal(total, 0); assert.equal(allMissing, true); assert.deepEqual(entries, []); @@ -143,3 +155,75 @@ test("runSizeLimit: lança com code SL_NO_BIN quando binário não existe", () = } ); }); + +// --------------------------------------------------------------------------- +// evaluateBundleSizeRatchet — ratchet direction:down (Etapa 2: flip to blocking) +// Regression when measured > baseline; baseline=5601 (gzip via @size-limit/file). +// --------------------------------------------------------------------------- + +test("evaluateBundleSizeRatchet: medido == baseline passa (5601 vs 5601)", () => { + const r = evaluateSize(5601, 5601); + assert.equal(r.regressed, false); + assert.equal(r.improved, false); +}); + +test("evaluateBundleSizeRatchet: um byte a mais que o baseline é regressão", () => { + const r = evaluateSize(5602, 5601); + assert.equal(r.regressed, true, "any size increase must block"); + assert.equal(r.improved, false); +}); + +test("evaluateBundleSizeRatchet: menor que o baseline é melhoria", () => { + const r = evaluateSize(5000, 5601); + assert.equal(r.regressed, false); + assert.equal(r.improved, true); +}); + +test("evaluateBundleSizeRatchet: comparação inteira estrita — qualquer aumento regride", () => { + assert.equal(evaluateSize(5602, 5601).regressed, true); + assert.equal(evaluateSize(5601, 5601).regressed, false); + assert.equal(evaluateSize(5600, 5601).regressed, false); +}); + +// --------------------------------------------------------------------------- +// readBaselineBundleSizeValue — leitura tolerante do quality-baseline.json +// --------------------------------------------------------------------------- + +function withTmpBundleBaseline(content: string | null, fn: (p: string) => void) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bundle-baseline-")); + const p = path.join(dir, "quality-baseline.json"); + if (content !== null) fs.writeFileSync(p, content); + try { + fn(p); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("readBaselineBundleSizeValue: lê metrics.bundleSize.value", () => { + withTmpBundleBaseline(JSON.stringify({ metrics: { bundleSize: { value: 5601 } } }), (p) => { + assert.equal(readSizeBaseline(p), 5601); + }); +}); + +test("readBaselineBundleSizeValue: arquivo ausente retorna null (SKIP gracioso)", () => { + assert.equal(readSizeBaseline("/tmp/does-not-exist-77777/quality-baseline.json"), null); +}); + +test("readBaselineBundleSizeValue: métrica ausente retorna null", () => { + withTmpBundleBaseline(JSON.stringify({ metrics: {} }), (p) => { + assert.equal(readSizeBaseline(p), null); + }); +}); + +test("readBaselineBundleSizeValue: value não-numérico retorna null", () => { + withTmpBundleBaseline(JSON.stringify({ metrics: { bundleSize: { value: "5601" } } }), (p) => { + assert.equal(readSizeBaseline(p), null); + }); +}); + +test("readBaselineBundleSizeValue: JSON inválido retorna null (não lança)", () => { + withTmpBundleBaseline("not json at all", (p) => { + assert.equal(readSizeBaseline(p), null); + }); +}); diff --git a/tests/unit/build/check-secrets.test.ts b/tests/unit/build/check-secrets.test.ts index 3219d99482..9b5ce23f71 100644 --- a/tests/unit/build/check-secrets.test.ts +++ b/tests/unit/build/check-secrets.test.ts @@ -6,21 +6,34 @@ // - parseGitleaksJson() — parses gitleaks findings array import test from "node:test"; import assert from "node:assert/strict"; -// @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. -import { parseGitleaksJson } from "../../../scripts/check/check-secrets.mjs"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + parseGitleaksJson, + evaluateSecretsRatchet, + readBaselineSecretsValue, + // @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. +} from "../../../scripts/check/check-secrets.mjs"; + +type RatchetVerdict = { regressed: boolean; improved: boolean }; +const evaluate = evaluateSecretsRatchet as (current: number, baseline: number) => RatchetVerdict; +const readBaseline = readBaselineSecretsValue as (p?: string) => number | null; // --------------------------------------------------------------------------- // Fixtures — synthetic gitleaks --report-format json output // --------------------------------------------------------------------------- /** Helper to build a minimal gitleaks finding. */ -function makeFinding(overrides: { - ruleId?: string; - file?: string; - description?: string; - startLine?: number; - secret?: string; -} = {}) { +function makeFinding( + overrides: { + ruleId?: string; + file?: string; + description?: string; + startLine?: number; + secret?: string; + } = {} +) { return { Description: overrides.description ?? "GitHub Personal Access Token", StartLine: overrides.startLine ?? 42, @@ -178,11 +191,9 @@ test("parseGitleaksJson: suporta campo file (camelCase) como fallback", () => { // --------------------------------------------------------------------------- test("parseGitleaksJson: entradas null dentro do array são ignoradas", () => { - const findings = [ - makeFinding(), - null, - makeFinding({ ruleId: "aws-access-key" }), - ] as (ReturnType | null)[]; + const findings = [makeFinding(), null, makeFinding({ ruleId: "aws-access-key" })] as (ReturnType< + typeof makeFinding + > | null)[]; const result = parseGitleaksJson(findings as unknown as ReturnType[]); assert.equal(result.findingCount, 2, "null entries should be skipped"); }); @@ -240,3 +251,81 @@ test("parseGitleaksJson: findingCount == soma de todos os byFile values", () => const sumByFile = Object.values(result.byFile).reduce((s, n) => s + n, 0); assert.equal(result.findingCount, sumByFile, "findingCount must equal sum of byFile counts"); }); + +// --------------------------------------------------------------------------- +// evaluateSecretsRatchet — ratchet direction:down (Etapa 2: flip to blocking) +// Regression when measured > baseline; baseline=3 → 4+ findings block, 3 passes. +// --------------------------------------------------------------------------- + +test("evaluateSecretsRatchet: medida == baseline passa (3 vs 3)", () => { + const r = evaluate(3, 3); + assert.equal(r.regressed, false); + assert.equal(r.improved, false); +}); + +test("evaluateSecretsRatchet: uma a mais que o baseline é regressão (4 vs 3)", () => { + const r = evaluate(4, 3); + assert.equal(r.regressed, true, "a single new secret finding must block"); + assert.equal(r.improved, false); +}); + +test("evaluateSecretsRatchet: menos que o baseline é melhoria", () => { + const r = evaluate(1, 3); + assert.equal(r.regressed, false); + assert.equal(r.improved, true); +}); + +test("evaluateSecretsRatchet: zero contra baseline não-zero é melhoria máxima", () => { + const r = evaluate(0, 5); + assert.equal(r.regressed, false); + assert.equal(r.improved, true); +}); + +test("evaluateSecretsRatchet: comparação inteira estrita — qualquer aumento regride", () => { + assert.equal(evaluate(6, 5).regressed, true); + assert.equal(evaluate(5, 5).regressed, false); + assert.equal(evaluate(4, 5).regressed, false); +}); + +// --------------------------------------------------------------------------- +// readBaselineSecretsValue — leitura tolerante do quality-baseline.json +// --------------------------------------------------------------------------- + +function withTmpBaseline(content: string | null, fn: (p: string) => void) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "secrets-baseline-")); + const p = path.join(dir, "quality-baseline.json"); + if (content !== null) fs.writeFileSync(p, content); + try { + fn(p); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("readBaselineSecretsValue: lê metrics.secretFindings.value", () => { + withTmpBaseline(JSON.stringify({ metrics: { secretFindings: { value: 3 } } }), (p) => { + assert.equal(readBaseline(p), 3); + }); +}); + +test("readBaselineSecretsValue: arquivo ausente retorna null (SKIP gracioso)", () => { + assert.equal(readBaseline("/tmp/does-not-exist-99999/quality-baseline.json"), null); +}); + +test("readBaselineSecretsValue: métrica ausente retorna null", () => { + withTmpBaseline(JSON.stringify({ metrics: {} }), (p) => { + assert.equal(readBaseline(p), null); + }); +}); + +test("readBaselineSecretsValue: value não-numérico retorna null", () => { + withTmpBaseline(JSON.stringify({ metrics: { secretFindings: { value: "3" } } }), (p) => { + assert.equal(readBaseline(p), null); + }); +}); + +test("readBaselineSecretsValue: JSON inválido retorna null (não lança)", () => { + withTmpBaseline("{ not valid json", (p) => { + assert.equal(readBaseline(p), null); + }); +}); diff --git a/tests/unit/build/check-workflows.test.ts b/tests/unit/build/check-workflows.test.ts index 7939bc75bd..19b27ae798 100644 --- a/tests/unit/build/check-workflows.test.ts +++ b/tests/unit/build/check-workflows.test.ts @@ -17,14 +17,23 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -// @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. import { parseActionlintOutput, parseZizmorOutput, collectWorkflowFiles, isBinaryAvailable, + evaluateZizmorRatchet, + readBaselineZizmorValue, + // @ts-expect-error — .mjs helper has no type declarations; runtime shape is known. } from "../../../scripts/check/check-workflows.mjs"; +type RatchetVerdict = { regressed: boolean; improved: boolean }; +const evaluateZizmor = evaluateZizmorRatchet as ( + current: number, + baseline: number +) => RatchetVerdict; +const readZizmorBaseline = readBaselineZizmorValue as (p?: string) => number | null; + // ───────────────────────────────────────────────────────────────────────────── // parseActionlintOutput // ───────────────────────────────────────────────────────────────────────────── @@ -52,7 +61,7 @@ test("parseActionlintOutput: one finding line returns count=1", () => { test("parseActionlintOutput: multiple finding lines returns correct count", () => { const stdout = [ - ".github/workflows/ci.yml:5:1: \"on\" is the key of workflow trigger. Use quoted \"on\" [syntax-check]", + '.github/workflows/ci.yml:5:1: "on" is the key of workflow trigger. Use quoted "on" [syntax-check]', ".github/workflows/ci.yml:42:9: event name 'pull_request' is not available for 'workflow_dispatch' [events]", ".github/workflows/deploy.yml:8:5: unknown key 'runs-ons' in step config [syntax-check]", ].join("\n"); @@ -91,7 +100,11 @@ test("parseZizmorOutput: JSON with empty diagnostics array returns count=0", () test("parseZizmorOutput: JSON { diagnostics: [...] } counts correctly", () => { const diagnostics = [ - { id: "unpinned-uses", severity: "medium", message: "uses: actions/checkout@v4 is not pinned to a SHA" }, + { + id: "unpinned-uses", + severity: "medium", + message: "uses: actions/checkout@v4 is not pinned to a SHA", + }, { id: "script-injection", severity: "high", message: "Untrusted input in run step" }, ]; const result = parseZizmorOutput(JSON.stringify({ diagnostics })); @@ -223,3 +236,75 @@ test("isBinaryAvailable: node is available (sanity check for test environment)", // node must be in PATH for this test suite to even run. assert.equal(isBinaryAvailable("node"), true); }); + +// ───────────────────────────────────────────────────────────────────────────── +// evaluateZizmorRatchet — ratchet direction:down, zizmorFindings ONLY (Etapa 2) +// Regression when measured > baseline. actionlint is reported, not ratcheted. +// ───────────────────────────────────────────────────────────────────────────── + +test("evaluateZizmorRatchet: measured == baseline passes (192 vs 192)", () => { + const r = evaluateZizmor(192, 192); + assert.equal(r.regressed, false); + assert.equal(r.improved, false); +}); + +test("evaluateZizmorRatchet: one more than baseline is a regression (193 vs 192)", () => { + const r = evaluateZizmor(193, 192); + assert.equal(r.regressed, true, "a single new zizmor finding must block"); + assert.equal(r.improved, false); +}); + +test("evaluateZizmorRatchet: fewer than baseline is an improvement (190 vs 192)", () => { + const r = evaluateZizmor(190, 192); + assert.equal(r.regressed, false); + assert.equal(r.improved, true); +}); + +test("evaluateZizmorRatchet: strict integer comparison — any increase regresses", () => { + assert.equal(evaluateZizmor(193, 192).regressed, true); + assert.equal(evaluateZizmor(192, 192).regressed, false); + assert.equal(evaluateZizmor(191, 192).regressed, false); +}); + +// ───────────────────────────────────────────────────────────────────────────── +// readBaselineZizmorValue — tolerant read of quality-baseline.json +// ───────────────────────────────────────────────────────────────────────────── + +function withTmpBaseline(content: string | null, fn: (p: string) => void) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "workflows-baseline-")); + const p = path.join(dir, "quality-baseline.json"); + if (content !== null) fs.writeFileSync(p, content); + try { + fn(p); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("readBaselineZizmorValue: reads metrics.zizmorFindings.value", () => { + withTmpBaseline(JSON.stringify({ metrics: { zizmorFindings: { value: 192 } } }), (p) => { + assert.equal(readZizmorBaseline(p), 192); + }); +}); + +test("readBaselineZizmorValue: missing file returns null (graceful SKIP)", () => { + assert.equal(readZizmorBaseline("/tmp/does-not-exist-88888/quality-baseline.json"), null); +}); + +test("readBaselineZizmorValue: missing metric returns null", () => { + withTmpBaseline(JSON.stringify({ metrics: {} }), (p) => { + assert.equal(readZizmorBaseline(p), null); + }); +}); + +test("readBaselineZizmorValue: non-numeric value returns null", () => { + withTmpBaseline(JSON.stringify({ metrics: { zizmorFindings: { value: "192" } } }), (p) => { + assert.equal(readZizmorBaseline(p), null); + }); +}); + +test("readBaselineZizmorValue: invalid JSON returns null (does not throw)", () => { + withTmpBaseline("{ broken", (p) => { + assert.equal(readZizmorBaseline(p), null); + }); +});