From e7214c72fc9d16d388fb67db2dc20c97b270adbf Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Wed, 16 Sep 2026 02:17:52 -0400 Subject: [PATCH] ci(acceptance): emit a shadow release-acceptance report next to release-green (#13701) Adds a shadow release-acceptance report alongside release-green: an inventory/reduce/oracle pipeline under `scripts/quality/release-acceptance/` with a JSON schema, fixtures and a workflow that uploads the report as an artifact. Contained by design, which is why it merges as-is: it runs only on push to `release/v*` and on manual dispatch (never on pull requests), the step is `continue-on-error`, `permissions: contents: read`, `persist-credentials: false`, and it consumes no secrets. Nothing in the product changes; the report is advisory until we decide to promote it. Validated as a combined board first (this PR merged with the 11 siblings of the same batch on the release tip): eslint on every changed file with the suppressions file, typecheck:core, check:open-sse-typecheck, complexity, cognitive-complexity, changelog-integrity, i18n new-key coverage, docs-sync, migration-numbering, provider-consistency and a duplicate-identifier audit all green, plus 275 passing / 0 failing focused node:test cases across the 28 test files the batch touches. Then re-validated alone on the fresh tip before this merge: conflicts re-resolved, file sizes rebaselined for this PR's own growth, eslint and this PR's focused tests re-run. Thanks @HouMinXi! --- .github/workflows/release-acceptance.yml | 42 +++ .../fixes/release-acceptance-shadow.md | 1 + config/quality/release-acceptance.schema.json | 183 +++++++++++ .../release-acceptance/closeOracle.mjs | 30 ++ .../quality/release-acceptance/inventory.mjs | 102 +++++++ .../release-acceptance/nodeReporter.mjs | 41 +++ scripts/quality/release-acceptance/reduce.mjs | 243 +++++++++++++++ .../release-acceptance/staticAdapter.mjs | 50 +++ scripts/quality/release-acceptance/types.mjs | 19 ++ .../quality/validate-release-acceptance.mjs | 81 +++++ .../release-acceptance/failed-pack-boot.json | 89 ++++++ .../release-acceptance/infra-pack-boot.json | 100 ++++++ .../release-acceptance/legacy-close-steps.yml | 39 +++ .../release-acceptance/plan-lint.json | 24 ++ .../shadow-manifests/lint.json | 51 ++++ .../unverified-required-skipped.json | 63 ++++ .../fixtures/release-acceptance/verified.json | 51 ++++ tests/unit/release-acceptance-cli.test.ts | 174 +++++++++++ .../release-acceptance-close-oracle.test.ts | 23 ++ .../unit/release-acceptance-inventory.test.ts | 40 +++ .../release-acceptance-node-reporter.test.ts | 86 ++++++ .../unit/release-acceptance-pack-boot.test.ts | 68 +++++ tests/unit/release-acceptance-reduce.test.ts | 287 ++++++++++++++++++ tests/unit/release-acceptance-schema.test.ts | 112 +++++++ .../release-acceptance-static-adapter.test.ts | 33 ++ 25 files changed, 2032 insertions(+) create mode 100644 .github/workflows/release-acceptance.yml create mode 100644 changelog.d/fixes/release-acceptance-shadow.md create mode 100644 config/quality/release-acceptance.schema.json create mode 100644 scripts/quality/release-acceptance/closeOracle.mjs create mode 100644 scripts/quality/release-acceptance/inventory.mjs create mode 100644 scripts/quality/release-acceptance/nodeReporter.mjs create mode 100644 scripts/quality/release-acceptance/reduce.mjs create mode 100644 scripts/quality/release-acceptance/staticAdapter.mjs create mode 100644 scripts/quality/release-acceptance/types.mjs create mode 100644 scripts/quality/validate-release-acceptance.mjs create mode 100644 tests/fixtures/release-acceptance/failed-pack-boot.json create mode 100644 tests/fixtures/release-acceptance/infra-pack-boot.json create mode 100644 tests/fixtures/release-acceptance/legacy-close-steps.yml create mode 100644 tests/fixtures/release-acceptance/plan-lint.json create mode 100644 tests/fixtures/release-acceptance/shadow-manifests/lint.json create mode 100644 tests/fixtures/release-acceptance/unverified-required-skipped.json create mode 100644 tests/fixtures/release-acceptance/verified.json create mode 100644 tests/unit/release-acceptance-cli.test.ts create mode 100644 tests/unit/release-acceptance-close-oracle.test.ts create mode 100644 tests/unit/release-acceptance-inventory.test.ts create mode 100644 tests/unit/release-acceptance-node-reporter.test.ts create mode 100644 tests/unit/release-acceptance-pack-boot.test.ts create mode 100644 tests/unit/release-acceptance-reduce.test.ts create mode 100644 tests/unit/release-acceptance-schema.test.ts create mode 100644 tests/unit/release-acceptance-static-adapter.test.ts diff --git a/.github/workflows/release-acceptance.yml b/.github/workflows/release-acceptance.yml new file mode 100644 index 0000000000..75527ad2dd --- /dev/null +++ b/.github/workflows/release-acceptance.yml @@ -0,0 +1,42 @@ +name: Release acceptance + +on: + push: + branches: ["release/v*"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-acceptance-${{ github.ref }} + cancel-in-progress: false + +jobs: + acceptance: + name: Release acceptance + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: "22" + cache: npm + - run: npm ci + - name: Emit shadow acceptance report + run: | + node scripts/quality/validate-release-acceptance.mjs \ + --plan tests/fixtures/release-acceptance/plan-lint.json \ + --manifests tests/fixtures/release-acceptance/shadow-manifests \ + --out release-acceptance-report.json + continue-on-error: true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: release-acceptance-report + path: release-acceptance-report.json + if-no-files-found: ignore + retention-days: 30 diff --git a/changelog.d/fixes/release-acceptance-shadow.md b/changelog.d/fixes/release-acceptance-shadow.md new file mode 100644 index 0000000000..aeae72889c --- /dev/null +++ b/changelog.d/fixes/release-acceptance-shadow.md @@ -0,0 +1 @@ +- Add a shadow release-acceptance report next to release-green.json. It does not close #12732 and is not a Mergify required check. diff --git a/config/quality/release-acceptance.schema.json b/config/quality/release-acceptance.schema.json new file mode 100644 index 0000000000..28c024ca5d --- /dev/null +++ b/config/quality/release-acceptance.schema.json @@ -0,0 +1,183 @@ +{ + "$id": "https://omniroute.local/quality/release-acceptance.schema.json", + "title": "Release acceptance report", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "identity", + "required_gates", + "gates", + "evidence_errors", + "verdict", + "artifact" + ], + "properties": { + "schema_version": { "type": "integer", "const": 1 }, + "identity": { "$ref": "#/$defs/identity" }, + "required_gates": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/gateInstanceKey" } + }, + "gates": { + "type": "array", + "items": { "$ref": "#/$defs/gateResult" } + }, + "evidence_errors": { + "type": "array", + "items": { "$ref": "#/$defs/evidenceError" } + }, + "verdict": { "enum": ["VERIFIED", "FAILED", "UNVERIFIED"] }, + "artifact": { + "anyOf": [ + { "type": "null" }, + { "$ref": "#/$defs/artifact" } + ] + } + }, + "allOf": [ + { + "if": { "properties": { "verdict": { "const": "VERIFIED" } }, "required": ["verdict"] }, + "then": { "properties": { "required_gates": { "minItems": 1 } } } + } + ], + "$defs": { + "sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "gateInstanceKey": { + "type": "object", + "additionalProperties": false, + "required": ["gate_id", "suite_id", "shard_index", "shard_total"], + "properties": { + "gate_id": { "type": "string", "minLength": 1 }, + "suite_id": { "type": ["string", "null"] }, + "shard_index": { "type": ["integer", "null"], "minimum": 0 }, + "shard_total": { "type": ["integer", "null"], "minimum": 1 } + } + }, + "identity": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository", + "run_id", + "run_attempt", + "workflow", + "trigger", + "scope", + "requested_ref", + "base_sha", + "candidate_sha", + "tested_sha" + ], + "properties": { + "repository": { "type": "string", "minLength": 1 }, + "run_id": { "type": "string", "minLength": 1 }, + "run_attempt": { "type": "integer", "minimum": 1 }, + "workflow": { "type": "string", "minLength": 1 }, + "trigger": { "type": "string", "minLength": 1 }, + "scope": { "enum": ["pr", "release", "scheduled"] }, + "requested_ref": { "type": "string", "minLength": 1 }, + "base_sha": { "$ref": "#/$defs/sha" }, + "candidate_sha": { "$ref": "#/$defs/sha" }, + "tested_sha": { "$ref": "#/$defs/sha" } + } + }, + "evidenceRef": { + "type": "object", + "additionalProperties": false, + "required": ["artifact_id", "member", "algorithm", "digest"], + "properties": { + "artifact_id": { "type": "string", "minLength": 1 }, + "member": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*(?:^|[/\\\\])\\.\\.(?:[/\\\\]|$))[^\\s]+$" + }, + "algorithm": { "const": "sha256" }, + "digest": { "$ref": "#/$defs/digest" } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["algorithm", "digest", "identity"], + "properties": { + "algorithm": { "const": "sha256" }, + "digest": { "$ref": "#/$defs/digest" }, + "identity": { "type": "string", "minLength": 1 } + } + }, + "evidenceError": { + "type": "object", + "additionalProperties": false, + "required": ["code", "gate", "detail"], + "properties": { + "code": { "type": "string", "minLength": 1 }, + "gate": { "$ref": "#/$defs/gateInstanceKey" }, + "detail": { "type": "string" } + } + }, + "gateResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "gate_id", + "suite_id", + "shard_index", + "shard_total", + "tested_sha", + "run_id", + "run_attempt", + "command_id", + "gate_type", + "status", + "cause", + "exit_code", + "duration_ms", + "evidence" + ], + "properties": { + "gate_id": { "type": "string", "minLength": 1 }, + "suite_id": { "type": ["string", "null"] }, + "shard_index": { "type": ["integer", "null"], "minimum": 0 }, + "shard_total": { "type": ["integer", "null"], "minimum": 1 }, + "tested_sha": { "$ref": "#/$defs/sha" }, + "run_id": { "type": "string", "minLength": 1 }, + "run_attempt": { "type": "integer", "minimum": 1 }, + "command_id": { "type": "string", "minLength": 1 }, + "gate_type": { "enum": ["static", "test", "artifact"] }, + "status": { "enum": ["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"] }, + "reason": { "type": "string", "minLength": 1 }, + "cause": { + "anyOf": [ + { "type": "null" }, + { "$ref": "#/$defs/gateInstanceKey" } + ] + }, + "exit_code": { "type": ["integer", "null"] }, + "duration_ms": { "type": "integer", "minimum": 0 }, + "evidence": { + "type": "array", + "items": { "$ref": "#/$defs/evidenceRef" } + } + }, + "allOf": [ + { + "if": { + "properties": { "status": { "const": "SKIPPED" } }, + "required": ["status"] + }, + "then": { "required": ["reason"] } + } + ] + } + } +} diff --git a/scripts/quality/release-acceptance/closeOracle.mjs b/scripts/quality/release-acceptance/closeOracle.mjs new file mode 100644 index 0000000000..0417c955aa --- /dev/null +++ b/scripts/quality/release-acceptance/closeOracle.mjs @@ -0,0 +1,30 @@ +const CLOSE_RE = + /gh issue close\b|issues\.update\b|state=closed/g; + +const KEYWORD_RE = new RegExp( + String.raw`\b(?:fix(?:es|ed)?|close[sd]?|resolve[sd]?)\s+#(\d+)\b`, + "i" +); + +export function findTrackerCloses(workflowText) { + const hits = []; + const lines = String(workflowText ?? "").split(/\n/); + for (let i = 0; i < lines.length; i++) { + CLOSE_RE.lastIndex = 0; + if (CLOSE_RE.test(lines[i])) { + hits.push({ line: i + 1, text: lines[i].trim() }); + } + CLOSE_RE.lastIndex = 0; + } + return hits; +} + +export function closingKeywordInBody(body, tracker = 12732) { + const re = new RegExp(KEYWORD_RE.source, KEYWORD_RE.flags.includes("g") ? KEYWORD_RE.flags : `${KEYWORD_RE.flags}g`); + const text = String(body ?? ""); + let m; + while ((m = re.exec(text)) !== null) { + if (Number(m[1]) === Number(tracker)) return true; + } + return false; +} diff --git a/scripts/quality/release-acceptance/inventory.mjs b/scripts/quality/release-acceptance/inventory.mjs new file mode 100644 index 0000000000..6145b32164 --- /dev/null +++ b/scripts/quality/release-acceptance/inventory.mjs @@ -0,0 +1,102 @@ +import fs from "node:fs"; +import path from "node:path"; +import { + COLLECTORS, + globToRegExp, +} from "../../check/check-test-discovery.mjs"; + +const UNIT_CI_GLOBS = new Set([ + "tests/unit/*.test.ts", + "tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,translator,ui,usage}/**/*.test.ts", + "tests/unit/dashboard/**/*.test.ts", + "tests/unit/serial/**/*.test.ts", + "tests/unit/**/*.test.mjs", +]); + +const INTEGRATION_GLOBS = new Set([ + "tests/integration/*.test.ts", + "tests/integration/combo-matrix/*.test.ts", +]); + +function inScope(collector, scopeSuites) { + const suites = new Set(scopeSuites); + if (suites.has("test:unit:ci") && UNIT_CI_GLOBS.has(collector.glob)) return true; + if (suites.has("test:integration") && INTEGRATION_GLOBS.has(collector.glob)) return true; + if (suites.has("test:vitest") && collector.sources?.includes("vitest.mcp.config.ts")) { + return true; + } + return false; +} + +function walkTestFiles(root = process.cwd()) { + const out = []; + function walk(dir) { + if (!fs.existsSync(dir)) return; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (e.name === "node_modules" || e.name === ".git") continue; + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p); + else if (/\.(test|spec)\.(ts|tsx|mjs|js)$/.test(e.name)) { + out.push(path.relative(root, p).split(path.sep).join("/")); + } + } + } + walk(path.join(root, "tests")); + walk(path.join(root, "open-sse")); + walk(path.join(root, "src")); + return out; +} + +export function canonicalSet(scopeSuites, collectors = COLLECTORS, files) { + const scoped = collectors.filter((c) => inScope(c, scopeSuites)); + const regexes = scoped.map((c) => globToRegExp(c.glob)); + const discovered = files ?? walkTestFiles(); + return discovered.filter((f) => regexes.some((re) => re.test(f))); +} + +export function knownUnexecuted(scopeSuites, collectors = COLLECTORS, baseline, files) { + const discovered = files ?? walkTestFiles(); + const orphans = baseline?.orphans ?? []; + const outOfScope = collectors.filter((c) => !inScope(c, scopeSuites)); + const collectorsOut = outOfScope.map((c) => { + const re = globToRegExp(c.glob); + const count = discovered.filter((f) => re.test(f)).length; + return { + glob: c.glob, + count, + reason: "collector runner is not a suite of this scope", + }; + }); + return { + orphans: { count: orphans.length, paths: orphans }, + collectors: collectorsOut, + }; +} + +export function inventoryErrors(scopeSuites, collectors, baseline, discoveredFiles) { + const errors = []; + const full = COLLECTORS; + const givenGlobs = new Set(collectors.map((c) => c.glob)); + for (const c of full) { + if (!givenGlobs.has(c.glob)) { + errors.push({ + code: "collector_omitted", + glob: c.glob, + detail: `collector ${c.glob} omitted without known_unexecuted listing`, + }); + } + } + const ku = knownUnexecuted(scopeSuites, collectors, baseline, discoveredFiles); + const knownGlobs = new Set(ku.collectors.map((c) => c.glob)); + const knownOrphans = new Set(ku.orphans.paths); + const scoped = collectors.filter((c) => inScope(c, scopeSuites)); + const regexes = scoped.map((c) => globToRegExp(c.glob)); + for (const f of discoveredFiles ?? []) { + const inCanonical = regexes.some((re) => re.test(f)); + const inKnown = knownOrphans.has(f) || [...knownGlobs].some((g) => globToRegExp(g).test(f)); + if (!inCanonical && !inKnown) { + errors.push({ code: "unmapped_file", path: f, detail: "discovered file belongs to no set" }); + } + } + return errors; +} diff --git a/scripts/quality/release-acceptance/nodeReporter.mjs b/scripts/quality/release-acceptance/nodeReporter.mjs new file mode 100644 index 0000000000..105f16ba38 --- /dev/null +++ b/scripts/quality/release-acceptance/nodeReporter.mjs @@ -0,0 +1,41 @@ +const SUBTEST = /^# Subtest:\s+(\S+)/; +const RESULT = /^(ok|not ok)\s+\d+\s+-\s+(\S+)/; + +export function fromNodeTestTap(tapText, argvFiles) { + const completed = []; + const failed = []; + const seen = new Set(); + const lines = String(tapText ?? "").split(/\r?\n/); + let pending = null; + for (const line of lines) { + const sub = line.match(SUBTEST); + if (sub) { + pending = sub[1]; + continue; + } + const res = line.match(RESULT); + if (res) { + const file = pending; + const ok = res[1] === "ok"; + if (file) { + seen.add(file); + if (!ok) { + if (!failed.includes(file)) failed.push(file); + const i = completed.indexOf(file); + if (i >= 0) completed.splice(i, 1); + } else if (!failed.includes(file) && !completed.includes(file)) { + completed.push(file); + } + } + } + } + const attempted = [...argvFiles]; + const missing = attempted.filter((f) => !seen.has(f)); + return { + completed, + attempted, + missing, + failed, + pass: completed.length > 0 && missing.length === 0 && failed.length === 0, + }; +} diff --git a/scripts/quality/release-acceptance/reduce.mjs b/scripts/quality/release-acceptance/reduce.mjs new file mode 100644 index 0000000000..6e3ddd6157 --- /dev/null +++ b/scripts/quality/release-acceptance/reduce.mjs @@ -0,0 +1,243 @@ +import { gateKey, sameKey } from "./types.mjs"; + +export function classifyDependent(prereqStatus, dependentKey, prereqKey) { + if (prereqStatus === "FAIL") { + return { status: "FAIL", cause: prereqKey }; + } + if (prereqStatus === "INFRA_ERROR") { + return { status: "INFRA_ERROR", cause: prereqKey }; + } + if (prereqStatus == null) { + return { + status: "INFRA_ERROR", + cause: prereqKey, + evidence_error: { + code: "prerequisite_missing", + gate: dependentKey, + detail: `missing prerequisite ${prereqKey.gate_id}`, + }, + }; + } + if (prereqStatus === "SKIPPED") { + return { status: "SKIPPED", cause: prereqKey }; + } + return { status: "RUN", cause: null }; +} + +function requiredSet(plan) { + return plan.required_gates ?? []; +} + +function optionalSet(plan) { + return plan.optional_gates ?? []; +} + +function isRequired(plan, k) { + return requiredSet(plan).some((r) => sameKey(r, k)); +} + +function copies(gates, k) { + return gates.filter((g) => sameKey(gateKey(g), k)); +} + +function copiesByGateId(gates, gateId) { + return gates.filter((g) => g.gate_id === gateId); +} + +function uniqueKeys(keys) { + const out = []; + for (const k of keys) { + if (!out.some((existing) => sameKey(existing, k))) out.push(k); + } + return out; +} + +function keysForGateId(plan, gates, gateId) { + return uniqueKeys([ + ...copiesByGateId(gates, gateId).map((g) => gateKey(g)), + ...requiredSet(plan).filter((k) => k.gate_id === gateId), + ...optionalSet(plan).filter((k) => k.gate_id === gateId), + ]); +} + +function statusOf(gates, k) { + const list = copies(gates, k); + if (list.length === 0) return null; + if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR"; + if (list.some((g) => g.status === "FAIL")) return "FAIL"; + if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED"; + return list[0].status; +} + +function statusOfGateId(gates, gateId) { + const list = copiesByGateId(gates, gateId); + if (list.length === 0) return null; + if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR"; + if (list.some((g) => g.status === "FAIL")) return "FAIL"; + if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED"; + return list[0].status; +} + +function pushEvidenceError(evidence_errors, err) { + if (!err) return; + const already = evidence_errors.some( + (e) => + e.code === err.code && + e.detail === err.detail && + e.gate?.gate_id === err.gate?.gate_id + ); + if (!already) evidence_errors.push(err); +} + +function patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity) { + const matches = copies(gates, depKey); + const reason = + classified.status === "SKIPPED" ? `classified from ${prereqKey.gate_id}` : undefined; + const exit_code = classified.status === "FAIL" ? 1 : 2; + if (matches.length === 0) { + gates.push({ + gate_id: depKey.gate_id, + suite_id: depKey.suite_id, + shard_index: depKey.shard_index, + shard_total: depKey.shard_total, + tested_sha: identity.tested_sha || "0".repeat(40), + run_id: identity.run_id ?? "0", + run_attempt: identity.run_attempt ?? 1, + command_id: depKey.gate_id, + gate_type: "artifact", + status: classified.status, + cause: classified.cause, + reason, + exit_code, + duration_ms: 0, + evidence: [], + }); + if (classified.evidence_error) pushEvidenceError(evidence_errors, classified.evidence_error); + return true; + } + let changed = false; + for (const existing of matches) { + if ( + existing.status === classified.status && + ((existing.cause == null && classified.cause == null) || + (existing.cause && classified.cause && sameKey(existing.cause, classified.cause))) + ) { + continue; + } + existing.status = classified.status; + existing.cause = classified.cause; + existing.exit_code = exit_code; + if (classified.status === "SKIPPED" && !existing.reason) existing.reason = reason; + changed = true; + } + if (changed && classified.evidence_error) { + pushEvidenceError(evidence_errors, classified.evidence_error); + } + return changed; +} + +function assertAcyclic(deps) { + const visiting = new Set(); + const done = new Set(); + function walk(id) { + if (done.has(id)) return; + if (visiting.has(id)) throw new Error("cyclic prerequisite"); + visiting.add(id); + if (Object.hasOwn(deps, id)) walk(deps[id]); + visiting.delete(id); + done.add(id); + } + for (const id of Object.keys(deps)) walk(id); +} + +export function reduce(plan, records) { + const deps = plan.dependencies ?? {}; + assertAcyclic(deps); + for (const [depId, prereqId] of Object.entries(deps)) { + const requiredDep = requiredSet(plan).some((k) => k.gate_id === depId); + const optionalPrereq = optionalSet(plan).some((k) => k.gate_id === prereqId); + if (requiredDep && optionalPrereq) { + throw new Error("optional prerequisite"); + } + } + + const gates = []; + const evidence_errors = []; + + for (const rec of records) { + const k = gateKey(rec); + const copy = { ...rec, cause: rec.cause ?? null }; + if (copy.status === "SKIPPED" && isRequired(plan, k) && !copy.reason) { + copy.reason = "required skipped"; + } + gates.push(copy); + } + + const identity = plan.identity ?? {}; + const edges = Object.entries(deps); + let changed = true; + let guard = edges.length + 1; + while (changed && guard-- > 0) { + changed = false; + for (const [depId, prereqId] of edges) { + const prereqKey = { gate_id: prereqId, suite_id: null, shard_index: null, shard_total: null }; + let depKeys = keysForGateId(plan, gates, depId); + if (depKeys.length === 0) { + depKeys = [{ gate_id: depId, suite_id: null, shard_index: null, shard_total: null }]; + } + const prereqStatus = statusOfGateId(gates, prereqId); + for (const depKey of depKeys) { + const classified = classifyDependent(prereqStatus, depKey, prereqKey); + if (classified.status === "RUN") continue; + if (patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity)) { + changed = true; + } + } + } + } + + for (const k of requiredSet(plan)) { + const rec = gates.find((g) => sameKey(gateKey(g), k)); + if (!rec) { + evidence_errors.push({ + code: "missing_record", + gate: k, + detail: `required gate ${k.gate_id} has no record`, + }); + } else if (rec.status === "SKIPPED") { + evidence_errors.push({ + code: "required_skipped", + gate: k, + detail: rec.reason ?? "required gate SKIPPED", + }); + } + } + + const required = requiredSet(plan); + if (required.length === 0) { + evidence_errors.push({ + code: "empty_required_set", + gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null }, + detail: "required_gates is empty", + }); + } + + let verdict = "VERIFIED"; + const hasFail = gates.some( + (g) => g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL" + ); + const hasUnverified = + evidence_errors.length > 0 || + gates.some( + (g) => + isRequired(plan, gateKey(g)) && + (g.status === "SKIPPED" || g.status === "INFRA_ERROR") + ); + if (hasFail) verdict = "FAILED"; + else if (hasUnverified) verdict = "UNVERIFIED"; + else if (required.some((k) => !gates.some((g) => sameKey(gateKey(g), k)))) { + verdict = "UNVERIFIED"; + } + + return { verdict, evidence_errors, gates }; +} diff --git a/scripts/quality/release-acceptance/staticAdapter.mjs b/scripts/quality/release-acceptance/staticAdapter.mjs new file mode 100644 index 0000000000..f00a55f44d --- /dev/null +++ b/scripts/quality/release-acceptance/staticAdapter.mjs @@ -0,0 +1,50 @@ +export function adaptCompiler({ commandId, inputDigest, exitCode, diagnostics }) { + const diags = Array.isArray(diagnostics) ? diagnostics : []; + const digest = typeof inputDigest === "string" ? inputDigest : ""; + if (exitCode === 0 && digest.length > 0) { + return { + command_id: commandId, + input_digest: digest, + exit_code: 0, + diagnostics: diags, + status: "PASS", + }; + } + if (exitCode === 0 && digest.length === 0) { + return { + command_id: commandId, + input_digest: digest, + exit_code: 0, + diagnostics: diags, + status: "INFRA_ERROR", + }; + } + return { + command_id: commandId, + input_digest: digest, + exit_code: exitCode, + diagnostics: diags, + status: "FAIL", + }; +} + +export function adaptScript({ commandId, inputDigest, exitCode, stdout }) { + const digest = typeof inputDigest === "string" ? inputDigest : ""; + let parsed = null; + if (typeof stdout === "string" && stdout.trim()) { + try { + parsed = JSON.parse(stdout); + } catch { + parsed = null; + } + } + const diagnostics = parsed ?? { input_digest: digest, exit_code: exitCode, diagnostics: stdout ?? "" }; + const status = exitCode === 0 ? (digest ? "PASS" : "INFRA_ERROR") : "FAIL"; + return { + command_id: commandId, + input_digest: digest, + exit_code: exitCode, + diagnostics, + status, + }; +} diff --git a/scripts/quality/release-acceptance/types.mjs b/scripts/quality/release-acceptance/types.mjs new file mode 100644 index 0000000000..8090da5424 --- /dev/null +++ b/scripts/quality/release-acceptance/types.mjs @@ -0,0 +1,19 @@ +export const STATUSES = Object.freeze(["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"]); +export const VERDICTS = Object.freeze(["VERIFIED", "FAILED", "UNVERIFIED"]); + +export function gateKey(rec) { + return { + gate_id: rec.gate_id, + suite_id: rec.suite_id ?? null, + shard_index: rec.shard_index ?? null, + shard_total: rec.shard_total ?? null, + }; +} + +export function keyId(k) { + return `${k.gate_id}\0${k.suite_id ?? ""}\0${k.shard_index ?? ""}\0${k.shard_total ?? ""}`; +} + +export function sameKey(a, b) { + return keyId(a) === keyId(b); +} diff --git a/scripts/quality/validate-release-acceptance.mjs b/scripts/quality/validate-release-acceptance.mjs new file mode 100644 index 0000000000..9a45ac95e5 --- /dev/null +++ b/scripts/quality/validate-release-acceptance.mjs @@ -0,0 +1,81 @@ +#!/usr/bin/env node +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import Ajv from "ajv"; +import { reduce } from "./release-acceptance/reduce.mjs"; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +function loadJson(p) { + return JSON.parse(readFileSync(p, "utf8")); +} + +export function exitFor(verdict) { + if (verdict === "VERIFIED") return 0; + if (verdict === "FAILED") return 1; + return 2; +} + +export function reduceManifests(plan, manifests) { + const records = []; + for (const m of manifests) { + if (Array.isArray(m.gates)) records.push(...m.gates); + else records.push(m); + } + return reduce(plan, records); +} + +export function validateReport(report, schema) { + const ajv = new Ajv({ allErrors: true, strict: false }); + const validate = ajv.compile(schema); + return { ok: validate(report), errors: validate.errors }; +} + +function parseArgs(argv) { + const out = { plan: null, manifests: null, out: join(ROOT, "release-acceptance-report.json") }; + for (let i = 2; i < argv.length; i++) { + if (argv[i] === "--plan") out.plan = argv[++i]; + else if (argv[i] === "--manifests") out.manifests = argv[++i]; + else if (argv[i] === "--out") out.out = argv[++i]; + } + return out; +} + +export async function main(argv = process.argv) { + const args = parseArgs(argv); + const plan = loadJson(args.plan); + const schema = loadJson(join(ROOT, "config/quality/release-acceptance.schema.json")); + const files = readdirSync(args.manifests) + .filter((f) => f.endsWith(".json")) + .map((f) => loadJson(join(args.manifests, f))); + const reduced = reduceManifests(plan, files); + const report = { + schema_version: 1, + identity: plan.identity, + required_gates: plan.required_gates ?? [], + gates: reduced.gates, + evidence_errors: reduced.evidence_errors, + verdict: reduced.verdict, + artifact: plan.artifact ?? null, + }; + const { ok, errors } = validateReport(report, schema); + if (!ok) { + if (report.verdict !== "FAILED") report.verdict = "UNVERIFIED"; + const gate = + Array.isArray(plan.required_gates) && plan.required_gates.length > 0 + ? plan.required_gates[0] + : { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null }; + report.evidence_errors = [ + ...(report.evidence_errors ?? []), + { code: "schema_invalid", gate, detail: JSON.stringify(errors) }, + ]; + } + mkdirSync(dirname(args.out), { recursive: true }); + writeFileSync(args.out, JSON.stringify(report, null, 2) + "\n"); + return exitFor(report.verdict); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().then((code) => process.exit(code)); +} diff --git a/tests/fixtures/release-acceptance/failed-pack-boot.json b/tests/fixtures/release-acceptance/failed-pack-boot.json new file mode 100644 index 0000000000..0f3fdefe11 --- /dev/null +++ b/tests/fixtures/release-acceptance/failed-pack-boot.json @@ -0,0 +1,89 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + { + "gate_id": "pack-boot", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "gates": [ + { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "check:pack-artifact", + "gate_type": "artifact", + "status": "FAIL", + "cause": null, + "exit_code": 1, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + }, + { + "gate_id": "pack-boot", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "check:pack-boot", + "gate_type": "artifact", + "status": "FAIL", + "cause": { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + "exit_code": 1, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + } + ], + "evidence_errors": [], + "verdict": "FAILED", + "artifact": { + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e", + "identity": "omniroute.tgz" + } +} diff --git a/tests/fixtures/release-acceptance/infra-pack-boot.json b/tests/fixtures/release-acceptance/infra-pack-boot.json new file mode 100644 index 0000000000..d7b721b021 --- /dev/null +++ b/tests/fixtures/release-acceptance/infra-pack-boot.json @@ -0,0 +1,100 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + { + "gate_id": "pack-boot", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "gates": [ + { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "check:pack-artifact", + "gate_type": "artifact", + "status": "INFRA_ERROR", + "cause": null, + "exit_code": 2, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + }, + { + "gate_id": "pack-boot", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "check:pack-boot", + "gate_type": "artifact", + "status": "INFRA_ERROR", + "cause": { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + "exit_code": 2, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + } + ], + "evidence_errors": [ + { + "code": "infra", + "gate": { + "gate_id": "pack-artifact", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + "detail": "evidence cap" + } + ], + "verdict": "UNVERIFIED", + "artifact": { + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e", + "identity": "omniroute.tgz" + } +} diff --git a/tests/fixtures/release-acceptance/legacy-close-steps.yml b/tests/fixtures/release-acceptance/legacy-close-steps.yml new file mode 100644 index 0000000000..ffaa57b679 --- /dev/null +++ b/tests/fixtures/release-acceptance/legacy-close-steps.yml @@ -0,0 +1,39 @@ + - name: Close tracking issue when the branch is green again + if: steps.validate.outputs.exit == '0' + env: + GH_TOKEN: ${{ github.token }} + TARGET: ${{ steps.branch.outputs.target }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + # The open/update step above is the UPWARD half of the loop; without this + # step a stale "not green" issue outlives the fix and every base-green check + # (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited. + TITLE="🔴 Release branch not green: ${TARGET}" + EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "") + if [ -n "$EXISTING" ]; then + gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \ + --comment "✅ \`${TARGET}\` is release-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)." + echo "Closed issue #$EXISTING" + fi + + - name: Close tracking issue when the branch is green again + if: steps.validate.outputs.exit == '0' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + # The open/update step above is the UPWARD half of the loop; without this + # step a stale "not green" issue outlives the fix and every base-green check + # (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited. + TITLE="🔴 main branch not green" + EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "") + if [ -n "$EXISTING" ]; then + gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \ + --comment "✅ \`main\` is main-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)." + echo "Closed issue #$EXISTING" + fi + diff --git a/tests/fixtures/release-acceptance/plan-lint.json b/tests/fixtures/release-acceptance/plan-lint.json new file mode 100644 index 0000000000..6def6c2dc1 --- /dev/null +++ b/tests/fixtures/release-acceptance/plan-lint.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "artifact": null +} diff --git a/tests/fixtures/release-acceptance/shadow-manifests/lint.json b/tests/fixtures/release-acceptance/shadow-manifests/lint.json new file mode 100644 index 0000000000..e4c632df67 --- /dev/null +++ b/tests/fixtures/release-acceptance/shadow-manifests/lint.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "lint", + "gate_type": "static", + "status": "PASS", + "cause": null, + "exit_code": 0, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + } + ], + "evidence_errors": [], + "verdict": "VERIFIED", + "artifact": null +} diff --git a/tests/fixtures/release-acceptance/unverified-required-skipped.json b/tests/fixtures/release-acceptance/unverified-required-skipped.json new file mode 100644 index 0000000000..b99c18123c --- /dev/null +++ b/tests/fixtures/release-acceptance/unverified-required-skipped.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "lint", + "gate_type": "static", + "status": "SKIPPED", + "cause": null, + "exit_code": null, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ], + "reason": "plan-optional-looking" + } + ], + "evidence_errors": [ + { + "code": "required_skipped", + "gate": { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null + }, + "detail": "required gate SKIPPED" + } + ], + "verdict": "UNVERIFIED", + "artifact": null +} diff --git a/tests/fixtures/release-acceptance/verified.json b/tests/fixtures/release-acceptance/verified.json new file mode 100644 index 0000000000..e4c632df67 --- /dev/null +++ b/tests/fixtures/release-acceptance/verified.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "identity": { + "repository": "diegosouzapw/OmniRoute", + "run_id": "1", + "run_attempt": 1, + "workflow": "release-acceptance.yml", + "trigger": "push", + "scope": "release", + "requested_ref": "refs/heads/release/v3.8.51", + "base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e" + }, + "required_gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null + } + ], + "gates": [ + { + "gate_id": "lint", + "suite_id": null, + "shard_index": null, + "shard_total": null, + "tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e", + "run_id": "1", + "run_attempt": 1, + "command_id": "lint", + "gate_type": "static", + "status": "PASS", + "cause": null, + "exit_code": 0, + "duration_ms": 10, + "evidence": [ + { + "artifact_id": "logs", + "member": "lint.log", + "algorithm": "sha256", + "digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e" + } + ] + } + ], + "evidence_errors": [], + "verdict": "VERIFIED", + "artifact": null +} diff --git a/tests/unit/release-acceptance-cli.test.ts b/tests/unit/release-acceptance-cli.test.ts new file mode 100644 index 0000000000..952399065d --- /dev/null +++ b/tests/unit/release-acceptance-cli.test.ts @@ -0,0 +1,174 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + exitFor, + reduceManifests, + main, +} from "../../scripts/quality/validate-release-acceptance.mjs"; + +const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"; +function key(id) { + return { gate_id: id, suite_id: null, shard_index: null, shard_total: null }; +} +function gate(id, status) { + return { + gate_id: id, + suite_id: null, + shard_index: null, + shard_total: null, + tested_sha: SHA, + run_id: "1", + run_attempt: 1, + command_id: id, + gate_type: "static", + status, + cause: null, + exit_code: status === "PASS" ? 0 : 1, + duration_ms: 1, + evidence: [ + { + artifact_id: "logs", + member: "lint.log", + algorithm: "sha256", + digest: "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e", + }, + ], + }; +} + +test("exit mapping", () => { + assert.equal(exitFor("VERIFIED"), 0); + assert.equal(exitFor("FAILED"), 1); + assert.equal(exitFor("UNVERIFIED"), 2); +}); + +test("three PASS manifests yield VERIFIED", () => { + const plan = { + required_gates: [key("a"), key("b"), key("c")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + }; + const out = reduceManifests(plan, [ + { gates: [gate("a", "PASS")] }, + { gates: [gate("b", "PASS")] }, + { gates: [gate("c", "PASS")] }, + ]); + assert.equal(out.verdict, "VERIFIED"); + assert.equal(exitFor(out.verdict), 0); +}); + +test("one FAIL yields FAILED", () => { + const plan = { + required_gates: [key("a")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + }; + const out = reduceManifests(plan, [{ gates: [gate("a", "FAIL")] }]); + assert.equal(out.verdict, "FAILED"); + assert.equal(exitFor(out.verdict), 1); +}); + +test("required missing yields UNVERIFIED", () => { + const plan = { + required_gates: [key("a"), key("b")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + }; + const out = reduceManifests(plan, [{ gates: [gate("a", "PASS")] }]); + assert.equal(out.verdict, "UNVERIFIED"); + assert.equal(exitFor(out.verdict), 2); +}); + +test("workflow source-guard", () => { + const text = readFileSync(".github/workflows/release-acceptance.yml", "utf8"); + assert.match(text, /name: Release acceptance/); + assert.match(text, /cancel-in-progress: false/); + assert.equal(text.includes("gh issue close"), false); + assert.match(text, /if: github.event_name != 'pull_request'/); +}); + +test("schema_invalid does not throw when required_gates is missing", async () => { + const dir = mkdtempSync(join(tmpdir(), "acc-")); + writeFileSync( + join(dir, "plan.json"), + JSON.stringify({ + identity: { + repository: "diegosouzapw/OmniRoute", + run_id: "1", + run_attempt: 1, + workflow: "release-acceptance.yml", + trigger: "push", + scope: "release", + requested_ref: "refs/heads/release/v3.8.51", + base_sha: SHA, + candidate_sha: SHA, + tested_sha: SHA, + }, + artifact: null, + }) + ); + const man = join(dir, "m"); + mkdirSync(man); + writeFileSync(join(man, "a.json"), JSON.stringify({ gates: [gate("a", "PASS")] })); + const out = join(dir, "report.json"); + const code = await main([ + "node", + "cli", + "--plan", + join(dir, "plan.json"), + "--manifests", + man, + "--out", + out, + ]); + assert.equal(code, 2); + const report = JSON.parse(readFileSync(out, "utf8")); + assert.equal(report.verdict, "UNVERIFIED"); + assert.ok(Array.isArray(report.required_gates)); + assert.ok(report.evidence_errors.some((e) => e.code === "empty_required_set")); + assert.equal( + report.evidence_errors.some((e) => e.code === "schema_invalid"), + false + ); +}); + +test("schema_invalid keeps FAILED when reduce already failed", async () => { + const dir = mkdtempSync(join(tmpdir(), "acc-fail-")); + const plan = { + required_gates: [key("a")], + identity: { + repository: "diegosouzapw/OmniRoute", + run_id: "1", + run_attempt: 1, + workflow: "release-acceptance.yml", + trigger: "push", + scope: "release", + requested_ref: "refs/heads/release/v3.8.51", + base_sha: SHA, + candidate_sha: SHA, + tested_sha: SHA, + }, + artifact: null, + }; + writeFileSync(join(dir, "plan.json"), JSON.stringify(plan)); + const man = join(dir, "m"); + mkdirSync(man); + const g = gate("a", "FAIL"); + g.unexpected = true; + writeFileSync(join(man, "a.json"), JSON.stringify({ gates: [g] })); + const out = join(dir, "report.json"); + const code = await main([ + "node", + "cli", + "--plan", + join(dir, "plan.json"), + "--manifests", + man, + "--out", + out, + ]); + assert.equal(code, 1); + const report = JSON.parse(readFileSync(out, "utf8")); + assert.equal(report.verdict, "FAILED"); + assert.ok(report.evidence_errors.some((e) => e.code === "schema_invalid")); +}); diff --git a/tests/unit/release-acceptance-close-oracle.test.ts b/tests/unit/release-acceptance-close-oracle.test.ts new file mode 100644 index 0000000000..96ae3f8659 --- /dev/null +++ b/tests/unit/release-acceptance-close-oracle.test.ts @@ -0,0 +1,23 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { + findTrackerCloses, + closingKeywordInBody, +} from "../../scripts/quality/release-acceptance/closeOracle.mjs"; + +test("nightly still auto-closes the tracker via two steps (deliberate, #12085)", () => { + const text = readFileSync(".github/workflows/nightly-release-green.yml", "utf8"); + assert.equal(findTrackerCloses(text).length, 2); + const legacy = readFileSync( + new URL("../fixtures/release-acceptance/legacy-close-steps.yml", import.meta.url), + "utf8" + ); + assert.equal(findTrackerCloses(legacy).length, 2); +}); + +test("Fixes #12732 is a closing keyword; Related to #12732 is not", () => { + assert.equal(closingKeywordInBody("Fixes #12732.\n"), true); + assert.equal(closingKeywordInBody("Related to #12732.\n"), false); + assert.equal(closingKeywordInBody("Fixes #1. Closes #12732\n"), true); +}); diff --git a/tests/unit/release-acceptance-inventory.test.ts b/tests/unit/release-acceptance-inventory.test.ts new file mode 100644 index 0000000000..dc5e2e496c --- /dev/null +++ b/tests/unit/release-acceptance-inventory.test.ts @@ -0,0 +1,40 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { COLLECTORS } from "../../scripts/check/check-test-discovery.mjs"; +import { + knownUnexecuted, + inventoryErrors, +} from "../../scripts/quality/release-acceptance/inventory.mjs"; + +const RELEASE_SUITES = ["test:unit:ci", "test:vitest", "test:integration"]; +const baseline = JSON.parse( + readFileSync(new URL("../../config/quality/test-discovery-baseline.json", import.meta.url), "utf8") +); + +test("tsx files under tests/unit are known_unexecuted for release scope, not inventory errors", () => { + const ku = knownUnexecuted(RELEASE_SUITES, COLLECTORS, baseline); + const tsx = ku.collectors.find((c) => c.glob === "tests/unit/**/*.test.tsx"); + assert.ok(tsx, "tsx collector must be listed as known_unexecuted"); + assert.equal(typeof tsx.count, "number"); + assert.ok(tsx.count > 0); +}); + +test("omitting a collector without listing it is an inventory error", () => { + const collectors = COLLECTORS.filter((c) => c.glob !== "tests/unit/**/*.test.tsx"); + const discoveredFiles = ["tests/unit/AutoComboCatalog.test.tsx"]; + const errors = inventoryErrors(RELEASE_SUITES, collectors, baseline, discoveredFiles); + assert.ok(errors.some((e) => e.code === "collector_omitted")); +}); + +test("combo-matrix glob is in release integration scope, not known_unexecuted", () => { + const ku = knownUnexecuted(RELEASE_SUITES, COLLECTORS, baseline); + assert.equal( + ku.collectors.some((c) => c.glob === "tests/integration/combo-matrix/*.test.ts"), + false + ); + const combo = COLLECTORS.find( + (c) => c.glob === "tests/integration/combo-matrix/*.test.ts" + ); + assert.ok(combo); +}); diff --git a/tests/unit/release-acceptance-node-reporter.test.ts b/tests/unit/release-acceptance-node-reporter.test.ts new file mode 100644 index 0000000000..4e8c47cdfc --- /dev/null +++ b/tests/unit/release-acceptance-node-reporter.test.ts @@ -0,0 +1,86 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { fromNodeTestTap } from "../../scripts/quality/release-acceptance/nodeReporter.mjs"; + +const TAP = `TAP version 13 +# Subtest: tests/unit/a.test.ts +ok 1 - tests/unit/a.test.ts +# Subtest: tests/unit/b.test.ts +ok 2 - tests/unit/b.test.ts +# Subtest: tests/unit/c.test.ts +not ok 3 - tests/unit/c.test.ts +`; + +test("argv file without TAP completion is missing", () => { + const out = fromNodeTestTap(TAP, [ + "tests/unit/a.test.ts", + "tests/unit/b.test.ts", + "tests/unit/c.test.ts", + "tests/unit/d.test.ts", + ]); + assert.equal(out.completed.length, 2); + assert.deepEqual(out.failed, ["tests/unit/c.test.ts"]); + assert.equal(out.missing.length, 1); + assert.equal(out.missing[0], "tests/unit/d.test.ts"); + assert.equal(out.pass, false); +}); + +test("zero completed files is not PASS", () => { + const out = fromNodeTestTap("TAP version 13\n", ["tests/unit/a.test.ts"]); + assert.equal(out.completed.length, 0); + assert.equal(out.pass, false); +}); + +test("Subtest path wins when the result line has a short name", () => { + const tap = `TAP version 13 +# Subtest: tests/unit/a.test.ts +ok 1 - some name +`; + const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]); + assert.equal(out.completed[0], "tests/unit/a.test.ts"); + assert.equal(out.missing.length, 0); +}); + +test("not ok is not pass", () => { + const tap = `TAP version 13 +# Subtest: tests/unit/a.test.ts +not ok 1 - tests/unit/a.test.ts +`; + const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]); + assert.equal(out.pass, false); + assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]); +}); + +test("ok line without Subtest does not complete an argv file", () => { + const tap = `# a malicious test printed: +ok 99 - tests/unit/missing.test.ts +`; + const out = fromNodeTestTap(tap, ["tests/unit/missing.test.ts"]); + assert.equal(out.pass, false); + assert.deepEqual(out.missing, ["tests/unit/missing.test.ts"]); +}); + +test("later not ok retracts an earlier ok for the same Subtest", () => { + const tap = `TAP version 13 +# Subtest: tests/unit/a.test.ts +ok 1 - tests/unit/a.test.ts +# Subtest: tests/unit/a.test.ts +not ok 2 - tests/unit/a.test.ts +`; + const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]); + assert.equal(out.pass, false); + assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]); + assert.equal(out.completed.includes("tests/unit/a.test.ts"), false); +}); + +test("later not ok on the same pending Subtest retracts ok", () => { + const tap = `TAP version 13 +# Subtest: tests/unit/a.test.ts +ok 1 - tests/unit/a.test.ts +not ok 2 - tests/unit/a.test.ts +`; + const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]); + assert.equal(out.pass, false); + assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]); + assert.equal(out.completed.includes("tests/unit/a.test.ts"), false); +}); diff --git a/tests/unit/release-acceptance-pack-boot.test.ts b/tests/unit/release-acceptance-pack-boot.test.ts new file mode 100644 index 0000000000..a594941cfb --- /dev/null +++ b/tests/unit/release-acceptance-pack-boot.test.ts @@ -0,0 +1,68 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { reduce } from "../../scripts/quality/release-acceptance/reduce.mjs"; + +const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"; +const planPack = { + required_gates: [ + { gate_id: "pack-artifact", suite_id: null, shard_index: null, shard_total: null }, + { gate_id: "pack-boot", suite_id: null, shard_index: null, shard_total: null }, + ], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { "pack-boot": "pack-artifact" }, +}; + +function record(partial) { + return { + gate_id: "pack-artifact", + suite_id: null, + shard_index: null, + shard_total: null, + tested_sha: SHA, + run_id: "1", + run_attempt: 1, + command_id: "check:pack-artifact", + gate_type: "artifact", + status: "PASS", + cause: null, + exit_code: 0, + duration_ms: 10, + evidence: [], + ...partial, + }; +} + +test("legacy computeVerdict still hard-fails pack-boot when pack-artifact times out", async () => { + const { computeVerdict } = await import("../../scripts/quality/validate-release-green.mjs"); + const v = computeVerdict([ + { id: "pack-artifact", kind: "hard", ok: false, detail: "timeout" }, + { + id: "pack-boot", + kind: "hard", + ok: false, + detail: "skipped because package-artifact did not produce a valid dist/ build", + }, + ]); + assert.equal(v.releaseGreen, false); +}); + +test("new reducer maps the same timeout to UNVERIFIED", () => { + const out = reduce(planPack, [ + record({ gate_id: "pack-artifact", status: "INFRA_ERROR" }), + ]); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("synthesized pack-boot without identity.tested_sha keeps a 40-hex sha and FAILED", () => { + const plan = { + required_gates: planPack.required_gates, + identity: { run_id: "1", run_attempt: 1 }, + dependencies: { "pack-boot": "pack-artifact" }, + }; + const out = reduce(plan, [record({ gate_id: "pack-artifact", status: "FAIL" })]); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.ok(boot); + assert.notEqual(boot.tested_sha, null); + assert.match(String(boot.tested_sha), /^[0-9a-f]{40}$/); + assert.equal(out.verdict, "FAILED"); +}); diff --git a/tests/unit/release-acceptance-reduce.test.ts b/tests/unit/release-acceptance-reduce.test.ts new file mode 100644 index 0000000000..02152362fd --- /dev/null +++ b/tests/unit/release-acceptance-reduce.test.ts @@ -0,0 +1,287 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { reduce } from "../../scripts/quality/release-acceptance/reduce.mjs"; + +const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"; + +function key(gate_id) { + return { gate_id, suite_id: null, shard_index: null, shard_total: null }; +} + +function record(partial) { + return { + gate_id: "lint", + suite_id: null, + shard_index: null, + shard_total: null, + tested_sha: SHA, + run_id: "1", + run_attempt: 1, + command_id: partial.gate_id ?? "lint", + gate_type: "static", + status: "PASS", + cause: null, + exit_code: 0, + duration_ms: 10, + evidence: [], + ...partial, + }; +} + +function planWithRequired(gateId, extra = {}) { + return { + required_gates: [key(gateId)], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + ...extra, + }; +} + +const planPack = { + required_gates: [key("pack-artifact"), key("pack-boot")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { "pack-boot": "pack-artifact" }, +}; + +test("required SKIPPED never yields VERIFIED", () => { + const out = reduce(planWithRequired("lint"), [ + record({ gate_id: "lint", status: "SKIPPED", reason: "optional-looking" }), + ]); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("pack-artifact FAIL classifies pack-boot as FAIL with cause", () => { + const out = reduce(planPack, [ + record({ gate_id: "pack-artifact", status: "FAIL", gate_type: "artifact" }), + ]); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.equal(boot.status, "FAIL"); + assert.equal(boot.cause.gate_id, "pack-artifact"); + assert.equal(out.verdict, "FAILED"); +}); + +test("pack-artifact INFRA_ERROR classifies pack-boot as INFRA_ERROR", () => { + const out = reduce(planPack, [ + record({ + gate_id: "pack-artifact", + status: "INFRA_ERROR", + gate_type: "artifact", + }), + ]); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.equal(boot.status, "INFRA_ERROR"); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("plan that marks a required gate's prerequisite optional is rejected", () => { + const illegalPlan = { + required_gates: [key("pack-boot")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { "pack-boot": "pack-artifact" }, + optional_gates: [key("pack-artifact")], + }; + assert.throws(() => reduce(illegalPlan, []), /optional prerequisite/); +}); + +test("required SKIPPED prerequisite classifies dependent as SKIPPED, does not throw", () => { + const out = reduce(planPack, [ + record({ + gate_id: "pack-artifact", + status: "SKIPPED", + reason: "runner skipped", + gate_type: "artifact", + }), + ]); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.equal(boot.status, "SKIPPED"); + assert.equal(boot.cause.gate_id, "pack-artifact"); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("INFRA_ERROR artifact reclassifies an already-emitted FAIL boot to INFRA_ERROR", () => { + const out = reduce(planPack, [ + record({ + gate_id: "pack-artifact", + status: "INFRA_ERROR", + gate_type: "artifact", + }), + record({ + gate_id: "pack-boot", + status: "FAIL", + gate_type: "artifact", + }), + ]); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.equal(boot.status, "INFRA_ERROR"); + assert.equal(boot.cause.gate_id, "pack-artifact"); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("empty required_gates is UNVERIFIED", () => { + const out = reduce( + { required_gates: [], identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 } }, + [record({ gate_id: "lint", status: "PASS" })] + ); + assert.equal(out.verdict, "UNVERIFIED"); + assert.ok(out.evidence_errors.some((e) => e.code === "empty_required_set")); +}); + +test("INFRA_ERROR artifact reclassifies every FAIL boot copy", () => { + const out = reduce(planPack, [ + record({ gate_id: "pack-artifact", status: "INFRA_ERROR", gate_type: "artifact" }), + record({ gate_id: "pack-boot", status: "FAIL", gate_type: "artifact" }), + record({ gate_id: "pack-boot", status: "FAIL", gate_type: "artifact" }), + ]); + const boots = out.gates.filter((g) => g.gate_id === "pack-boot"); + assert.ok(boots.length >= 1); + assert.ok(boots.every((g) => g.status === "INFRA_ERROR")); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("transitive INFRA on a three-gate chain is UNVERIFIED, not leaked FAIL", () => { + const plan = { + required_gates: [key("a"), key("b"), key("c")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { b: "a", c: "b" }, + }; + const out = reduce(plan, [ + record({ gate_id: "a", status: "INFRA_ERROR", gate_type: "artifact" }), + record({ gate_id: "b", status: "FAIL", gate_type: "artifact" }), + record({ gate_id: "c", status: "PASS", gate_type: "artifact" }), + ]); + assert.equal(out.gates.find((g) => g.gate_id === "a").status, "INFRA_ERROR"); + assert.equal(out.gates.find((g) => g.gate_id === "b").status, "INFRA_ERROR"); + assert.equal(out.gates.find((g) => g.gate_id === "c").status, "INFRA_ERROR"); + assert.equal(out.verdict, "UNVERIFIED"); +}); + +test("transitive FAIL on a three-gate chain classifies every dependent", () => { + const plan = { + required_gates: [key("a"), key("b"), key("c")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { b: "a", c: "b" }, + }; + const out = reduce(plan, [ + record({ gate_id: "a", status: "FAIL", gate_type: "artifact" }), + record({ gate_id: "b", status: "PASS", gate_type: "artifact" }), + record({ gate_id: "c", status: "PASS", gate_type: "artifact" }), + ]); + assert.equal(out.gates.find((g) => g.gate_id === "b").status, "FAIL"); + assert.equal(out.gates.find((g) => g.gate_id === "c").status, "FAIL"); + assert.equal(out.verdict, "FAILED"); +}); + +test("INFRA copy of a required gate dominates a FAIL copy of the same key", () => { + const out = reduce(planPack, [ + record({ gate_id: "pack-artifact", status: "INFRA_ERROR", gate_type: "artifact" }), + record({ gate_id: "pack-artifact", status: "FAIL", gate_type: "artifact" }), + record({ gate_id: "pack-boot", status: "PASS", gate_type: "artifact" }), + ]); + assert.equal(out.verdict, "UNVERIFIED"); + const boot = out.gates.find((g) => g.gate_id === "pack-boot"); + assert.equal(boot.status, "INFRA_ERROR"); +}); + +test("cyclic dependencies are rejected", () => { + const cyclic = { + required_gates: [key("a"), key("b")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { a: "b", b: "a" }, + }; + assert.throws( + () => reduce(cyclic, [record({ gate_id: "a", status: "INFRA_ERROR" }), record({ gate_id: "b", status: "FAIL" })]), + /cyclic prerequisite/ + ); +}); + +test("missing prerequisite records one evidence error, not one per loop", () => { + const out = reduce( + { + required_gates: [key("boot")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { boot: "art" }, + }, + [] + ); + assert.equal(out.verdict, "UNVERIFIED"); + assert.equal( + out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length, + 1 + ); +}); + +test("missing prerequisite records one evidence error for all shards of a gate_id", () => { + const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 }; + const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 }; + const out = reduce( + { + required_gates: [shard0, shard1], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { u: "art" }, + }, + [] + ); + assert.equal(out.verdict, "UNVERIFIED"); + assert.equal( + out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length, + 1 + ); +}); + +test("two dependents of the same missing prerequisite keep one error per edge", () => { + const out = reduce( + { + required_gates: [key("boot"), key("pack")], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { boot: "art", pack: "art" }, + }, + [] + ); + assert.equal(out.verdict, "UNVERIFIED"); + const missing = out.evidence_errors.filter((e) => e.code === "prerequisite_missing"); + assert.equal(missing.length, 2); + const gates = missing.map((e) => e.gate?.gate_id).sort(); + assert.deepEqual(gates, ["boot", "pack"]); + assert.equal(out.gates.find((g) => g.gate_id === "boot")?.status, "INFRA_ERROR"); + assert.equal(out.gates.find((g) => g.gate_id === "pack")?.status, "INFRA_ERROR"); +}); + +test("sharded required dependents inherit a FAIL prerequisite of the same gate_id", () => { + const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 }; + const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 }; + const out = reduce( + { + required_gates: [shard0, shard1], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { u: "art" }, + }, + [ + record({ gate_id: "art", status: "FAIL", gate_type: "artifact" }), + record({ ...shard0, status: "PASS", gate_type: "artifact" }), + record({ ...shard1, status: "PASS", gate_type: "artifact" }), + ] + ); + const shards = out.gates.filter((g) => g.gate_id === "u" && g.suite_id === "s"); + assert.equal(shards.length, 2); + assert.ok(shards.every((g) => g.status === "FAIL")); + assert.equal(out.verdict, "FAILED"); +}); + +test("sharded required dependents inherit an INFRA prerequisite of the same gate_id", () => { + const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 }; + const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 }; + const out = reduce( + { + required_gates: [shard0, shard1], + identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 }, + dependencies: { u: "art" }, + }, + [ + record({ gate_id: "art", status: "INFRA_ERROR", gate_type: "artifact" }), + record({ ...shard0, status: "PASS", gate_type: "artifact" }), + record({ ...shard1, status: "PASS", gate_type: "artifact" }), + ] + ); + const shards = out.gates.filter((g) => g.gate_id === "u" && g.suite_id === "s"); + assert.ok(shards.every((g) => g.status === "INFRA_ERROR")); + assert.equal(out.verdict, "UNVERIFIED"); +}); diff --git a/tests/unit/release-acceptance-schema.test.ts b/tests/unit/release-acceptance-schema.test.ts new file mode 100644 index 0000000000..461667be6b --- /dev/null +++ b/tests/unit/release-acceptance-schema.test.ts @@ -0,0 +1,112 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import Ajv from "ajv"; + +const schema = JSON.parse( + readFileSync( + new URL("../../config/quality/release-acceptance.schema.json", import.meta.url), + "utf8" + ) +); + +function compile() { + const ajv = new Ajv({ allErrors: true, strict: false }); + return ajv.compile(schema); +} + +test("version 1 requires cause when status is classified by a prerequisite", () => { + const validate = compile(); + const missingCause = JSON.parse( + readFileSync( + new URL("../fixtures/release-acceptance/failed-pack-boot.json", import.meta.url), + "utf8" + ) + ); + delete missingCause.gates[1].cause; + assert.equal(validate(missingCause), false); +}); + +test("unknown top-level gate field is invalid in version 1", () => { + const validate = compile(); + const extra = JSON.parse( + readFileSync( + new URL("../fixtures/release-acceptance/verified.json", import.meta.url), + "utf8" + ) + ); + extra.gates[0].unexpected = true; + assert.equal(validate(extra), false); +}); + +test("evidence member rejects parent traversal", () => { + const validate = compile(); + const report = JSON.parse( + readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8") + ); + report.gates[0].evidence[0].member = "foo/../../etc/passwd"; + assert.equal(validate(report), false); + report.gates[0].evidence[0].member = ".."; + assert.equal(validate(report), false); + report.gates[0].evidence[0].member = "foo/.."; + assert.equal(validate(report), false); + report.gates[0].evidence[0].member = String.raw`foo\..\x`; + assert.equal(validate(report), false); +}); + +test("empty required_gates cannot be VERIFIED", () => { + const validate = compile(); + const report = JSON.parse( + readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8") + ); + report.required_gates = []; + report.gates = []; + report.evidence_errors = []; + report.verdict = "VERIFIED"; + assert.equal(validate(report), false); +}); + +test("empty required_gates is valid when UNVERIFIED", () => { + const validate = compile(); + const report = JSON.parse( + readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8") + ); + report.required_gates = []; + report.gates = []; + report.evidence_errors = [ + { + code: "empty_required_set", + gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null }, + detail: "required_gates is empty", + }, + ]; + report.verdict = "UNVERIFIED"; + assert.equal(validate(report), true, JSON.stringify(validate.errors)); +}); + +test("unknown extensions field is invalid in version 1", () => { + const validate = compile(); + const extra = JSON.parse( + readFileSync( + new URL("../fixtures/release-acceptance/verified.json", import.meta.url), + "utf8" + ) + ); + extra.gates[0].extensions = { unexpected: true }; + assert.equal(validate(extra), false); +}); + +test("known-answer fixtures validate", () => { + const validate = compile(); + for (const name of [ + "verified.json", + "failed-pack-boot.json", + "unverified-required-skipped.json", + "infra-pack-boot.json", + ]) { + const report = JSON.parse( + readFileSync(new URL(`../fixtures/release-acceptance/${name}`, import.meta.url), "utf8") + ); + assert.equal(validate(report), true, `${name}: ${JSON.stringify(validate.errors)}`); + } +}); diff --git a/tests/unit/release-acceptance-static-adapter.test.ts b/tests/unit/release-acceptance-static-adapter.test.ts new file mode 100644 index 0000000000..64d70292a6 --- /dev/null +++ b/tests/unit/release-acceptance-static-adapter.test.ts @@ -0,0 +1,33 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { adaptCompiler } from "../../scripts/quality/release-acceptance/staticAdapter.mjs"; + +test("empty diagnostics with nonempty digest and exit 0 is PASS", () => { + const out = adaptCompiler({ + commandId: "tsc", + inputDigest: "a".repeat(64), + exitCode: 0, + diagnostics: [], + }); + assert.equal(out.status, "PASS"); +}); + +test("empty digest plus empty diagnostics is INFRA_ERROR", () => { + const out = adaptCompiler({ + commandId: "tsc", + inputDigest: "", + exitCode: 0, + diagnostics: [], + }); + assert.equal(out.status, "INFRA_ERROR"); +}); + +test("exit 1 with diagnostics is FAIL", () => { + const out = adaptCompiler({ + commandId: "tsc", + inputDigest: "a".repeat(64), + exitCode: 1, + diagnostics: ["error TS2304"], + }); + assert.equal(out.status, "FAIL"); +});