ci(acceptance): emit a shadow release-acceptance report next to release-green (#13701)

Adds a shadow release-acceptance report alongside release-green: an inventory/reduce/oracle pipeline under `scripts/quality/release-acceptance/` with a JSON schema, fixtures and a workflow that uploads the report as an artifact.

Contained by design, which is why it merges as-is: it runs only on push to `release/v*` and on manual dispatch (never on pull requests), the step is `continue-on-error`, `permissions: contents: read`, `persist-credentials: false`, and it consumes no secrets. Nothing in the product changes; the report is advisory until we decide to promote it.

Validated as a combined board first (this PR merged with the 11 siblings of the same batch on the release tip): eslint on every changed file with the suppressions file, typecheck:core, check:open-sse-typecheck, complexity, cognitive-complexity, changelog-integrity, i18n new-key coverage, docs-sync, migration-numbering, provider-consistency and a duplicate-identifier audit all green, plus 275 passing / 0 failing focused node:test cases across the 28 test files the batch touches. Then re-validated alone on the fresh tip before this merge: conflicts re-resolved, file sizes rebaselined for this PR's own growth, eslint and this PR's focused tests re-run.

Thanks @HouMinXi!
This commit is contained in:
Bob.Hou
2026-09-16 02:17:52 -04:00
committed by GitHub
parent f36fcd8c31
commit e7214c72fc
25 changed files with 2032 additions and 0 deletions

View File

@@ -0,0 +1,42 @@
name: Release acceptance
on:
push:
branches: ["release/v*"]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: release-acceptance-${{ github.ref }}
cancel-in-progress: false
jobs:
acceptance:
name: Release acceptance
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
with:
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: "22"
cache: npm
- run: npm ci
- name: Emit shadow acceptance report
run: |
node scripts/quality/validate-release-acceptance.mjs \
--plan tests/fixtures/release-acceptance/plan-lint.json \
--manifests tests/fixtures/release-acceptance/shadow-manifests \
--out release-acceptance-report.json
continue-on-error: true
- uses: actions/upload-artifact@v4
if: always()
with:
name: release-acceptance-report
path: release-acceptance-report.json
if-no-files-found: ignore
retention-days: 30

View File

@@ -0,0 +1 @@
- Add a shadow release-acceptance report next to release-green.json. It does not close #12732 and is not a Mergify required check.

View File

@@ -0,0 +1,183 @@
{
"$id": "https://omniroute.local/quality/release-acceptance.schema.json",
"title": "Release acceptance report",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"identity",
"required_gates",
"gates",
"evidence_errors",
"verdict",
"artifact"
],
"properties": {
"schema_version": { "type": "integer", "const": 1 },
"identity": { "$ref": "#/$defs/identity" },
"required_gates": {
"type": "array",
"uniqueItems": true,
"items": { "$ref": "#/$defs/gateInstanceKey" }
},
"gates": {
"type": "array",
"items": { "$ref": "#/$defs/gateResult" }
},
"evidence_errors": {
"type": "array",
"items": { "$ref": "#/$defs/evidenceError" }
},
"verdict": { "enum": ["VERIFIED", "FAILED", "UNVERIFIED"] },
"artifact": {
"anyOf": [
{ "type": "null" },
{ "$ref": "#/$defs/artifact" }
]
}
},
"allOf": [
{
"if": { "properties": { "verdict": { "const": "VERIFIED" } }, "required": ["verdict"] },
"then": { "properties": { "required_gates": { "minItems": 1 } } }
}
],
"$defs": {
"sha": {
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
"digest": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"gateInstanceKey": {
"type": "object",
"additionalProperties": false,
"required": ["gate_id", "suite_id", "shard_index", "shard_total"],
"properties": {
"gate_id": { "type": "string", "minLength": 1 },
"suite_id": { "type": ["string", "null"] },
"shard_index": { "type": ["integer", "null"], "minimum": 0 },
"shard_total": { "type": ["integer", "null"], "minimum": 1 }
}
},
"identity": {
"type": "object",
"additionalProperties": false,
"required": [
"repository",
"run_id",
"run_attempt",
"workflow",
"trigger",
"scope",
"requested_ref",
"base_sha",
"candidate_sha",
"tested_sha"
],
"properties": {
"repository": { "type": "string", "minLength": 1 },
"run_id": { "type": "string", "minLength": 1 },
"run_attempt": { "type": "integer", "minimum": 1 },
"workflow": { "type": "string", "minLength": 1 },
"trigger": { "type": "string", "minLength": 1 },
"scope": { "enum": ["pr", "release", "scheduled"] },
"requested_ref": { "type": "string", "minLength": 1 },
"base_sha": { "$ref": "#/$defs/sha" },
"candidate_sha": { "$ref": "#/$defs/sha" },
"tested_sha": { "$ref": "#/$defs/sha" }
}
},
"evidenceRef": {
"type": "object",
"additionalProperties": false,
"required": ["artifact_id", "member", "algorithm", "digest"],
"properties": {
"artifact_id": { "type": "string", "minLength": 1 },
"member": {
"type": "string",
"minLength": 1,
"pattern": "^(?!/)(?!.*(?:^|[/\\\\])\\.\\.(?:[/\\\\]|$))[^\\s]+$"
},
"algorithm": { "const": "sha256" },
"digest": { "$ref": "#/$defs/digest" }
}
},
"artifact": {
"type": "object",
"additionalProperties": false,
"required": ["algorithm", "digest", "identity"],
"properties": {
"algorithm": { "const": "sha256" },
"digest": { "$ref": "#/$defs/digest" },
"identity": { "type": "string", "minLength": 1 }
}
},
"evidenceError": {
"type": "object",
"additionalProperties": false,
"required": ["code", "gate", "detail"],
"properties": {
"code": { "type": "string", "minLength": 1 },
"gate": { "$ref": "#/$defs/gateInstanceKey" },
"detail": { "type": "string" }
}
},
"gateResult": {
"type": "object",
"additionalProperties": false,
"required": [
"gate_id",
"suite_id",
"shard_index",
"shard_total",
"tested_sha",
"run_id",
"run_attempt",
"command_id",
"gate_type",
"status",
"cause",
"exit_code",
"duration_ms",
"evidence"
],
"properties": {
"gate_id": { "type": "string", "minLength": 1 },
"suite_id": { "type": ["string", "null"] },
"shard_index": { "type": ["integer", "null"], "minimum": 0 },
"shard_total": { "type": ["integer", "null"], "minimum": 1 },
"tested_sha": { "$ref": "#/$defs/sha" },
"run_id": { "type": "string", "minLength": 1 },
"run_attempt": { "type": "integer", "minimum": 1 },
"command_id": { "type": "string", "minLength": 1 },
"gate_type": { "enum": ["static", "test", "artifact"] },
"status": { "enum": ["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"] },
"reason": { "type": "string", "minLength": 1 },
"cause": {
"anyOf": [
{ "type": "null" },
{ "$ref": "#/$defs/gateInstanceKey" }
]
},
"exit_code": { "type": ["integer", "null"] },
"duration_ms": { "type": "integer", "minimum": 0 },
"evidence": {
"type": "array",
"items": { "$ref": "#/$defs/evidenceRef" }
}
},
"allOf": [
{
"if": {
"properties": { "status": { "const": "SKIPPED" } },
"required": ["status"]
},
"then": { "required": ["reason"] }
}
]
}
}
}

View File

@@ -0,0 +1,30 @@
const CLOSE_RE =
/gh issue close\b|issues\.update\b|state=closed/g;
const KEYWORD_RE = new RegExp(
String.raw`\b(?:fix(?:es|ed)?|close[sd]?|resolve[sd]?)\s+#(\d+)\b`,
"i"
);
export function findTrackerCloses(workflowText) {
const hits = [];
const lines = String(workflowText ?? "").split(/\n/);
for (let i = 0; i < lines.length; i++) {
CLOSE_RE.lastIndex = 0;
if (CLOSE_RE.test(lines[i])) {
hits.push({ line: i + 1, text: lines[i].trim() });
}
CLOSE_RE.lastIndex = 0;
}
return hits;
}
export function closingKeywordInBody(body, tracker = 12732) {
const re = new RegExp(KEYWORD_RE.source, KEYWORD_RE.flags.includes("g") ? KEYWORD_RE.flags : `${KEYWORD_RE.flags}g`);
const text = String(body ?? "");
let m;
while ((m = re.exec(text)) !== null) {
if (Number(m[1]) === Number(tracker)) return true;
}
return false;
}

View File

@@ -0,0 +1,102 @@
import fs from "node:fs";
import path from "node:path";
import {
COLLECTORS,
globToRegExp,
} from "../../check/check-test-discovery.mjs";
const UNIT_CI_GLOBS = new Set([
"tests/unit/*.test.ts",
"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,translator,ui,usage}/**/*.test.ts",
"tests/unit/dashboard/**/*.test.ts",
"tests/unit/serial/**/*.test.ts",
"tests/unit/**/*.test.mjs",
]);
const INTEGRATION_GLOBS = new Set([
"tests/integration/*.test.ts",
"tests/integration/combo-matrix/*.test.ts",
]);
function inScope(collector, scopeSuites) {
const suites = new Set(scopeSuites);
if (suites.has("test:unit:ci") && UNIT_CI_GLOBS.has(collector.glob)) return true;
if (suites.has("test:integration") && INTEGRATION_GLOBS.has(collector.glob)) return true;
if (suites.has("test:vitest") && collector.sources?.includes("vitest.mcp.config.ts")) {
return true;
}
return false;
}
function walkTestFiles(root = process.cwd()) {
const out = [];
function walk(dir) {
if (!fs.existsSync(dir)) return;
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
if (e.name === "node_modules" || e.name === ".git") continue;
const p = path.join(dir, e.name);
if (e.isDirectory()) walk(p);
else if (/\.(test|spec)\.(ts|tsx|mjs|js)$/.test(e.name)) {
out.push(path.relative(root, p).split(path.sep).join("/"));
}
}
}
walk(path.join(root, "tests"));
walk(path.join(root, "open-sse"));
walk(path.join(root, "src"));
return out;
}
export function canonicalSet(scopeSuites, collectors = COLLECTORS, files) {
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
const regexes = scoped.map((c) => globToRegExp(c.glob));
const discovered = files ?? walkTestFiles();
return discovered.filter((f) => regexes.some((re) => re.test(f)));
}
export function knownUnexecuted(scopeSuites, collectors = COLLECTORS, baseline, files) {
const discovered = files ?? walkTestFiles();
const orphans = baseline?.orphans ?? [];
const outOfScope = collectors.filter((c) => !inScope(c, scopeSuites));
const collectorsOut = outOfScope.map((c) => {
const re = globToRegExp(c.glob);
const count = discovered.filter((f) => re.test(f)).length;
return {
glob: c.glob,
count,
reason: "collector runner is not a suite of this scope",
};
});
return {
orphans: { count: orphans.length, paths: orphans },
collectors: collectorsOut,
};
}
export function inventoryErrors(scopeSuites, collectors, baseline, discoveredFiles) {
const errors = [];
const full = COLLECTORS;
const givenGlobs = new Set(collectors.map((c) => c.glob));
for (const c of full) {
if (!givenGlobs.has(c.glob)) {
errors.push({
code: "collector_omitted",
glob: c.glob,
detail: `collector ${c.glob} omitted without known_unexecuted listing`,
});
}
}
const ku = knownUnexecuted(scopeSuites, collectors, baseline, discoveredFiles);
const knownGlobs = new Set(ku.collectors.map((c) => c.glob));
const knownOrphans = new Set(ku.orphans.paths);
const scoped = collectors.filter((c) => inScope(c, scopeSuites));
const regexes = scoped.map((c) => globToRegExp(c.glob));
for (const f of discoveredFiles ?? []) {
const inCanonical = regexes.some((re) => re.test(f));
const inKnown = knownOrphans.has(f) || [...knownGlobs].some((g) => globToRegExp(g).test(f));
if (!inCanonical && !inKnown) {
errors.push({ code: "unmapped_file", path: f, detail: "discovered file belongs to no set" });
}
}
return errors;
}

View File

@@ -0,0 +1,41 @@
const SUBTEST = /^# Subtest:\s+(\S+)/;
const RESULT = /^(ok|not ok)\s+\d+\s+-\s+(\S+)/;
export function fromNodeTestTap(tapText, argvFiles) {
const completed = [];
const failed = [];
const seen = new Set();
const lines = String(tapText ?? "").split(/\r?\n/);
let pending = null;
for (const line of lines) {
const sub = line.match(SUBTEST);
if (sub) {
pending = sub[1];
continue;
}
const res = line.match(RESULT);
if (res) {
const file = pending;
const ok = res[1] === "ok";
if (file) {
seen.add(file);
if (!ok) {
if (!failed.includes(file)) failed.push(file);
const i = completed.indexOf(file);
if (i >= 0) completed.splice(i, 1);
} else if (!failed.includes(file) && !completed.includes(file)) {
completed.push(file);
}
}
}
}
const attempted = [...argvFiles];
const missing = attempted.filter((f) => !seen.has(f));
return {
completed,
attempted,
missing,
failed,
pass: completed.length > 0 && missing.length === 0 && failed.length === 0,
};
}

View File

@@ -0,0 +1,243 @@
import { gateKey, sameKey } from "./types.mjs";
export function classifyDependent(prereqStatus, dependentKey, prereqKey) {
if (prereqStatus === "FAIL") {
return { status: "FAIL", cause: prereqKey };
}
if (prereqStatus === "INFRA_ERROR") {
return { status: "INFRA_ERROR", cause: prereqKey };
}
if (prereqStatus == null) {
return {
status: "INFRA_ERROR",
cause: prereqKey,
evidence_error: {
code: "prerequisite_missing",
gate: dependentKey,
detail: `missing prerequisite ${prereqKey.gate_id}`,
},
};
}
if (prereqStatus === "SKIPPED") {
return { status: "SKIPPED", cause: prereqKey };
}
return { status: "RUN", cause: null };
}
function requiredSet(plan) {
return plan.required_gates ?? [];
}
function optionalSet(plan) {
return plan.optional_gates ?? [];
}
function isRequired(plan, k) {
return requiredSet(plan).some((r) => sameKey(r, k));
}
function copies(gates, k) {
return gates.filter((g) => sameKey(gateKey(g), k));
}
function copiesByGateId(gates, gateId) {
return gates.filter((g) => g.gate_id === gateId);
}
function uniqueKeys(keys) {
const out = [];
for (const k of keys) {
if (!out.some((existing) => sameKey(existing, k))) out.push(k);
}
return out;
}
function keysForGateId(plan, gates, gateId) {
return uniqueKeys([
...copiesByGateId(gates, gateId).map((g) => gateKey(g)),
...requiredSet(plan).filter((k) => k.gate_id === gateId),
...optionalSet(plan).filter((k) => k.gate_id === gateId),
]);
}
function statusOf(gates, k) {
const list = copies(gates, k);
if (list.length === 0) return null;
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
if (list.some((g) => g.status === "FAIL")) return "FAIL";
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
return list[0].status;
}
function statusOfGateId(gates, gateId) {
const list = copiesByGateId(gates, gateId);
if (list.length === 0) return null;
if (list.some((g) => g.status === "INFRA_ERROR")) return "INFRA_ERROR";
if (list.some((g) => g.status === "FAIL")) return "FAIL";
if (list.some((g) => g.status === "SKIPPED")) return "SKIPPED";
return list[0].status;
}
function pushEvidenceError(evidence_errors, err) {
if (!err) return;
const already = evidence_errors.some(
(e) =>
e.code === err.code &&
e.detail === err.detail &&
e.gate?.gate_id === err.gate?.gate_id
);
if (!already) evidence_errors.push(err);
}
function patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity) {
const matches = copies(gates, depKey);
const reason =
classified.status === "SKIPPED" ? `classified from ${prereqKey.gate_id}` : undefined;
const exit_code = classified.status === "FAIL" ? 1 : 2;
if (matches.length === 0) {
gates.push({
gate_id: depKey.gate_id,
suite_id: depKey.suite_id,
shard_index: depKey.shard_index,
shard_total: depKey.shard_total,
tested_sha: identity.tested_sha || "0".repeat(40),
run_id: identity.run_id ?? "0",
run_attempt: identity.run_attempt ?? 1,
command_id: depKey.gate_id,
gate_type: "artifact",
status: classified.status,
cause: classified.cause,
reason,
exit_code,
duration_ms: 0,
evidence: [],
});
if (classified.evidence_error) pushEvidenceError(evidence_errors, classified.evidence_error);
return true;
}
let changed = false;
for (const existing of matches) {
if (
existing.status === classified.status &&
((existing.cause == null && classified.cause == null) ||
(existing.cause && classified.cause && sameKey(existing.cause, classified.cause)))
) {
continue;
}
existing.status = classified.status;
existing.cause = classified.cause;
existing.exit_code = exit_code;
if (classified.status === "SKIPPED" && !existing.reason) existing.reason = reason;
changed = true;
}
if (changed && classified.evidence_error) {
pushEvidenceError(evidence_errors, classified.evidence_error);
}
return changed;
}
function assertAcyclic(deps) {
const visiting = new Set();
const done = new Set();
function walk(id) {
if (done.has(id)) return;
if (visiting.has(id)) throw new Error("cyclic prerequisite");
visiting.add(id);
if (Object.hasOwn(deps, id)) walk(deps[id]);
visiting.delete(id);
done.add(id);
}
for (const id of Object.keys(deps)) walk(id);
}
export function reduce(plan, records) {
const deps = plan.dependencies ?? {};
assertAcyclic(deps);
for (const [depId, prereqId] of Object.entries(deps)) {
const requiredDep = requiredSet(plan).some((k) => k.gate_id === depId);
const optionalPrereq = optionalSet(plan).some((k) => k.gate_id === prereqId);
if (requiredDep && optionalPrereq) {
throw new Error("optional prerequisite");
}
}
const gates = [];
const evidence_errors = [];
for (const rec of records) {
const k = gateKey(rec);
const copy = { ...rec, cause: rec.cause ?? null };
if (copy.status === "SKIPPED" && isRequired(plan, k) && !copy.reason) {
copy.reason = "required skipped";
}
gates.push(copy);
}
const identity = plan.identity ?? {};
const edges = Object.entries(deps);
let changed = true;
let guard = edges.length + 1;
while (changed && guard-- > 0) {
changed = false;
for (const [depId, prereqId] of edges) {
const prereqKey = { gate_id: prereqId, suite_id: null, shard_index: null, shard_total: null };
let depKeys = keysForGateId(plan, gates, depId);
if (depKeys.length === 0) {
depKeys = [{ gate_id: depId, suite_id: null, shard_index: null, shard_total: null }];
}
const prereqStatus = statusOfGateId(gates, prereqId);
for (const depKey of depKeys) {
const classified = classifyDependent(prereqStatus, depKey, prereqKey);
if (classified.status === "RUN") continue;
if (patchDependent(gates, depKey, classified, prereqKey, evidence_errors, identity)) {
changed = true;
}
}
}
}
for (const k of requiredSet(plan)) {
const rec = gates.find((g) => sameKey(gateKey(g), k));
if (!rec) {
evidence_errors.push({
code: "missing_record",
gate: k,
detail: `required gate ${k.gate_id} has no record`,
});
} else if (rec.status === "SKIPPED") {
evidence_errors.push({
code: "required_skipped",
gate: k,
detail: rec.reason ?? "required gate SKIPPED",
});
}
}
const required = requiredSet(plan);
if (required.length === 0) {
evidence_errors.push({
code: "empty_required_set",
gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null },
detail: "required_gates is empty",
});
}
let verdict = "VERIFIED";
const hasFail = gates.some(
(g) => g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL"
);
const hasUnverified =
evidence_errors.length > 0 ||
gates.some(
(g) =>
isRequired(plan, gateKey(g)) &&
(g.status === "SKIPPED" || g.status === "INFRA_ERROR")
);
if (hasFail) verdict = "FAILED";
else if (hasUnverified) verdict = "UNVERIFIED";
else if (required.some((k) => !gates.some((g) => sameKey(gateKey(g), k)))) {
verdict = "UNVERIFIED";
}
return { verdict, evidence_errors, gates };
}

View File

@@ -0,0 +1,50 @@
export function adaptCompiler({ commandId, inputDigest, exitCode, diagnostics }) {
const diags = Array.isArray(diagnostics) ? diagnostics : [];
const digest = typeof inputDigest === "string" ? inputDigest : "";
if (exitCode === 0 && digest.length > 0) {
return {
command_id: commandId,
input_digest: digest,
exit_code: 0,
diagnostics: diags,
status: "PASS",
};
}
if (exitCode === 0 && digest.length === 0) {
return {
command_id: commandId,
input_digest: digest,
exit_code: 0,
diagnostics: diags,
status: "INFRA_ERROR",
};
}
return {
command_id: commandId,
input_digest: digest,
exit_code: exitCode,
diagnostics: diags,
status: "FAIL",
};
}
export function adaptScript({ commandId, inputDigest, exitCode, stdout }) {
const digest = typeof inputDigest === "string" ? inputDigest : "";
let parsed = null;
if (typeof stdout === "string" && stdout.trim()) {
try {
parsed = JSON.parse(stdout);
} catch {
parsed = null;
}
}
const diagnostics = parsed ?? { input_digest: digest, exit_code: exitCode, diagnostics: stdout ?? "" };
const status = exitCode === 0 ? (digest ? "PASS" : "INFRA_ERROR") : "FAIL";
return {
command_id: commandId,
input_digest: digest,
exit_code: exitCode,
diagnostics,
status,
};
}

View File

@@ -0,0 +1,19 @@
export const STATUSES = Object.freeze(["PASS", "FAIL", "INFRA_ERROR", "SKIPPED"]);
export const VERDICTS = Object.freeze(["VERIFIED", "FAILED", "UNVERIFIED"]);
export function gateKey(rec) {
return {
gate_id: rec.gate_id,
suite_id: rec.suite_id ?? null,
shard_index: rec.shard_index ?? null,
shard_total: rec.shard_total ?? null,
};
}
export function keyId(k) {
return `${k.gate_id}\0${k.suite_id ?? ""}\0${k.shard_index ?? ""}\0${k.shard_total ?? ""}`;
}
export function sameKey(a, b) {
return keyId(a) === keyId(b);
}

View File

@@ -0,0 +1,81 @@
#!/usr/bin/env node
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import Ajv from "ajv";
import { reduce } from "./release-acceptance/reduce.mjs";
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
function loadJson(p) {
return JSON.parse(readFileSync(p, "utf8"));
}
export function exitFor(verdict) {
if (verdict === "VERIFIED") return 0;
if (verdict === "FAILED") return 1;
return 2;
}
export function reduceManifests(plan, manifests) {
const records = [];
for (const m of manifests) {
if (Array.isArray(m.gates)) records.push(...m.gates);
else records.push(m);
}
return reduce(plan, records);
}
export function validateReport(report, schema) {
const ajv = new Ajv({ allErrors: true, strict: false });
const validate = ajv.compile(schema);
return { ok: validate(report), errors: validate.errors };
}
function parseArgs(argv) {
const out = { plan: null, manifests: null, out: join(ROOT, "release-acceptance-report.json") };
for (let i = 2; i < argv.length; i++) {
if (argv[i] === "--plan") out.plan = argv[++i];
else if (argv[i] === "--manifests") out.manifests = argv[++i];
else if (argv[i] === "--out") out.out = argv[++i];
}
return out;
}
export async function main(argv = process.argv) {
const args = parseArgs(argv);
const plan = loadJson(args.plan);
const schema = loadJson(join(ROOT, "config/quality/release-acceptance.schema.json"));
const files = readdirSync(args.manifests)
.filter((f) => f.endsWith(".json"))
.map((f) => loadJson(join(args.manifests, f)));
const reduced = reduceManifests(plan, files);
const report = {
schema_version: 1,
identity: plan.identity,
required_gates: plan.required_gates ?? [],
gates: reduced.gates,
evidence_errors: reduced.evidence_errors,
verdict: reduced.verdict,
artifact: plan.artifact ?? null,
};
const { ok, errors } = validateReport(report, schema);
if (!ok) {
if (report.verdict !== "FAILED") report.verdict = "UNVERIFIED";
const gate =
Array.isArray(plan.required_gates) && plan.required_gates.length > 0
? plan.required_gates[0]
: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null };
report.evidence_errors = [
...(report.evidence_errors ?? []),
{ code: "schema_invalid", gate, detail: JSON.stringify(errors) },
];
}
mkdirSync(dirname(args.out), { recursive: true });
writeFileSync(args.out, JSON.stringify(report, null, 2) + "\n");
return exitFor(report.verdict);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
main().then((code) => process.exit(code));
}

View File

@@ -0,0 +1,89 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
{
"gate_id": "pack-boot",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"gates": [
{
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "check:pack-artifact",
"gate_type": "artifact",
"status": "FAIL",
"cause": null,
"exit_code": 1,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
},
{
"gate_id": "pack-boot",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "check:pack-boot",
"gate_type": "artifact",
"status": "FAIL",
"cause": {
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
"exit_code": 1,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
}
],
"evidence_errors": [],
"verdict": "FAILED",
"artifact": {
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e",
"identity": "omniroute.tgz"
}
}

View File

@@ -0,0 +1,100 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
{
"gate_id": "pack-boot",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"gates": [
{
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "check:pack-artifact",
"gate_type": "artifact",
"status": "INFRA_ERROR",
"cause": null,
"exit_code": 2,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
},
{
"gate_id": "pack-boot",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "check:pack-boot",
"gate_type": "artifact",
"status": "INFRA_ERROR",
"cause": {
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
"exit_code": 2,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
}
],
"evidence_errors": [
{
"code": "infra",
"gate": {
"gate_id": "pack-artifact",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
"detail": "evidence cap"
}
],
"verdict": "UNVERIFIED",
"artifact": {
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e",
"identity": "omniroute.tgz"
}
}

View File

@@ -0,0 +1,39 @@
- name: Close tracking issue when the branch is green again
if: steps.validate.outputs.exit == '0'
env:
GH_TOKEN: ${{ github.token }}
TARGET: ${{ steps.branch.outputs.target }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
# The open/update step above is the UPWARD half of the loop; without this
# step a stale "not green" issue outlives the fix and every base-green check
# (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited.
TITLE="🔴 Release branch not green: ${TARGET}"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
--search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "")
if [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \
--comment "✅ \`${TARGET}\` is release-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)."
echo "Closed issue #$EXISTING"
fi
- name: Close tracking issue when the branch is green again
if: steps.validate.outputs.exit == '0'
env:
GH_TOKEN: ${{ github.token }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
# The open/update step above is the UPWARD half of the loop; without this
# step a stale "not green" issue outlives the fix and every base-green check
# (`AGENTS.md` → "Base-green check") keeps stamping new PRs as base-red inherited.
TITLE="🔴 main branch not green"
EXISTING=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \
--search "in:title $TITLE" --json number --jq '.[0].number' 2>/dev/null || echo "")
if [ -n "$EXISTING" ]; then
gh issue close "$EXISTING" --repo "$GITHUB_REPOSITORY" --reason completed \
--comment "✅ \`main\` is main-green again at \`${GITHUB_SHA:0:9}\` — ${RUN_URL}. Auto-closed by Release-Green (continuous)."
echo "Closed issue #$EXISTING"
fi

View File

@@ -0,0 +1,24 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"artifact": null
}

View File

@@ -0,0 +1,51 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "lint",
"gate_type": "static",
"status": "PASS",
"cause": null,
"exit_code": 0,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
}
],
"evidence_errors": [],
"verdict": "VERIFIED",
"artifact": null
}

View File

@@ -0,0 +1,63 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "lint",
"gate_type": "static",
"status": "SKIPPED",
"cause": null,
"exit_code": null,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
],
"reason": "plan-optional-looking"
}
],
"evidence_errors": [
{
"code": "required_skipped",
"gate": {
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null
},
"detail": "required gate SKIPPED"
}
],
"verdict": "UNVERIFIED",
"artifact": null
}

View File

@@ -0,0 +1,51 @@
{
"schema_version": 1,
"identity": {
"repository": "diegosouzapw/OmniRoute",
"run_id": "1",
"run_attempt": 1,
"workflow": "release-acceptance.yml",
"trigger": "push",
"scope": "release",
"requested_ref": "refs/heads/release/v3.8.51",
"base_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"candidate_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e"
},
"required_gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null
}
],
"gates": [
{
"gate_id": "lint",
"suite_id": null,
"shard_index": null,
"shard_total": null,
"tested_sha": "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e",
"run_id": "1",
"run_attempt": 1,
"command_id": "lint",
"gate_type": "static",
"status": "PASS",
"cause": null,
"exit_code": 0,
"duration_ms": 10,
"evidence": [
{
"artifact_id": "logs",
"member": "lint.log",
"algorithm": "sha256",
"digest": "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e"
}
]
}
],
"evidence_errors": [],
"verdict": "VERIFIED",
"artifact": null
}

View File

@@ -0,0 +1,174 @@
import test from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
exitFor,
reduceManifests,
main,
} from "../../scripts/quality/validate-release-acceptance.mjs";
const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e";
function key(id) {
return { gate_id: id, suite_id: null, shard_index: null, shard_total: null };
}
function gate(id, status) {
return {
gate_id: id,
suite_id: null,
shard_index: null,
shard_total: null,
tested_sha: SHA,
run_id: "1",
run_attempt: 1,
command_id: id,
gate_type: "static",
status,
cause: null,
exit_code: status === "PASS" ? 0 : 1,
duration_ms: 1,
evidence: [
{
artifact_id: "logs",
member: "lint.log",
algorithm: "sha256",
digest: "7f227db1653b6b723b07c8f2f6eb488f1f09e2f083ca7a3f5e02bbb274f5ff2e",
},
],
};
}
test("exit mapping", () => {
assert.equal(exitFor("VERIFIED"), 0);
assert.equal(exitFor("FAILED"), 1);
assert.equal(exitFor("UNVERIFIED"), 2);
});
test("three PASS manifests yield VERIFIED", () => {
const plan = {
required_gates: [key("a"), key("b"), key("c")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
};
const out = reduceManifests(plan, [
{ gates: [gate("a", "PASS")] },
{ gates: [gate("b", "PASS")] },
{ gates: [gate("c", "PASS")] },
]);
assert.equal(out.verdict, "VERIFIED");
assert.equal(exitFor(out.verdict), 0);
});
test("one FAIL yields FAILED", () => {
const plan = {
required_gates: [key("a")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
};
const out = reduceManifests(plan, [{ gates: [gate("a", "FAIL")] }]);
assert.equal(out.verdict, "FAILED");
assert.equal(exitFor(out.verdict), 1);
});
test("required missing yields UNVERIFIED", () => {
const plan = {
required_gates: [key("a"), key("b")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
};
const out = reduceManifests(plan, [{ gates: [gate("a", "PASS")] }]);
assert.equal(out.verdict, "UNVERIFIED");
assert.equal(exitFor(out.verdict), 2);
});
test("workflow source-guard", () => {
const text = readFileSync(".github/workflows/release-acceptance.yml", "utf8");
assert.match(text, /name: Release acceptance/);
assert.match(text, /cancel-in-progress: false/);
assert.equal(text.includes("gh issue close"), false);
assert.match(text, /if: github.event_name != 'pull_request'/);
});
test("schema_invalid does not throw when required_gates is missing", async () => {
const dir = mkdtempSync(join(tmpdir(), "acc-"));
writeFileSync(
join(dir, "plan.json"),
JSON.stringify({
identity: {
repository: "diegosouzapw/OmniRoute",
run_id: "1",
run_attempt: 1,
workflow: "release-acceptance.yml",
trigger: "push",
scope: "release",
requested_ref: "refs/heads/release/v3.8.51",
base_sha: SHA,
candidate_sha: SHA,
tested_sha: SHA,
},
artifact: null,
})
);
const man = join(dir, "m");
mkdirSync(man);
writeFileSync(join(man, "a.json"), JSON.stringify({ gates: [gate("a", "PASS")] }));
const out = join(dir, "report.json");
const code = await main([
"node",
"cli",
"--plan",
join(dir, "plan.json"),
"--manifests",
man,
"--out",
out,
]);
assert.equal(code, 2);
const report = JSON.parse(readFileSync(out, "utf8"));
assert.equal(report.verdict, "UNVERIFIED");
assert.ok(Array.isArray(report.required_gates));
assert.ok(report.evidence_errors.some((e) => e.code === "empty_required_set"));
assert.equal(
report.evidence_errors.some((e) => e.code === "schema_invalid"),
false
);
});
test("schema_invalid keeps FAILED when reduce already failed", async () => {
const dir = mkdtempSync(join(tmpdir(), "acc-fail-"));
const plan = {
required_gates: [key("a")],
identity: {
repository: "diegosouzapw/OmniRoute",
run_id: "1",
run_attempt: 1,
workflow: "release-acceptance.yml",
trigger: "push",
scope: "release",
requested_ref: "refs/heads/release/v3.8.51",
base_sha: SHA,
candidate_sha: SHA,
tested_sha: SHA,
},
artifact: null,
};
writeFileSync(join(dir, "plan.json"), JSON.stringify(plan));
const man = join(dir, "m");
mkdirSync(man);
const g = gate("a", "FAIL");
g.unexpected = true;
writeFileSync(join(man, "a.json"), JSON.stringify({ gates: [g] }));
const out = join(dir, "report.json");
const code = await main([
"node",
"cli",
"--plan",
join(dir, "plan.json"),
"--manifests",
man,
"--out",
out,
]);
assert.equal(code, 1);
const report = JSON.parse(readFileSync(out, "utf8"));
assert.equal(report.verdict, "FAILED");
assert.ok(report.evidence_errors.some((e) => e.code === "schema_invalid"));
});

View File

@@ -0,0 +1,23 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import {
findTrackerCloses,
closingKeywordInBody,
} from "../../scripts/quality/release-acceptance/closeOracle.mjs";
test("nightly still auto-closes the tracker via two steps (deliberate, #12085)", () => {
const text = readFileSync(".github/workflows/nightly-release-green.yml", "utf8");
assert.equal(findTrackerCloses(text).length, 2);
const legacy = readFileSync(
new URL("../fixtures/release-acceptance/legacy-close-steps.yml", import.meta.url),
"utf8"
);
assert.equal(findTrackerCloses(legacy).length, 2);
});
test("Fixes #12732 is a closing keyword; Related to #12732 is not", () => {
assert.equal(closingKeywordInBody("Fixes #12732.\n"), true);
assert.equal(closingKeywordInBody("Related to #12732.\n"), false);
assert.equal(closingKeywordInBody("Fixes #1. Closes #12732\n"), true);
});

View File

@@ -0,0 +1,40 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { COLLECTORS } from "../../scripts/check/check-test-discovery.mjs";
import {
knownUnexecuted,
inventoryErrors,
} from "../../scripts/quality/release-acceptance/inventory.mjs";
const RELEASE_SUITES = ["test:unit:ci", "test:vitest", "test:integration"];
const baseline = JSON.parse(
readFileSync(new URL("../../config/quality/test-discovery-baseline.json", import.meta.url), "utf8")
);
test("tsx files under tests/unit are known_unexecuted for release scope, not inventory errors", () => {
const ku = knownUnexecuted(RELEASE_SUITES, COLLECTORS, baseline);
const tsx = ku.collectors.find((c) => c.glob === "tests/unit/**/*.test.tsx");
assert.ok(tsx, "tsx collector must be listed as known_unexecuted");
assert.equal(typeof tsx.count, "number");
assert.ok(tsx.count > 0);
});
test("omitting a collector without listing it is an inventory error", () => {
const collectors = COLLECTORS.filter((c) => c.glob !== "tests/unit/**/*.test.tsx");
const discoveredFiles = ["tests/unit/AutoComboCatalog.test.tsx"];
const errors = inventoryErrors(RELEASE_SUITES, collectors, baseline, discoveredFiles);
assert.ok(errors.some((e) => e.code === "collector_omitted"));
});
test("combo-matrix glob is in release integration scope, not known_unexecuted", () => {
const ku = knownUnexecuted(RELEASE_SUITES, COLLECTORS, baseline);
assert.equal(
ku.collectors.some((c) => c.glob === "tests/integration/combo-matrix/*.test.ts"),
false
);
const combo = COLLECTORS.find(
(c) => c.glob === "tests/integration/combo-matrix/*.test.ts"
);
assert.ok(combo);
});

View File

@@ -0,0 +1,86 @@
import test from "node:test";
import assert from "node:assert/strict";
import { fromNodeTestTap } from "../../scripts/quality/release-acceptance/nodeReporter.mjs";
const TAP = `TAP version 13
# Subtest: tests/unit/a.test.ts
ok 1 - tests/unit/a.test.ts
# Subtest: tests/unit/b.test.ts
ok 2 - tests/unit/b.test.ts
# Subtest: tests/unit/c.test.ts
not ok 3 - tests/unit/c.test.ts
`;
test("argv file without TAP completion is missing", () => {
const out = fromNodeTestTap(TAP, [
"tests/unit/a.test.ts",
"tests/unit/b.test.ts",
"tests/unit/c.test.ts",
"tests/unit/d.test.ts",
]);
assert.equal(out.completed.length, 2);
assert.deepEqual(out.failed, ["tests/unit/c.test.ts"]);
assert.equal(out.missing.length, 1);
assert.equal(out.missing[0], "tests/unit/d.test.ts");
assert.equal(out.pass, false);
});
test("zero completed files is not PASS", () => {
const out = fromNodeTestTap("TAP version 13\n", ["tests/unit/a.test.ts"]);
assert.equal(out.completed.length, 0);
assert.equal(out.pass, false);
});
test("Subtest path wins when the result line has a short name", () => {
const tap = `TAP version 13
# Subtest: tests/unit/a.test.ts
ok 1 - some name
`;
const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]);
assert.equal(out.completed[0], "tests/unit/a.test.ts");
assert.equal(out.missing.length, 0);
});
test("not ok is not pass", () => {
const tap = `TAP version 13
# Subtest: tests/unit/a.test.ts
not ok 1 - tests/unit/a.test.ts
`;
const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]);
assert.equal(out.pass, false);
assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]);
});
test("ok line without Subtest does not complete an argv file", () => {
const tap = `# a malicious test printed:
ok 99 - tests/unit/missing.test.ts
`;
const out = fromNodeTestTap(tap, ["tests/unit/missing.test.ts"]);
assert.equal(out.pass, false);
assert.deepEqual(out.missing, ["tests/unit/missing.test.ts"]);
});
test("later not ok retracts an earlier ok for the same Subtest", () => {
const tap = `TAP version 13
# Subtest: tests/unit/a.test.ts
ok 1 - tests/unit/a.test.ts
# Subtest: tests/unit/a.test.ts
not ok 2 - tests/unit/a.test.ts
`;
const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]);
assert.equal(out.pass, false);
assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]);
assert.equal(out.completed.includes("tests/unit/a.test.ts"), false);
});
test("later not ok on the same pending Subtest retracts ok", () => {
const tap = `TAP version 13
# Subtest: tests/unit/a.test.ts
ok 1 - tests/unit/a.test.ts
not ok 2 - tests/unit/a.test.ts
`;
const out = fromNodeTestTap(tap, ["tests/unit/a.test.ts"]);
assert.equal(out.pass, false);
assert.deepEqual(out.failed, ["tests/unit/a.test.ts"]);
assert.equal(out.completed.includes("tests/unit/a.test.ts"), false);
});

View File

@@ -0,0 +1,68 @@
import test from "node:test";
import assert from "node:assert/strict";
import { reduce } from "../../scripts/quality/release-acceptance/reduce.mjs";
const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e";
const planPack = {
required_gates: [
{ gate_id: "pack-artifact", suite_id: null, shard_index: null, shard_total: null },
{ gate_id: "pack-boot", suite_id: null, shard_index: null, shard_total: null },
],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { "pack-boot": "pack-artifact" },
};
function record(partial) {
return {
gate_id: "pack-artifact",
suite_id: null,
shard_index: null,
shard_total: null,
tested_sha: SHA,
run_id: "1",
run_attempt: 1,
command_id: "check:pack-artifact",
gate_type: "artifact",
status: "PASS",
cause: null,
exit_code: 0,
duration_ms: 10,
evidence: [],
...partial,
};
}
test("legacy computeVerdict still hard-fails pack-boot when pack-artifact times out", async () => {
const { computeVerdict } = await import("../../scripts/quality/validate-release-green.mjs");
const v = computeVerdict([
{ id: "pack-artifact", kind: "hard", ok: false, detail: "timeout" },
{
id: "pack-boot",
kind: "hard",
ok: false,
detail: "skipped because package-artifact did not produce a valid dist/ build",
},
]);
assert.equal(v.releaseGreen, false);
});
test("new reducer maps the same timeout to UNVERIFIED", () => {
const out = reduce(planPack, [
record({ gate_id: "pack-artifact", status: "INFRA_ERROR" }),
]);
assert.equal(out.verdict, "UNVERIFIED");
});
test("synthesized pack-boot without identity.tested_sha keeps a 40-hex sha and FAILED", () => {
const plan = {
required_gates: planPack.required_gates,
identity: { run_id: "1", run_attempt: 1 },
dependencies: { "pack-boot": "pack-artifact" },
};
const out = reduce(plan, [record({ gate_id: "pack-artifact", status: "FAIL" })]);
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.ok(boot);
assert.notEqual(boot.tested_sha, null);
assert.match(String(boot.tested_sha), /^[0-9a-f]{40}$/);
assert.equal(out.verdict, "FAILED");
});

View File

@@ -0,0 +1,287 @@
import test from "node:test";
import assert from "node:assert/strict";
import { reduce } from "../../scripts/quality/release-acceptance/reduce.mjs";
const SHA = "30b5bf18fbe827a0283ce17e91bda22cc8b4c13e";
function key(gate_id) {
return { gate_id, suite_id: null, shard_index: null, shard_total: null };
}
function record(partial) {
return {
gate_id: "lint",
suite_id: null,
shard_index: null,
shard_total: null,
tested_sha: SHA,
run_id: "1",
run_attempt: 1,
command_id: partial.gate_id ?? "lint",
gate_type: "static",
status: "PASS",
cause: null,
exit_code: 0,
duration_ms: 10,
evidence: [],
...partial,
};
}
function planWithRequired(gateId, extra = {}) {
return {
required_gates: [key(gateId)],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
...extra,
};
}
const planPack = {
required_gates: [key("pack-artifact"), key("pack-boot")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { "pack-boot": "pack-artifact" },
};
test("required SKIPPED never yields VERIFIED", () => {
const out = reduce(planWithRequired("lint"), [
record({ gate_id: "lint", status: "SKIPPED", reason: "optional-looking" }),
]);
assert.equal(out.verdict, "UNVERIFIED");
});
test("pack-artifact FAIL classifies pack-boot as FAIL with cause", () => {
const out = reduce(planPack, [
record({ gate_id: "pack-artifact", status: "FAIL", gate_type: "artifact" }),
]);
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.equal(boot.status, "FAIL");
assert.equal(boot.cause.gate_id, "pack-artifact");
assert.equal(out.verdict, "FAILED");
});
test("pack-artifact INFRA_ERROR classifies pack-boot as INFRA_ERROR", () => {
const out = reduce(planPack, [
record({
gate_id: "pack-artifact",
status: "INFRA_ERROR",
gate_type: "artifact",
}),
]);
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.equal(boot.status, "INFRA_ERROR");
assert.equal(out.verdict, "UNVERIFIED");
});
test("plan that marks a required gate's prerequisite optional is rejected", () => {
const illegalPlan = {
required_gates: [key("pack-boot")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { "pack-boot": "pack-artifact" },
optional_gates: [key("pack-artifact")],
};
assert.throws(() => reduce(illegalPlan, []), /optional prerequisite/);
});
test("required SKIPPED prerequisite classifies dependent as SKIPPED, does not throw", () => {
const out = reduce(planPack, [
record({
gate_id: "pack-artifact",
status: "SKIPPED",
reason: "runner skipped",
gate_type: "artifact",
}),
]);
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.equal(boot.status, "SKIPPED");
assert.equal(boot.cause.gate_id, "pack-artifact");
assert.equal(out.verdict, "UNVERIFIED");
});
test("INFRA_ERROR artifact reclassifies an already-emitted FAIL boot to INFRA_ERROR", () => {
const out = reduce(planPack, [
record({
gate_id: "pack-artifact",
status: "INFRA_ERROR",
gate_type: "artifact",
}),
record({
gate_id: "pack-boot",
status: "FAIL",
gate_type: "artifact",
}),
]);
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.equal(boot.status, "INFRA_ERROR");
assert.equal(boot.cause.gate_id, "pack-artifact");
assert.equal(out.verdict, "UNVERIFIED");
});
test("empty required_gates is UNVERIFIED", () => {
const out = reduce(
{ required_gates: [], identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 } },
[record({ gate_id: "lint", status: "PASS" })]
);
assert.equal(out.verdict, "UNVERIFIED");
assert.ok(out.evidence_errors.some((e) => e.code === "empty_required_set"));
});
test("INFRA_ERROR artifact reclassifies every FAIL boot copy", () => {
const out = reduce(planPack, [
record({ gate_id: "pack-artifact", status: "INFRA_ERROR", gate_type: "artifact" }),
record({ gate_id: "pack-boot", status: "FAIL", gate_type: "artifact" }),
record({ gate_id: "pack-boot", status: "FAIL", gate_type: "artifact" }),
]);
const boots = out.gates.filter((g) => g.gate_id === "pack-boot");
assert.ok(boots.length >= 1);
assert.ok(boots.every((g) => g.status === "INFRA_ERROR"));
assert.equal(out.verdict, "UNVERIFIED");
});
test("transitive INFRA on a three-gate chain is UNVERIFIED, not leaked FAIL", () => {
const plan = {
required_gates: [key("a"), key("b"), key("c")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { b: "a", c: "b" },
};
const out = reduce(plan, [
record({ gate_id: "a", status: "INFRA_ERROR", gate_type: "artifact" }),
record({ gate_id: "b", status: "FAIL", gate_type: "artifact" }),
record({ gate_id: "c", status: "PASS", gate_type: "artifact" }),
]);
assert.equal(out.gates.find((g) => g.gate_id === "a").status, "INFRA_ERROR");
assert.equal(out.gates.find((g) => g.gate_id === "b").status, "INFRA_ERROR");
assert.equal(out.gates.find((g) => g.gate_id === "c").status, "INFRA_ERROR");
assert.equal(out.verdict, "UNVERIFIED");
});
test("transitive FAIL on a three-gate chain classifies every dependent", () => {
const plan = {
required_gates: [key("a"), key("b"), key("c")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { b: "a", c: "b" },
};
const out = reduce(plan, [
record({ gate_id: "a", status: "FAIL", gate_type: "artifact" }),
record({ gate_id: "b", status: "PASS", gate_type: "artifact" }),
record({ gate_id: "c", status: "PASS", gate_type: "artifact" }),
]);
assert.equal(out.gates.find((g) => g.gate_id === "b").status, "FAIL");
assert.equal(out.gates.find((g) => g.gate_id === "c").status, "FAIL");
assert.equal(out.verdict, "FAILED");
});
test("INFRA copy of a required gate dominates a FAIL copy of the same key", () => {
const out = reduce(planPack, [
record({ gate_id: "pack-artifact", status: "INFRA_ERROR", gate_type: "artifact" }),
record({ gate_id: "pack-artifact", status: "FAIL", gate_type: "artifact" }),
record({ gate_id: "pack-boot", status: "PASS", gate_type: "artifact" }),
]);
assert.equal(out.verdict, "UNVERIFIED");
const boot = out.gates.find((g) => g.gate_id === "pack-boot");
assert.equal(boot.status, "INFRA_ERROR");
});
test("cyclic dependencies are rejected", () => {
const cyclic = {
required_gates: [key("a"), key("b")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { a: "b", b: "a" },
};
assert.throws(
() => reduce(cyclic, [record({ gate_id: "a", status: "INFRA_ERROR" }), record({ gate_id: "b", status: "FAIL" })]),
/cyclic prerequisite/
);
});
test("missing prerequisite records one evidence error, not one per loop", () => {
const out = reduce(
{
required_gates: [key("boot")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { boot: "art" },
},
[]
);
assert.equal(out.verdict, "UNVERIFIED");
assert.equal(
out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length,
1
);
});
test("missing prerequisite records one evidence error for all shards of a gate_id", () => {
const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 };
const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 };
const out = reduce(
{
required_gates: [shard0, shard1],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { u: "art" },
},
[]
);
assert.equal(out.verdict, "UNVERIFIED");
assert.equal(
out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length,
1
);
});
test("two dependents of the same missing prerequisite keep one error per edge", () => {
const out = reduce(
{
required_gates: [key("boot"), key("pack")],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { boot: "art", pack: "art" },
},
[]
);
assert.equal(out.verdict, "UNVERIFIED");
const missing = out.evidence_errors.filter((e) => e.code === "prerequisite_missing");
assert.equal(missing.length, 2);
const gates = missing.map((e) => e.gate?.gate_id).sort();
assert.deepEqual(gates, ["boot", "pack"]);
assert.equal(out.gates.find((g) => g.gate_id === "boot")?.status, "INFRA_ERROR");
assert.equal(out.gates.find((g) => g.gate_id === "pack")?.status, "INFRA_ERROR");
});
test("sharded required dependents inherit a FAIL prerequisite of the same gate_id", () => {
const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 };
const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 };
const out = reduce(
{
required_gates: [shard0, shard1],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { u: "art" },
},
[
record({ gate_id: "art", status: "FAIL", gate_type: "artifact" }),
record({ ...shard0, status: "PASS", gate_type: "artifact" }),
record({ ...shard1, status: "PASS", gate_type: "artifact" }),
]
);
const shards = out.gates.filter((g) => g.gate_id === "u" && g.suite_id === "s");
assert.equal(shards.length, 2);
assert.ok(shards.every((g) => g.status === "FAIL"));
assert.equal(out.verdict, "FAILED");
});
test("sharded required dependents inherit an INFRA prerequisite of the same gate_id", () => {
const shard0 = { gate_id: "u", suite_id: "s", shard_index: 0, shard_total: 2 };
const shard1 = { gate_id: "u", suite_id: "s", shard_index: 1, shard_total: 2 };
const out = reduce(
{
required_gates: [shard0, shard1],
identity: { tested_sha: SHA, run_id: "1", run_attempt: 1 },
dependencies: { u: "art" },
},
[
record({ gate_id: "art", status: "INFRA_ERROR", gate_type: "artifact" }),
record({ ...shard0, status: "PASS", gate_type: "artifact" }),
record({ ...shard1, status: "PASS", gate_type: "artifact" }),
]
);
const shards = out.gates.filter((g) => g.gate_id === "u" && g.suite_id === "s");
assert.ok(shards.every((g) => g.status === "INFRA_ERROR"));
assert.equal(out.verdict, "UNVERIFIED");
});

View File

@@ -0,0 +1,112 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import Ajv from "ajv";
const schema = JSON.parse(
readFileSync(
new URL("../../config/quality/release-acceptance.schema.json", import.meta.url),
"utf8"
)
);
function compile() {
const ajv = new Ajv({ allErrors: true, strict: false });
return ajv.compile(schema);
}
test("version 1 requires cause when status is classified by a prerequisite", () => {
const validate = compile();
const missingCause = JSON.parse(
readFileSync(
new URL("../fixtures/release-acceptance/failed-pack-boot.json", import.meta.url),
"utf8"
)
);
delete missingCause.gates[1].cause;
assert.equal(validate(missingCause), false);
});
test("unknown top-level gate field is invalid in version 1", () => {
const validate = compile();
const extra = JSON.parse(
readFileSync(
new URL("../fixtures/release-acceptance/verified.json", import.meta.url),
"utf8"
)
);
extra.gates[0].unexpected = true;
assert.equal(validate(extra), false);
});
test("evidence member rejects parent traversal", () => {
const validate = compile();
const report = JSON.parse(
readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8")
);
report.gates[0].evidence[0].member = "foo/../../etc/passwd";
assert.equal(validate(report), false);
report.gates[0].evidence[0].member = "..";
assert.equal(validate(report), false);
report.gates[0].evidence[0].member = "foo/..";
assert.equal(validate(report), false);
report.gates[0].evidence[0].member = String.raw`foo\..\x`;
assert.equal(validate(report), false);
});
test("empty required_gates cannot be VERIFIED", () => {
const validate = compile();
const report = JSON.parse(
readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8")
);
report.required_gates = [];
report.gates = [];
report.evidence_errors = [];
report.verdict = "VERIFIED";
assert.equal(validate(report), false);
});
test("empty required_gates is valid when UNVERIFIED", () => {
const validate = compile();
const report = JSON.parse(
readFileSync(new URL("../fixtures/release-acceptance/verified.json", import.meta.url), "utf8")
);
report.required_gates = [];
report.gates = [];
report.evidence_errors = [
{
code: "empty_required_set",
gate: { gate_id: "schema", suite_id: null, shard_index: null, shard_total: null },
detail: "required_gates is empty",
},
];
report.verdict = "UNVERIFIED";
assert.equal(validate(report), true, JSON.stringify(validate.errors));
});
test("unknown extensions field is invalid in version 1", () => {
const validate = compile();
const extra = JSON.parse(
readFileSync(
new URL("../fixtures/release-acceptance/verified.json", import.meta.url),
"utf8"
)
);
extra.gates[0].extensions = { unexpected: true };
assert.equal(validate(extra), false);
});
test("known-answer fixtures validate", () => {
const validate = compile();
for (const name of [
"verified.json",
"failed-pack-boot.json",
"unverified-required-skipped.json",
"infra-pack-boot.json",
]) {
const report = JSON.parse(
readFileSync(new URL(`../fixtures/release-acceptance/${name}`, import.meta.url), "utf8")
);
assert.equal(validate(report), true, `${name}: ${JSON.stringify(validate.errors)}`);
}
});

View File

@@ -0,0 +1,33 @@
import test from "node:test";
import assert from "node:assert/strict";
import { adaptCompiler } from "../../scripts/quality/release-acceptance/staticAdapter.mjs";
test("empty diagnostics with nonempty digest and exit 0 is PASS", () => {
const out = adaptCompiler({
commandId: "tsc",
inputDigest: "a".repeat(64),
exitCode: 0,
diagnostics: [],
});
assert.equal(out.status, "PASS");
});
test("empty digest plus empty diagnostics is INFRA_ERROR", () => {
const out = adaptCompiler({
commandId: "tsc",
inputDigest: "",
exitCode: 0,
diagnostics: [],
});
assert.equal(out.status, "INFRA_ERROR");
});
test("exit 1 with diagnostics is FAIL", () => {
const out = adaptCompiler({
commandId: "tsc",
inputDigest: "a".repeat(64),
exitCode: 1,
diagnostics: ["error TS2304"],
});
assert.equal(out.status, "FAIL");
});