fix(quality): reject mismatched and contradictory acceptance evidence

docs/architecture/QUALITY_EVIDENCE_CONTRACT.md

@@ -0,0 +1,81 @@
+---
+title: "Quality Evidence Contract"
+version: 3.8.51
+lastUpdated: 2026-09-21
+---
+
+# Quality evidence contract
+
+## Scope and owners
+
+The release-acceptance report is a versioned interface, not an informal summary of
+green check marks. Its schema is
+[`config/quality/release-acceptance.schema.json`](../../config/quality/release-acceptance.schema.json).
+The repository owner and quality-gate maintainers own the schema and reducer. Gate
+maintainers own their command results. CI and the release captain consume those
+results; contributors do not own failures inherited from the base or runner.
+
+The producer adapters live in
+[`scripts/quality/release-acceptance/`](../../scripts/quality/release-acceptance/).
+[`validate-release-acceptance.mjs`](../../scripts/quality/validate-release-acceptance.mjs)
+loads a plan and manifests, reduces them, validates the report schema and writes the
+report. Its command-line interface accepts `--plan`, `--manifests` and `--out`.
+
+The current `release-acceptance.yml` workflow is a **shadow** integration using
+fixtures. A successful fixture run is not proof that the actual release, its tests,
+or its artifact passed. Enforcing real release admission requires real producers
+and a reviewed required-gate plan; this document does not claim that rollout is complete.
+
+## Version 1 semantics
+
+Each planned gate instance is identified by `gate_id`, `suite_id`, `shard_index`
+and `shard_total`. Every result must belong to the plan's exact `tested_sha`,
+`run_id` and `run_attempt`. A valid SHA-shaped string is insufficient if it names
+a different revision.
+
+| Observation                                                             | Interpretation                                                                      |
+| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
+| One matching required `PASS`, with exit code zero                       | Eligible evidence for that gate, subject to the rest of the report validation       |
+| Required `FAIL` with a positive exit code                               | Failed execution; causality still requires comparison with the exact base           |
+| Missing record, duplicate instance, unknown status or identity mismatch | Incomplete or conflicting evidence; never `VERIFIED`                                |
+| `PASS` with a nonzero or absent exit, or `FAIL` with exit zero          | Inconsistent producer result; treated as `INFRA_ERROR`                              |
+| Required `SKIPPED` or `INFRA_ERROR`                                     | `UNVERIFIED`, unless a separate valid required failure already establishes `FAILED` |
+| Empty required-gate set                                                 | `UNVERIFIED`                                                                        |
+
+The final verdict is `VERIFIED`, `FAILED` or `UNVERIFIED`. The CLI exits 0, 1 or 2,
+respectively. These values describe validation evidence, not contributor blame.
+A failure in an artifact prerequisite propagates to its dependents with a `cause`;
+infrastructure uncertainty must not become a fabricated test failure.
+
+## Freshness, compatibility and rollout
+
+A new candidate SHA requires new evidence. Results from another run attempt cannot
+be spliced into a current report merely because their gate names match. Historical
+reports remain historical: never rewrite their identities to make them current.
+Re-execute the affected gates when evidence must be refreshed.
+
+The identity and consistency checks tighten version 1's existing meaning without
+renaming fields. Producers that emitted mismatched identities, contradictory exits,
+or duplicate results now receive `UNVERIFIED` instead of a false verification. Fix
+the producer and replay the commands; do not remove the validation to retain green.
+
+Breaking field or meaning changes require an explicit schema version and a migration
+review with both producers and consumers. Additional fields also require review:
+version 1 rejects unknown properties. Deploy parser and producer changes together
+in shadow mode, exercise valid and invalid fixtures, then review activation of any
+required GitHub check separately.
+
+The schema and reducer are not a process supervisor. They do not by themselves
+provide heartbeat, descendant-process cleanup, artifact-byte verification or a
+complete production gate inventory. Those capabilities need their own execution
+and integration evidence before admission can be declared complete.
+
+## Regression evidence
+
+The reducer, schema and CLI are exercised by
+[`release-acceptance-reduce.test.ts`](../../tests/unit/release-acceptance-reduce.test.ts),
+[`release-acceptance-schema.test.ts`](../../tests/unit/release-acceptance-schema.test.ts)
+and [`release-acceptance-cli.test.ts`](../../tests/unit/release-acceptance-cli.test.ts).
+The identity regressions cover another SHA, run and attempt, duplicate PASS records,
+unknown execution status and contradictory exits. Existing prerequisite and shard
+tests remain part of the acceptance boundary.

scripts/quality/release-acceptance/reduce.mjs

@@ -1,4 +1,4 @@
-import { gateKey, sameKey } from "./types.mjs";
+import { gateKey, keyId, sameKey, STATUSES } from "./types.mjs";
 
 export function classifyDependent(prereqStatus, dependentKey, prereqKey) {
   if (prereqStatus === "FAIL") {
@@ -81,10 +81,7 @@ function statusOfGateId(gates, gateId) {
 function pushEvidenceError(evidence_errors, err) {
   if (!err) return;
   const already = evidence_errors.some(
-    (e) =>
-      e.code === err.code &&
-      e.detail === err.detail &&
-      e.gate?.gate_id === err.gate?.gate_id
+    (e) => e.code === err.code && e.detail === err.detail && e.gate?.gate_id === err.gate?.gate_id
   );
   if (!already) evidence_errors.push(err);
 }
@@ -163,17 +160,58 @@ export function reduce(plan, records) {
 
   const gates = [];
   const evidence_errors = [];
+  const identity = plan.identity ?? {};
+  const validIdentity =
+    /^[0-9a-f]{40}$/.test(identity.tested_sha ?? "") &&
+    typeof identity.run_id === "string" &&
+    identity.run_id.length > 0 &&
+    Number.isInteger(identity.run_attempt) &&
+    identity.run_attempt > 0;
+  const seen = new Set();
 
   for (const rec of records) {
     const k = gateKey(rec);
     const copy = { ...rec, cause: rec.cause ?? null };
+    const id = keyId(k);
+    if (seen.has(id)) {
+      pushEvidenceError(evidence_errors, {
+        code: "duplicate_record",
+        gate: k,
+        detail: `multiple records for ${JSON.stringify(k)}`,
+      });
+    }
+    seen.add(id);
+    let invalid;
+    if (
+      !validIdentity ||
+      ["tested_sha", "run_id", "run_attempt"].some((field) => copy[field] !== identity[field])
+    ) {
+      invalid = {
+        code: "identity_mismatch",
+        detail: "record does not belong to the planned SHA, run and attempt",
+      };
+    } else if (!STATUSES.includes(copy.status)) {
+      invalid = {
+        code: "invalid_status",
+        detail: "record has no recognized terminal execution status",
+      };
+    } else if (
+      (copy.status === "PASS" && copy.exit_code !== 0) ||
+      (copy.status === "FAIL" && (!Number.isInteger(copy.exit_code) || copy.exit_code <= 0))
+    ) {
+      invalid = { code: "exit_mismatch", detail: "execution exit and reported status disagree" };
+    }
+    if (invalid) {
+      pushEvidenceError(evidence_errors, { ...invalid, gate: k });
+      copy.status = "INFRA_ERROR";
+      copy.exit_code = 2;
+    }
     if (copy.status === "SKIPPED" && isRequired(plan, k) && !copy.reason) {
       copy.reason = "required skipped";
     }
     gates.push(copy);
   }
 
-  const identity = plan.identity ?? {};
   const edges = Object.entries(deps);
   let changed = true;
   let guard = edges.length + 1;
@@ -224,14 +262,13 @@ export function reduce(plan, records) {
 
   let verdict = "VERIFIED";
   const hasFail = gates.some(
-    (g) => g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL"
+    (g) =>
+      g.status === "FAIL" && isRequired(plan, gateKey(g)) && statusOf(gates, gateKey(g)) === "FAIL"
   );
   const hasUnverified =
     evidence_errors.length > 0 ||
     gates.some(
-      (g) =>
-        isRequired(plan, gateKey(g)) &&
-        (g.status === "SKIPPED" || g.status === "INFRA_ERROR")
+      (g) => isRequired(plan, gateKey(g)) && (g.status === "SKIPPED" || g.status === "INFRA_ERROR")
     );
   if (hasFail) verdict = "FAILED";
   else if (hasUnverified) verdict = "UNVERIFIED";

tests/unit/release-acceptance-pack-boot.test.ts

@@ -47,22 +47,22 @@ test("legacy computeVerdict still hard-fails pack-boot when pack-artifact times
 });
 
 test("new reducer maps the same timeout to UNVERIFIED", () => {
-  const out = reduce(planPack, [
-    record({ gate_id: "pack-artifact", status: "INFRA_ERROR" }),
-  ]);
+  const out = reduce(planPack, [record({ gate_id: "pack-artifact", status: "INFRA_ERROR" })]);
   assert.equal(out.verdict, "UNVERIFIED");
 });
 
-test("synthesized pack-boot without identity.tested_sha keeps a 40-hex sha and FAILED", () => {
+test("synthesized pack-boot without a tested identity remains UNVERIFIED", () => {
   const plan = {
     required_gates: planPack.required_gates,
     identity: { run_id: "1", run_attempt: 1 },
     dependencies: { "pack-boot": "pack-artifact" },
   };
-  const out = reduce(plan, [record({ gate_id: "pack-artifact", status: "FAIL" })]);
+  const out = reduce(plan, [record({ gate_id: "pack-artifact", status: "FAIL", exit_code: 1 })]);
   const boot = out.gates.find((g) => g.gate_id === "pack-boot");
   assert.ok(boot);
   assert.notEqual(boot.tested_sha, null);
   assert.match(String(boot.tested_sha), /^[0-9a-f]{40}$/);
-  assert.equal(out.verdict, "FAILED");
+  assert.equal(boot.status, "INFRA_ERROR");
+  assert.equal(out.verdict, "UNVERIFIED");
+  assert.ok(out.evidence_errors.some((error) => error.code === "identity_mismatch"));
 });

tests/unit/release-acceptance-reduce.test.ts

@@ -21,7 +21,7 @@ function record(partial) {
     gate_type: "static",
     status: "PASS",
     cause: null,
-    exit_code: 0,
+    exit_code: partial.status === "FAIL" ? 1 : partial.status === "INFRA_ERROR" ? 2 : 0,
     duration_ms: 10,
     evidence: [],
     ...partial,
@@ -49,6 +49,45 @@ test("required SKIPPED never yields VERIFIED", () => {
   assert.equal(out.verdict, "UNVERIFIED");
 });
 
+test("a complete matching required PASS is VERIFIED", () => {
+  assert.equal(reduce(planWithRequired("lint"), [record({})]).verdict, "VERIFIED");
+});
+
+for (const mismatch of [
+  { tested_sha: "a".repeat(40) },
+  { run_id: "older-run" },
+  { run_attempt: 2 },
+]) {
+  test(`PASS from a different validation identity is UNVERIFIED: ${JSON.stringify(mismatch)}`, () => {
+    const out = reduce(planWithRequired("lint"), [record(mismatch)]);
+    assert.equal(out.verdict, "UNVERIFIED");
+    assert.ok(out.evidence_errors.some((error) => error.code === "identity_mismatch"));
+  });
+}
+
+test("duplicate required PASS records cannot be counted as independent proof", () => {
+  const out = reduce(planWithRequired("lint"), [record({}), record({})]);
+  assert.equal(out.verdict, "UNVERIFIED");
+  assert.ok(out.evidence_errors.some((error) => error.code === "duplicate_record"));
+});
+
+for (const inconsistent of [
+  { status: "PASS", exit_code: 42 },
+  { status: "PASS", exit_code: null },
+  { status: "FAIL", exit_code: 0 },
+  { status: "PENDING", exit_code: 0 },
+]) {
+  test(`inconsistent or non-terminal command evidence is UNVERIFIED: ${JSON.stringify(inconsistent)}`, () => {
+    const out = reduce(planWithRequired("lint"), [record(inconsistent)]);
+    assert.equal(out.verdict, "UNVERIFIED");
+  });
+}
+
+test("missing plan identity cannot certify an otherwise passing record", () => {
+  const out = reduce(planWithRequired("lint", { identity: {} }), [record({})]);
+  assert.equal(out.verdict, "UNVERIFIED");
+});
+
 test("pack-artifact FAIL classifies pack-boot as FAIL with cause", () => {
   const out = reduce(planPack, [
     record({ gate_id: "pack-artifact", status: "FAIL", gate_type: "artifact" }),
@@ -188,7 +227,11 @@ test("cyclic dependencies are rejected", () => {
     dependencies: { a: "b", b: "a" },
   };
   assert.throws(
-    () => reduce(cyclic, [record({ gate_id: "a", status: "INFRA_ERROR" }), record({ gate_id: "b", status: "FAIL" })]),
+    () =>
+      reduce(cyclic, [
+        record({ gate_id: "a", status: "INFRA_ERROR" }),
+        record({ gate_id: "b", status: "FAIL" }),
+      ]),
     /cyclic prerequisite/
   );
 });
@@ -203,10 +246,7 @@ test("missing prerequisite records one evidence error, not one per loop", () =>
     []
   );
   assert.equal(out.verdict, "UNVERIFIED");
-  assert.equal(
-    out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length,
-    1
-  );
+  assert.equal(out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length, 1);
 });
 
 test("missing prerequisite records one evidence error for all shards of a gate_id", () => {
@@ -221,10 +261,7 @@ test("missing prerequisite records one evidence error for all shards of a gate_i
     []
   );
   assert.equal(out.verdict, "UNVERIFIED");
-  assert.equal(
-    out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length,
-    1
-  );
+  assert.equal(out.evidence_errors.filter((e) => e.code === "prerequisite_missing").length, 1);
 });
 
 test("two dependents of the same missing prerequisite keep one error per edge", () => {