From 4505e67c04f4fde41855f0a317969baa08cf0d6d Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 14 Jul 2026 07:07:27 -0300 Subject: [PATCH] feat(ci): duration-balanced E2E shards via LPT bin-packing (WS4.1) (#7090) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Playwright --shard distributes by count (per file with fullyParallel:false), blind to duration — measured skew on the 9-shard matrix: 24m47s worst vs 1m47s best (14x), putting E2E on the CI critical path (~25min of the 33min gate). - scripts/quality/balance-e2e-shards.mjs: LPT greedy (heaviest first into the lightest shard) over config/quality/e2e-timings.json; deterministic (weight desc, filename tiebreak); new specs get the median weight; the CLI self-verifies the shard union equals the discovered spec list and exits non-zero on ANY inconsistency (missing timings, lost spec) so the CI step falls back to plain --shard — never fewer specs than before. - config/quality/e2e-timings.json: relative weights seeded from spec LOC (proxy); replace with real per-file durations from a full run when convenient (documented in _meta). LOC-seeded packing already lands at 742-761 per shard (1.03x skew) vs the alphabetical round-robin that produced 14x. - ci.yml test-e2e: balanced list per shard with logged assignment + fallback. TDD: 5 unit tests (LPT invariants, determinism, completeness, median fallback, seed-vs-specs drift guard). --- .github/workflows/ci.yml | 18 +++- changelog.d/maintenance/e2e-shard-balance.md | 1 + config/quality/e2e-timings.json | 39 ++++++++ scripts/quality/balance-e2e-shards.mjs | 99 ++++++++++++++++++++ tests/unit/balance-e2e-shards.test.ts | 75 +++++++++++++++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 changelog.d/maintenance/e2e-shard-balance.md create mode 100644 config/quality/e2e-timings.json create mode 100644 scripts/quality/balance-e2e-shards.mjs create mode 100644 tests/unit/balance-e2e-shards.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e99af27cb..de620de31e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1000,7 +1000,23 @@ jobs: - name: Extract Next.js build artifact run: | tar -xzf /tmp/e2e-build.tar.gz - - run: npx playwright test tests/e2e/*.spec.ts --shard=${{ matrix.shard }}/9 + # WS4.1: duration-balanced shards (LPT over config/quality/e2e-timings.json). + # Measured skew of plain --shard was 14× (24m47s vs 1m47s) — E2E was the CI + # critical path. The balancer self-verifies completeness and exits non-zero on + # any inconsistency, falling back to plain --shard (never fewer specs). + - name: Run E2E tests (duration-balanced shard) + env: + SHARD: ${{ matrix.shard }} + run: | + if FILES=$(node scripts/quality/balance-e2e-shards.mjs "$SHARD" 9); then + if [ -z "$FILES" ]; then echo "[e2e-balance] shard $SHARD has no files"; exit 0; fi + echo "[e2e-balance] shard $SHARD runs:"; echo "$FILES" + # shellcheck disable=SC2086 — FILES is our own newline-separated path list + npx playwright test $(echo "$FILES" | tr '\n' ' ') + else + echo "[e2e-balance] balancer unavailable — plain --shard fallback" + npx playwright test tests/e2e/*.spec.ts --shard="$SHARD"/9 + fi test-integration: name: Integration Tests (${{ matrix.shard }}/2) diff --git a/changelog.d/maintenance/e2e-shard-balance.md b/changelog.d/maintenance/e2e-shard-balance.md new file mode 100644 index 0000000000..a950f1ba44 --- /dev/null +++ b/changelog.d/maintenance/e2e-shard-balance.md @@ -0,0 +1 @@ +- **CI**: E2E matrix shards are now duration-balanced (LPT bin-packing over `config/quality/e2e-timings.json`) instead of Playwright's count-based `--shard` — measured skew was 14× (24m47s vs 1m47s), making E2E the CI critical path; the balancer self-verifies that every spec lands in exactly one shard and falls back to plain `--shard` on any inconsistency diff --git a/config/quality/e2e-timings.json b/config/quality/e2e-timings.json new file mode 100644 index 0000000000..77571fcbf9 --- /dev/null +++ b/config/quality/e2e-timings.json @@ -0,0 +1,39 @@ +{ + "_meta": "Relative weights for scripts/quality/balance-e2e-shards.mjs (LPT shard packing). Unitless \u2014 only ratios matter. Seeded from spec LOC (proxy) on 2026-07-13; replace values with real per-file durations (seconds) from a full CI run's reports whenever convenient. New specs without an entry get the median weight.", + "a11y-resilience.spec.ts": 59, + "a11y.spec.ts": 281, + "agent-bridge-traffic-cross.spec.ts": 155, + "agent-bridge.spec.ts": 160, + "agent-skills-page.spec.ts": 205, + "analytics-tabs.spec.ts": 334, + "api-keys-flow.spec.ts": 643, + "api.spec.ts": 30, + "combo-live-studio.spec.ts": 35, + "combo-unification.spec.ts": 192, + "combos-flow.spec.ts": 629, + "compression-studio.spec.ts": 55, + "error-pages.spec.ts": 101, + "group-b-activity-feed.spec.ts": 96, + "group-b-quota-plans-config.spec.ts": 154, + "group-b-quota-share-pools.spec.ts": 98, + "group-b-redirect-logs-activity.spec.ts": 56, + "memory-engine.spec.ts": 607, + "memory-qdrant-routes.spec.ts": 360, + "memory-settings.spec.ts": 193, + "navigation.spec.ts": 28, + "playground-compare.spec.ts": 138, + "playground-studio.spec.ts": 101, + "protocol-visibility.spec.ts": 25, + "providers-bailian-coding-plan.spec.ts": 240, + "providers-management.spec.ts": 324, + "proxy-registry.smoke.spec.ts": 218, + "resilience-plan-alignment.spec.ts": 382, + "responsive.spec.ts": 21, + "search-tools-studio.spec.ts": 133, + "settings-toggles.spec.ts": 127, + "skills-marketplace.spec.ts": 209, + "smoke.spec.ts": 33, + "traffic-inspector.spec.ts": 212, + "translator-friendly.spec.ts": 115, + "visual-resilience-smoke.spec.ts": 20 +} \ No newline at end of file diff --git a/scripts/quality/balance-e2e-shards.mjs b/scripts/quality/balance-e2e-shards.mjs new file mode 100644 index 0000000000..450a5438bb --- /dev/null +++ b/scripts/quality/balance-e2e-shards.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node +/** + * balance-e2e-shards — duration-aware LPT bin-packing for the Playwright matrix (WS4.1). + * + * Playwright's --shard=N/M distributes by COUNT (per file with fullyParallel:false), + * blind to duration — measured skew on the 9-shard matrix: worst 24m47s vs best 1m47s + * (14×), putting E2E on the CI critical path. This script assigns spec FILES to shards + * by weight (Longest Processing Time greedy: heaviest first, always into the lightest + * shard) using config/quality/e2e-timings.json, and prints shard N's files (one per + * line) for `npx playwright test $FILES`. + * + * Safety: the union of all shards is verified to equal the discovered spec list — + * losing a spec silently would hollow the suite. Any inconsistency (or a missing + * timings file) exits non-zero so the CI step falls back to plain --shard=N/M. + * + * Weights are RELATIVE (unitless). The seed uses LOC as a proxy; regenerate from real + * durations by editing config/quality/e2e-timings.json (see its _meta note). + */ +import fs from "node:fs"; +import path from "node:path"; + +const E2E_DIR = path.join("tests", "e2e"); +const TIMINGS_PATH = path.join("config", "quality", "e2e-timings.json"); + +/** + * LPT greedy assignment. Deterministic: weight desc, then filename asc; ties on + * shard totals resolve to the lowest shard index. + * @param {{file: string, weight: number}[]} items + * @param {number} shardCount + * @returns {{files: string[], total: number}[]} + */ +export function lptAssign(items, shardCount) { + const shards = Array.from({ length: shardCount }, () => ({ files: [], total: 0 })); + const sorted = [...items].sort( + (a, b) => b.weight - a.weight || a.file.localeCompare(b.file) + ); + for (const item of sorted) { + let target = shards[0]; + for (const s of shards) if (s.total < target.total) target = s; + target.files.push(item.file); + target.total += item.weight; + } + return shards; +} + +/** + * Weight lookup with a median fallback so a NEW spec (no timing yet) lands mid-pack + * instead of skewing a shard. + * @param {string[]} files basenames + * @param {Record} timings + */ +export function weightItems(files, timings) { + const known = Object.entries(timings) + .filter(([k]) => !k.startsWith("_")) + .map(([, v]) => v) + .filter((v) => Number.isFinite(v) && v > 0) + .sort((a, b) => a - b); + const median = known.length ? known[Math.floor(known.length / 2)] : 1; + return files.map((file) => ({ + file, + weight: Number.isFinite(timings[file]) && timings[file] > 0 ? timings[file] : median, + })); +} + +function main() { + const shard = Number(process.argv[2]); + const total = Number(process.argv[3]); + if (!Number.isInteger(shard) || !Number.isInteger(total) || shard < 1 || shard > total) { + console.error("usage: node scripts/quality/balance-e2e-shards.mjs "); + process.exit(2); + } + if (!fs.existsSync(TIMINGS_PATH)) { + console.error(`[e2e-balance] ${TIMINGS_PATH} missing — caller should fall back to --shard`); + process.exit(3); + } + const timings = JSON.parse(fs.readFileSync(TIMINGS_PATH, "utf8")); + const files = fs + .readdirSync(E2E_DIR) + .filter((f) => f.endsWith(".spec.ts")) + .sort(); + if (!files.length) { + console.error(`[e2e-balance] no specs found under ${E2E_DIR}`); + process.exit(3); + } + const shards = lptAssign(weightItems(files, timings), total); + const assigned = shards.flatMap((s) => s.files).sort(); + if (assigned.length !== files.length || assigned.some((f, i) => f !== files[i])) { + console.error("[e2e-balance] INTERNAL: shard union != discovered specs — falling back"); + process.exit(3); + } + process.stdout.write( + shards[shard - 1].files.map((f) => path.join(E2E_DIR, f)).join("\n") + "\n" + ); +} + +const isDirectRun = + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname); +if (isDirectRun) main(); diff --git a/tests/unit/balance-e2e-shards.test.ts b/tests/unit/balance-e2e-shards.test.ts new file mode 100644 index 0000000000..6484dfccb2 --- /dev/null +++ b/tests/unit/balance-e2e-shards.test.ts @@ -0,0 +1,75 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { lptAssign, weightItems } from "../../scripts/quality/balance-e2e-shards.mjs"; + +// WS4.1 (v3.8.49 quality plan) — the E2E matrix skew was 14× (24m47s vs 1m47s) +// because Playwright --shard distributes by count, not duration. These tests pin +// the LPT packing invariants; the hard one is COMPLETENESS (a lost spec would +// silently hollow the suite — the CLI self-checks it and falls back to --shard). + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +test("lptAssign puts the heaviest item alone before doubling up lighter shards", () => { + const shards = lptAssign( + [ + { file: "huge.spec.ts", weight: 100 }, + { file: "a.spec.ts", weight: 30 }, + { file: "b.spec.ts", weight: 30 }, + { file: "c.spec.ts", weight: 30 }, + ], + 2 + ); + assert.deepEqual(shards[0].files, ["huge.spec.ts"]); + assert.deepEqual(shards[1].files, ["a.spec.ts", "b.spec.ts", "c.spec.ts"]); + assert.equal(shards[0].total, 100); + assert.equal(shards[1].total, 90); +}); + +test("lptAssign is deterministic on equal weights (filename tiebreak)", () => { + const items = [ + { file: "b.spec.ts", weight: 10 }, + { file: "a.spec.ts", weight: 10 }, + ]; + const s1 = lptAssign(items, 2); + const s2 = lptAssign([...items].reverse(), 2); + assert.deepEqual(s1, s2); +}); + +test("completeness: every file lands in exactly one shard", () => { + const items = Array.from({ length: 37 }, (_, i) => ({ + file: `f${String(i).padStart(2, "0")}.spec.ts`, + weight: (i * 7) % 40, + })); + const shards = lptAssign(items, 9); + const union = shards.flatMap((s) => s.files).sort(); + assert.deepEqual(union, items.map((i) => i.file).sort()); +}); + +test("weightItems gives unknown/new specs the median weight, not an extreme", () => { + const items = weightItems(["new.spec.ts", "big.spec.ts", "small.spec.ts"], { + _meta: "x", + "big.spec.ts": 600, + "small.spec.ts": 20, + "other.spec.ts": 100, + }); + const byFile = Object.fromEntries(items.map((i) => [i.file, i.weight])); + assert.equal(byFile["big.spec.ts"], 600); + assert.equal(byFile["small.spec.ts"], 20); + assert.equal(byFile["new.spec.ts"], 100); // median of [20,100,600] +}); + +test("the committed timings seed covers every current e2e spec (no drift)", () => { + const timings = JSON.parse( + fs.readFileSync(path.join(ROOT, "config", "quality", "e2e-timings.json"), "utf8") + ); + const specs = fs + .readdirSync(path.join(ROOT, "tests", "e2e")) + .filter((f) => f.endsWith(".spec.ts")); + const missing = specs.filter((f) => !(f in timings)); + // Missing entries are tolerated at runtime (median fallback) — this assert keeps + // the seed honest so balance quality does not silently rot as specs are added. + assert.deepEqual(missing, [], `add to config/quality/e2e-timings.json: ${missing.join(", ")}`); +});