From 240f466f54b95aaeb52c1e7155a9ff50e873d95f Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:46:28 -0300 Subject: [PATCH] =?UTF-8?q?test(infra):=20isolate=20DATA=5FDIR=20per=20tes?= =?UTF-8?q?t=20process;=20raise=20Stryker=20concurrency=201=E2=86=924=20(#?= =?UTF-8?q?4078)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(infra): isolate DATA_DIR per test process; raise Stryker concurrency 1→4 Every test process resolved DATA_DIR to the same default (~/.omniroute) when the env var was unset (src/lib/dataPaths.ts::resolveDataDir), so concurrent test files opened the SAME on-disk storage.sqlite. node:test spawns a process per file and Stryker spawns one per sandbox, so this shared file caused cross-file state races: - SQLite lock contention that hung `npm run test:unit` under high --test-concurrency (the ~95-min local hang), and - the non-deterministic baseline that forced stryker.conf.json to concurrency: 1, which in turn could not finish the ~15k-mutant run inside the nightly timeout (the cancelled 2026-06-16/17 nightly-mutation runs) — blocking Quality Gate v2 / Fase 9 Onda 2. open-sse/utils/setupPolyfill.ts could NOT host the fix: it is imported by production (bin/omniroute.mjs, proxyFetch.ts, proxyDispatcher.ts), where redirecting DATA_DIR would point the live SQLite DB at a throwaway temp dir. So this adds a TEST-ONLY tests/_setup/isolateDataDir.ts that gives each process its own temp DATA_DIR when none is set (tests that set DATA_DIR explicitly still win), wired via --import into the test, mutation and CI invocations. Verified: - Stryker dry-run A/B at concurrency=4: FAILS without the isolation import (account-fallback-service tap exit 9, a cross-file race) and PASSES with it. - Full `npm run test:unit` green with isolation (0 fail; a one-off chatcore-translation-paths timeout flake did not reproduce and passes 3/3 isolated) and noticeably faster — the DB lock contention is gone. - New tests/unit/isolate-datadir.test.ts guards the contract (unique temp DATA_DIR when unset; explicit DATA_DIR respected). Wired the --import into: package.json (13 test scripts), stryker.conf.json (tap.nodeArgs + concurrency 1→4), .github/workflows/quality.yml (TIA step), ci.yml (the 5 unit/coverage/integration commands), and bumped nightly-mutation.yml timeout 120→180 for the first cold run before the incremental cache is seeded. * ci(quality): run the TIA gate at CI concurrency (4) to stop oversubscription flakes The TIA "Impacted unit tests" step (made blocking in #4069) ran its fail-safe via `npm run test:unit` — concurrency=20, tuned for multi-core dev machines. On a 4-vCPU CI runner that is 5x oversubscribed, so timing-sensitive tests flake under the load (e.g. `db-backup-extended` "The database connection is not open", `chatcore-translation-paths` upstream-timeout). That intermittently fails a blocking gate on legitimate PRs — exactly what surfaced on the DATA_DIR-isolation PR, whose package.json/workflow changes trip the __RUN_ALL__ fail-safe. Run both the impacted set and the fail-safe at --test-concurrency=4, matching the stable ci.yml unit job. Adds a `test:unit:ci` script (test:unit at concurrency=4). The DATA_DIR isolation in this PR keeps the parallel run race-free, so the only change here is matching the runner's core count. Verified locally: db-backup-extended passes 8/8 in isolation (5 with isolation, 3 without). --- .github/workflows/ci.yml | 10 +++--- .github/workflows/nightly-mutation.yml | 9 ++++-- .github/workflows/quality.yml | 9 ++++-- package.json | 27 ++++++++-------- stryker.conf.json | 6 ++-- tests/_setup/isolateDataDir.ts | 36 +++++++++++++++++++++ tests/unit/isolate-datadir.test.ts | 45 ++++++++++++++++++++++++++ 7 files changed, 118 insertions(+), 24 deletions(-) create mode 100644 tests/_setup/isolateDataDir.ts create mode 100644 tests/unit/isolate-datadir.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06bb391e4..2237f8bb67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -513,7 +513,7 @@ jobs: cache: npm - run: npm ci - run: npm run check:node-runtime - - run: node --max-old-space-size=4096 --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --max-old-space-size=4096 --import tsx --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" test-vitest: name: Vitest (MCP / autoCombo / UI components) @@ -566,7 +566,7 @@ jobs: - run: npm ci - run: npm run check:node-runtime - run: npm run build - - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --import tsx --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" node-26-compat: name: Node 26 Compatibility (${{ matrix.shard }}/2) @@ -592,7 +592,7 @@ jobs: - run: npm ci - run: npm run check:node-runtime - run: npm run build - - run: node --import tsx --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" + - run: node --import tsx --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 --test-shard=${{ matrix.shard }}/2 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" test-coverage-shard: name: Coverage Shard (${{ matrix.shard }}/8) @@ -632,7 +632,7 @@ jobs: --reporter=json \ --exclude=tests/** \ --exclude=**/*.test.* \ - node --max-old-space-size=4096 --import tsx --test --test-force-exit --test-concurrency=4 \ + node --max-old-space-size=4096 --import tsx --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 \ --test-shard=${{ matrix.shard }}/8 tests/unit/*.test.ts "tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts" - name: Upload raw shard coverage if: always() @@ -902,7 +902,7 @@ jobs: cache: npm - run: npm ci - run: npm run check:node-runtime - - run: node --import tsx --test --test-force-exit --test-concurrency=1 --test-shard=${{ matrix.shard }}/2 tests/integration/*.test.ts + - run: node --import tsx --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 --test-shard=${{ matrix.shard }}/2 tests/integration/*.test.ts test-security: name: Security Tests diff --git a/.github/workflows/nightly-mutation.yml b/.github/workflows/nightly-mutation.yml index 6e36037f1d..789144fbc0 100644 --- a/.github/workflows/nightly-mutation.yml +++ b/.github/workflows/nightly-mutation.yml @@ -11,11 +11,16 @@ jobs: stryker: name: Stryker mutation testing (8 critical modules — advisory) runs-on: ubuntu-latest - # Mutation testing is expensive (~200-500 mutants, 30-90 min). It runs only + # Mutation testing is expensive (~15k mutants across 8 modules). It runs only # on the nightly schedule / manual dispatch, never on PRs. The score is NOT # yet enforced as a ratchet (wired in a later INT phase) — for now the job # just produces the HTML/JSON report and uploads it as an artifact. - timeout-minutes: 120 + # + # Runs at stryker concurrency=4 (was 1) now that ./tests/_setup/isolateDataDir.ts + # gives each test process its own DATA_DIR — at concurrency=1 the full phase did + # not finish inside the timeout (the cancelled 2026-06-16/17 runs). 180 min leaves + # headroom for the first cold run before the incremental cache is seeded. + timeout-minutes: 180 steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index eab3575822..61015b27fd 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -71,9 +71,14 @@ jobs: node scripts/quality/build-test-impact-map.mjs SEL="$(node scripts/quality/select-impacted-tests.mjs)" if [ -z "$SEL" ]; then echo "No source/test changes — skipping unit tests"; exit 0; fi + # CI runners are 4-vCPU; run at --test-concurrency=4 (matching the ci.yml unit + # job) rather than test:unit's local-tuned concurrency=20. Oversubscribing the + # runner makes timing-sensitive tests (db-backup, upstream-timeout, ...) flake, + # which must not happen on a blocking gate. DATA_DIR isolation keeps the parallel + # run race-free regardless of concurrency. if echo "$SEL" | grep -q "__RUN_ALL__"; then - echo "Fail-safe: running FULL unit suite"; npm run test:unit; exit $? + echo "Fail-safe: running FULL unit suite (CI concurrency)"; npm run test:unit:ci; exit $? fi echo "Running impacted tests:"; echo "$SEL" mapfile -t FILES <<< "$SEL" - node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=8 "${FILES[@]}" + node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 "${FILES[@]}" diff --git a/package.json b/package.json index 69e24e4bc4..fb07ec75af 100644 --- a/package.json +++ b/package.json @@ -86,15 +86,16 @@ "electron:build:mac": "npm run build && cd electron && npm run build:mac", "electron:build:linux": "npm run build && cd electron && npm run build:linux", "electron:smoke:packaged": "node scripts/dev/smoke-electron-packaged.mjs", - "test": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit:fast": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-isolation=none tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=20 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:ci": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=4 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:fast": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-isolation=none tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", "test:unit:shard": "concurrently --kill-others-on-fail -n s1,s2 \"npm:test:unit:shard:1\" \"npm:test:unit:shard:2\"", - "test:unit:shard:1": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:unit:shard:2": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", - "test:plan3": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/plan3-p0.test.ts", - "test:fixes": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/fixes-p1.test.ts", - "test:security": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test tests/unit/security-fase01.test.ts", + "test:unit:shard:1": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:unit:shard:2": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:plan3": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/plan3-p0.test.ts", + "test:fixes": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/fixes-p1.test.ts", + "test:security": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/security-fase01.test.ts", "test:property": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --test --test-force-exit tests/unit/correctness/*.property.test.ts", "check:cycles": "node scripts/check/check-cycles.mjs", "check:route-validation:t06": "node scripts/check/check-route-validation.mjs", @@ -159,17 +160,17 @@ "typecheck:noimplicit:core": "tsc --pretty false -p tsconfig.typecheck-noimplicit-core.json", "backfill-aggregation": "node --import tsx src/scripts/backfillAggregation.ts", "env:sync": "node scripts/dev/sync-env.mjs", - "test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts", - "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit tests/integration/heap-growth.test.ts", - "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts", + "test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts", + "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit tests/integration/heap-growth.test.ts", + "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts", "test:e2e": "node scripts/dev/run-playwright-tests.mjs test tests/e2e/*.spec.ts", "test:protocols:e2e": "node scripts/dev/run-protocol-clients-tests.mjs", "test:vitest": "vitest run --config vitest.mcp.config.ts", "test:vitest:ui": "vitest run --config vitest.config.ts tests/unit/ui", "test:mutation": "stryker run", "test:ecosystem": "node scripts/dev/run-ecosystem-tests.mjs", - "test:system": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/e2e/system-failover.test.ts", - "test:coverage": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true c8 --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", + "test:system": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 tests/e2e/system-failover.test.ts", + "test:coverage": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true c8 --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,compression,correctness,cors,dashboard,db,db-adapters,docs,gamification,guardrails,lib,mcp,runtime,security,services,settings,shared,ui}/**/*.test.ts\"", "test:coverage:legacy": "c8 --output-dir coverage --exclude=open-sse --check-coverage --lines 50 --functions 50 --branches 50 node --import tsx --test tests/unit/*.test.ts", "coverage:report": "c8 report --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov", "coverage:summary": "node scripts/check/test-report-summary.mjs --input coverage/coverage-summary.json --output coverage/coverage-report.md", diff --git a/stryker.conf.json b/stryker.conf.json index ef0624be91..d31ba7b6e3 100644 --- a/stryker.conf.json +++ b/stryker.conf.json @@ -177,6 +177,8 @@ "tsx", "--import", "./open-sse/utils/setupPolyfill.ts", + "--import", + "./tests/_setup/isolateDataDir.ts", "--test-reporter=tap", "-r", "{{hookFile}}", @@ -223,7 +225,7 @@ "coverageAnalysis": "perTest", "timeoutMS": 60000, "timeoutFactor": 2.5, - "concurrency": 1, + "concurrency": 4, "disableTypeChecks": true, "checkers": [], "thresholds": { @@ -246,5 +248,5 @@ " grep -rlE \"circuitBreaker|publicCreds|accountFallback|routeGuard|services/auth|chatCore|services/combo|utils/error|public-client|account-fallback|route-guard|circuit-breaker\" tests/unit --include=\"*.test.ts\" | sort -u" ], "dryRunTimeoutMinutes": 30, - "_concurrency_comment": "concurrency=1: the covering node:test files share SQLite/module state; running multiple files concurrently in the Stryker sandbox causes cross-file state races that fail the all-green baseline dry-run. Single-runner trades speed for a deterministic baseline (acceptable for a nightly)." + "_concurrency_comment": "concurrency=4 (was 1): the covering node:test files used to share SQLite/module state via the default DATA_DIR (~/.omniroute), so running them concurrently in the Stryker sandbox caused cross-file races that failed the all-green baseline. tap.nodeArgs now imports ./tests/_setup/isolateDataDir.ts, which gives each spawned test process its own temp DATA_DIR — eliminating the shared on-disk DB, so concurrency>1 is deterministic. A/B verified 2026-06-17: dry-run at concurrency=4 fails WITHOUT the isolation import (account-fallback-service tap exit 9) and passes WITH it. Raise further only if the runner has spare cores." } diff --git a/tests/_setup/isolateDataDir.ts b/tests/_setup/isolateDataDir.ts new file mode 100644 index 0000000000..f13db04863 --- /dev/null +++ b/tests/_setup/isolateDataDir.ts @@ -0,0 +1,36 @@ +// Test-only DATA_DIR isolation. +// +// Loaded via `node --import ./tests/_setup/isolateDataDir.ts` from the test/mutation +// invocations (package.json test scripts, stryker.conf.json tap.nodeArgs, the +// quality.yml TIA step, and the CI test jobs) — NEVER from production. It MUST stay +// out of open-sse/utils/setupPolyfill.ts, which is also imported by production +// (bin/omniroute.mjs, proxyFetch.ts, proxyDispatcher.ts) where redirecting DATA_DIR +// would point the live SQLite DB at a throwaway temp dir. +// +// Why: node:test spawns a process per test file and Stryker spawns one per sandbox, +// but every process resolves DATA_DIR to the SAME default (~/.omniroute) when the env +// var is unset (see src/lib/dataPaths.ts::resolveDataDir). Concurrent processes then +// open the SAME on-disk storage.sqlite, causing cross-file state races: SQLite lock +// contention that hangs `test:unit` under high `--test-concurrency`, and the +// non-deterministic baseline that forced Stryker to `concurrency: 1`. +// +// Giving each process its own DATA_DIR under the OS temp dir removes the shared file, +// so concurrent test processes never collide. Tests that set DATA_DIR explicitly keep +// winning — this only fills in an isolated default when none was chosen. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +if (!process.env.DATA_DIR) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-test-")); + process.env.DATA_DIR = dir; + + // Best-effort cleanup so a long suite run does not leak hundreds of temp DBs. + process.on("exit", () => { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // ignore — the OS reaps its temp dir eventually. + } + }); +} diff --git a/tests/unit/isolate-datadir.test.ts b/tests/unit/isolate-datadir.test.ts new file mode 100644 index 0000000000..2f14870b80 --- /dev/null +++ b/tests/unit/isolate-datadir.test.ts @@ -0,0 +1,45 @@ +// Regression guard for tests/_setup/isolateDataDir.ts — the test-only module that +// gives each test process its own DATA_DIR so concurrent test files never share the +// on-disk SQLite DB. Removing or breaking it brings back the cross-file state races +// (the `test:unit` hang under high concurrency and the non-deterministic Stryker +// baseline that forced concurrency: 1). +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import os from "node:os"; + +function dataDirFromChild(envDataDir: string | undefined): string { + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + "--import", + "./tests/_setup/isolateDataDir.ts", + "-e", + "console.log(process.env.DATA_DIR ?? '')", + ], + { + encoding: "utf8", + cwd: process.cwd(), + // Pass DATA_DIR through verbatim; an empty string means "unset" for the module's + // `if (!process.env.DATA_DIR)` guard. + env: { ...process.env, DATA_DIR: envDataDir ?? "" }, + } + ); + return result.stdout.trim().split("\n").pop() ?? ""; +} + +test("isolateDataDir assigns a unique temp DATA_DIR when none is set", () => { + const a = dataDirFromChild(undefined); + const b = dataDirFromChild(undefined); + + assert.ok(a.startsWith(os.tmpdir()), `expected a temp dir under ${os.tmpdir()}, got ${a}`); + assert.match(a, /omniroute-test-/, `expected the omniroute-test- prefix, got ${a}`); + assert.notEqual(a, b, "two processes must each get their own DATA_DIR"); +}); + +test("isolateDataDir respects an explicitly set DATA_DIR", () => { + const explicit = "/tmp/omniroute-explicit-fixture"; + assert.equal(dataDirFromChild(explicit), explicit); +});