diff --git a/.env.example b/.env.example index 5da7e84564..e9b64d0ee6 100644 --- a/.env.example +++ b/.env.example @@ -1013,6 +1013,13 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # to disable the check. Used by: src/lib/db/migrationRunner.ts. Default: 50. #OMNIROUTE_MAX_PENDING_MIGRATIONS=50 +# Working directory for the check:install-upgrade release gate. It builds two ~3 GB +# install trees plus a ~275 MB tarball, so it needs roughly 12 GB — more than the +# 12 GB RAM-backed tmpfs that /tmp is on the self-hosted runner, where it exhausted +# the tmpfs and npm silently truncated the package. Defaults to /.install-upgrade +# on real disk. Used by: scripts/check/check-install-upgrade.mjs. Default: /.install-upgrade. +#OMNIROUTE_INSTALL_UPGRADE_WORKDIR=/var/tmp/omniroute-install-upgrade + # Trust user-managed RTK project filter rules without strict signature checks. # Used by: open-sse/services/compression/engines/rtk/filterLoader.ts. Default: 0. #OMNIROUTE_RTK_TRUST_PROJECT_FILTERS=0 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a3caead30c..37578d6e4f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -195,6 +195,13 @@ jobs: - name: Build and push BUN base platform image by digest id: build-bun-base + # Bun is a best-effort compatibility target, not a supported runtime + # (AGENTS.md -> Environment). Its `bun run build` has been OOM-killing on + # both arches; letting that sink the whole publish means the SUPPORTED + # runner-base / runner-web images never reach the registry either. The + # image is still built and pushed whenever it succeeds — only its power to + # block the release is removed. + continue-on-error: true uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: . @@ -213,6 +220,13 @@ jobs: - name: Build and push BUN web platform image by digest id: build-bun-web + # Bun is a best-effort compatibility target, not a supported runtime + # (AGENTS.md -> Environment). Its `bun run build` has been OOM-killing on + # both arches; letting that sink the whole publish means the SUPPORTED + # runner-base / runner-web images never reach the registry either. The + # image is still built and pushed whenever it succeeds — only its power to + # block the release is removed. + continue-on-error: true uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 with: context: . @@ -240,8 +254,15 @@ jobs: mkdir -p /tmp/digests/base /tmp/digests/web /tmp/digests/bun-base /tmp/digests/bun-web touch "/tmp/digests/base/${DIGEST_BASE#sha256:}" touch "/tmp/digests/web/${DIGEST_WEB#sha256:}" - touch "/tmp/digests/bun-base/${DIGEST_BUN_BASE#sha256:}" - touch "/tmp/digests/bun-web/${DIGEST_BUN_WEB#sha256:}" + # Empty when the (non-blocking) bun build produced no image. `if` blocks, + # not `[ -n ] && touch`: under `set -e` a failing AND-list aborts the step, + # which is precisely the case being handled here. + if [ -n "$DIGEST_BUN_BASE" ]; then + touch "/tmp/digests/bun-base/${DIGEST_BUN_BASE#sha256:}" + fi + if [ -n "$DIGEST_BUN_WEB" ]; then + touch "/tmp/digests/bun-web/${DIGEST_BUN_WEB#sha256:}" + fi - name: Upload base digests uses: actions/upload-artifact@v7 @@ -264,7 +285,11 @@ jobs: with: name: digests-bun-base-${{ matrix.arch }} path: /tmp/digests/bun-base/* - if-no-files-found: error + # `ignore`, not `error`: the bun build is non-blocking, so an absent + # digest is the expected outcome of a failed/skipped bun image — the + # manifest step already treats these tags as optional. Leaving `error` + # here just relocates the blocker from the manifest to the upload. + if-no-files-found: ignore retention-days: 1 - name: Upload bun-web digests @@ -272,7 +297,11 @@ jobs: with: name: digests-bun-web-${{ matrix.arch }} path: /tmp/digests/bun-web/* - if-no-files-found: error + # `ignore`, not `error`: the bun build is non-blocking, so an absent + # digest is the expected outcome of a failed/skipped bun image — the + # manifest step already treats these tags as optional. Leaving `error` + # here just relocates the blocker from the manifest to the upload. + if-no-files-found: ignore retention-days: 1 merge: @@ -330,6 +359,9 @@ jobs: merge-multiple: true - name: Download bun-base digests + # Non-blocking: the bun image is best-effort, so its artifact may not + # exist at all. The manifest step treats these tags as optional. + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: digests-bun-base-* @@ -337,6 +369,9 @@ jobs: merge-multiple: true - name: Download bun-web digests + # Non-blocking: the bun image is best-effort, so its artifact may not + # exist at all. The manifest step treats these tags as optional. + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: digests-bun-web-* @@ -348,7 +383,7 @@ jobs: set -euo pipefail create_manifest() { - local image="$1" suffix="$2" dir="$3" + local image="$1" suffix="$2" dir="$3" optional="${4:-}" local tags=(-t "${image}:${VERSION}${suffix}") if [ "$PROMOTE_LATEST" = "true" ]; then tags+=(-t "${image}:latest${suffix}") @@ -358,6 +393,10 @@ jobs: refs+=("${image}@sha256:$(basename "$digest_file")") done < <(find "$dir" -type f | sort) if [ "${#refs[@]}" -eq 0 ]; then + if [ -n "$optional" ]; then + echo "::warning::No image digests in $dir — skipping optional tag ${image}:${VERSION}${suffix}" >&2 + return 0 + fi echo "No image digests in $dir" >&2 exit 1 fi @@ -366,15 +405,15 @@ jobs: create_manifest "${IMAGE_NAME}" "" /tmp/digests/base create_manifest "${IMAGE_NAME}" "-web" /tmp/digests/web - create_manifest "${IMAGE_NAME}" "-bun" /tmp/digests/bun-base - create_manifest "${IMAGE_NAME}" "-web-bun" /tmp/digests/bun-web + create_manifest "${IMAGE_NAME}" "-bun" /tmp/digests/bun-base optional + create_manifest "${IMAGE_NAME}" "-web-bun" /tmp/digests/bun-web optional - name: Create GHCR manifest run: | set -euo pipefail create_manifest() { - local image="$1" suffix="$2" dir="$3" + local image="$1" suffix="$2" dir="$3" optional="${4:-}" local tags=(-t "${image}:${VERSION}${suffix}") if [ "$PROMOTE_LATEST" = "true" ]; then tags+=(-t "${image}:latest${suffix}") @@ -384,6 +423,10 @@ jobs: refs+=("${image}@sha256:$(basename "$digest_file")") done < <(find "$dir" -type f | sort) if [ "${#refs[@]}" -eq 0 ]; then + if [ -n "$optional" ]; then + echo "::warning::No image digests in $dir — skipping optional tag ${image}:${VERSION}${suffix}" >&2 + return 0 + fi echo "No image digests in $dir" >&2 exit 1 fi @@ -392,8 +435,8 @@ jobs: create_manifest "${GHCR_IMAGE_NAME}" "" /tmp/digests/base create_manifest "${GHCR_IMAGE_NAME}" "-web" /tmp/digests/web - create_manifest "${GHCR_IMAGE_NAME}" "-bun" /tmp/digests/bun-base - create_manifest "${GHCR_IMAGE_NAME}" "-web-bun" /tmp/digests/bun-web + create_manifest "${GHCR_IMAGE_NAME}" "-bun" /tmp/digests/bun-base optional + create_manifest "${GHCR_IMAGE_NAME}" "-web-bun" /tmp/digests/bun-web optional - name: Inspect image if: needs.prepare.outputs.version != 'main' diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 16dc215a92..b316ed6708 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -63,10 +63,14 @@ jobs: # This job never runs on `pull_request`, so the fork-safety clause is always true here; # it is kept verbatim so the expression stays greppable against ci.yml. runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }} + outputs: + version: ${{ steps.resolve.outputs.version }} + tag: ${{ steps.resolve.outputs.tag }} + skip: ${{ steps.resolve.outputs.skip }} permissions: actions: read # find + download the CI run's next-build artifact for this SHA contents: write # gh release upload (attach SBOM to the GitHub Release) - id-token: write # npm provenance + id-token: write # npm provenance (GitHub Packages step) packages: write # publish to npm.pkg.github.com steps: - name: Checkout @@ -226,6 +230,28 @@ jobs: JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation run: npm run build:cli + # `build:cli` assembles dist/ but does NOT write dist/BUILD_SHA — only + # `build:release` does, by calling write-build-sha.mjs. The #10427 provenance + # guard inside check:pack-artifact rejects an artifact with no SHA (and rejects + # it even under OMNIROUTE_ALLOW_CANARY_BUILD=1: what cannot be identified cannot + # be vouched for). Without this step the build+validate pair in this job is + # structurally incompatible and fails 100% of the time — the same gap that was + # fixed in ci.yml's Package Artifact job. + - name: Stamp dist/BUILD_SHA for the provenance guard (#10427) + if: steps.resolve.outputs.skip != 'true' + env: + OMNIROUTE_BUILD_SHA: ${{ github.sha }} + run: | + export OMNIROUTE_BUILD_SHA="${OMNIROUTE_BUILD_SHA:0:7}" + node scripts/build/write-build-sha.mjs + + # The guard checks ancestry against origin/main by default, which is correct + # here (a release tag is cut from main), but the ref has to exist locally for + # `git merge-base` to resolve it. + - name: Fetch main for the provenance probe + if: steps.resolve.outputs.skip != 'true' + run: git fetch --no-tags --depth=50 origin +refs/heads/main:refs/remotes/origin/main + - name: Validate npm package artifact if: steps.resolve.outputs.skip != 'true' run: npm run check:pack-artifact @@ -265,7 +291,12 @@ jobs: # a staged package that is never approved simply expires, with no `npm deprecate` needed. - name: Prove clean-install AND upgrade-over-previous both boot if: steps.resolve.outputs.skip != 'true' - timeout-minutes: 30 + # 60, not 30. This gate was added in #8953 and the 2026-08-27 v3.8.50 publish + # was the FIRST run to ever reach it — every earlier attempt died upstream, so + # its budget had never been measured against a real run. It then blew the limit + # on its debut: `npm pack` alone took 24m37s, leaving 5 minutes for two installs + # and two boots. 30 was a guess; 60 is sized to the one measurement we have. + timeout-minutes: 60 run: npm run check:install-upgrade # WS1.3 (D2, v3.8.49 plan): STAGED publishing by default — `npm stage publish` @@ -287,41 +318,33 @@ jobs: fi npm --version - - name: Publish to npm (staged — owner approves with 2FA) - if: steps.resolve.outputs.skip != 'true' && (github.event_name != 'workflow_dispatch' || inputs.publish_mode != 'direct') + # The registry upload itself moved to the `stage-npm` job below: npm REFUSES + # `--provenance` from a self-hosted runner (422 "Unsupported GitHub Actions + # runner environment"), and the heavy verification above cannot move to a + # hosted one (16 GB is not enough for build:cli's next-build fallback — see + # this job's runs-on comment). So this job proves the bytes and hands them + # over; a tiny hosted job does the upload. + - name: Pack the verified tarball for the upload job + if: steps.resolve.outputs.skip != 'true' env: VERSION: ${{ steps.resolve.outputs.version }} - TAG: ${{ steps.resolve.outputs.tag }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: | set -euo pipefail - # Always pass --tag explicitly. Defense in depth: even if VERSION is - # accidentally an older release, the historic tag will NOT claim `@latest`. - npm stage publish --provenance --access public --tag "$TAG" - { - echo "## 📦 omniroute@$VERSION STAGED (not yet installable)" - echo "" - echo "The exact bytes are parked on the registry. To release them:" - echo '```' - echo "npm stage list omniroute # find the stage id" - echo "npm stage approve # owner 2FA — THE publish" - echo '```' - echo "To verify the staged bytes first: npm stage download → run" - echo "scripts/check/check-pack-boot.mjs against them (see RELEASE_CHECKLIST)." - echo "To discard: npm stage reject ." - } >> "$GITHUB_STEP_SUMMARY" - echo "✅ Staged omniroute@$VERSION (dist-tag=$TAG) — awaiting owner 'npm stage approve'" + # --ignore-scripts: prepublishOnly would re-run build:cli-api && build:cli, + # rebuilding bytes this job has already built, validated and boot-smoked. + npm pack --ignore-scripts + TARBALL="omniroute-${VERSION}.tgz" + test -f "$TARBALL" || { echo "expected $TARBALL to exist after npm pack" >&2; ls -la ./*.tgz || true; exit 1; } + echo "packed $TARBALL ($(du -h "$TARBALL" | cut -f1))" - - name: Publish to npm (DIRECT — emergency fallback) - if: steps.resolve.outputs.skip != 'true' && github.event_name == 'workflow_dispatch' && inputs.publish_mode == 'direct' - env: - VERSION: ${{ steps.resolve.outputs.version }} - TAG: ${{ steps.resolve.outputs.tag }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - set -euo pipefail - npm publish --provenance --access public --tag "$TAG" - echo "✅ Published omniroute@$VERSION (dist-tag=$TAG) [DIRECT mode]" + - name: Hand the tarball to the hosted publish job + if: steps.resolve.outputs.skip != 'true' + uses: actions/upload-artifact@v7 + with: + name: npm-tarball + path: omniroute-${{ steps.resolve.outputs.version }}.tgz + retention-days: 1 + if-no-files-found: error - name: Publish to GitHub Packages if: steps.resolve.outputs.skip != 'true' @@ -338,6 +361,91 @@ jobs: || echo "⚠️ omniroute@${VERSION} might already be published on GitHub Packages." echo "✅ Action finished for GitHub Packages" + # npm REFUSES `--provenance` from a self-hosted runner: + # 422 Unprocessable Entity - Error verifying sigstore provenance bundle: + # Unsupported GitHub Actions runner environment: "self-hosted". + # Only "github-hosted" runners are supported when publishing with provenance. + # v3.8.49 published fine because it predates USE_VPS_RUNNER being turned on + # (2026-08-02); v3.8.50 was the first release after it, so this had been latent + # for four weeks. Dropping --provenance was not an option: 3.8.49 carries a + # SLSA attestation and 3.8.50 must not regress that. + # The `publish` job cannot simply move to a hosted runner either — 16 GB is not + # enough for build:cli's next-build fallback. So it keeps proving the bytes and + # this job, which needs no memory at all, performs the upload. + stage-npm: + needs: publish + if: needs.publish.outputs.skip != 'true' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # npm provenance — the whole reason this job is separate + steps: + - name: Download the tarball the publish job proved + uses: actions/download-artifact@v8 + with: + name: npm-tarball + path: . + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: ${{ env.NPM_PUBLISH_NODE_VERSION }} + registry-url: https://registry.npmjs.org + + - name: Ensure npm supports staged publishing + run: | + set -euo pipefail + CUR=$(npm --version) + if ! node -e "const [a,b]='$(npm --version)'.split('.').map(Number); process.exit(a>11||(a===11&&b>=15)?0:1)"; then + # Pinned exact version (supply-chain: never float @latest in a publish + # job); bump deliberately when a newer npm is required. + echo "npm $CUR < 11.15 — installing pinned npm 11.15.0 for staged publishing" + npm install -g --ignore-scripts npm@11.15.0 + fi + npm --version + + - name: Publish to npm (staged — owner approves with 2FA) + if: github.event_name != 'workflow_dispatch' || inputs.publish_mode != 'direct' + env: + VERSION: ${{ needs.publish.outputs.version }} + TAG: ${{ needs.publish.outputs.tag }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + TARBALL="omniroute-${VERSION}.tgz" + test -f "$TARBALL" || { echo "tarball $TARBALL did not arrive from the publish job" >&2; ls -la; exit 1; } + # Always pass --tag explicitly. Defense in depth: even if VERSION is + # accidentally an older release, the historic tag will NOT claim `@latest`. + # --ignore-scripts: publishing a built tarball must never re-run + # prepublishOnly (build:cli-api && build:cli) on this small runner. + npm stage publish "$TARBALL" --provenance --access public --tag "$TAG" --ignore-scripts + { + echo "## 📦 omniroute@$VERSION STAGED (not yet installable)" + echo "" + echo "The exact bytes are parked on the registry. To release them:" + echo '```' + echo "npm stage list omniroute # find the stage id" + echo "npm stage approve # owner 2FA — THE publish" + echo '```' + echo "To verify the staged bytes first: npm stage download → run" + echo "scripts/check/check-pack-boot.mjs against them (see RELEASE_CHECKLIST)." + echo "To discard: npm stage reject ." + } >> "$GITHUB_STEP_SUMMARY" + echo "✅ Staged omniroute@$VERSION (dist-tag=$TAG) — awaiting owner 'npm stage approve'" + + - name: Publish to npm (DIRECT — emergency fallback) + if: github.event_name == 'workflow_dispatch' && inputs.publish_mode == 'direct' + env: + VERSION: ${{ needs.publish.outputs.version }} + TAG: ${{ needs.publish.outputs.tag }} + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + TARBALL="omniroute-${VERSION}.tgz" + test -f "$TARBALL" || { echo "tarball $TARBALL did not arrive from the publish job" >&2; ls -la; exit 1; } + npm publish "$TARBALL" --provenance --access public --tag "$TAG" --ignore-scripts + echo "✅ Published omniroute@$VERSION (dist-tag=$TAG) [DIRECT mode]" + publish-opencode-plugin: runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index 07545f2a37..31c9e3b96a 100644 --- a/.gitignore +++ b/.gitignore @@ -293,3 +293,6 @@ docker-compose.yml.bak # Ad-hoc test sandboxes (never tracked — may contain local DBs) /.sandbox/ .aider* + +# check:install-upgrade work trees (~12 GB, disposable) +/.install-upgrade/ diff --git a/changelog.d/fixes/11741-install-upgrade-schema-convergence.md b/changelog.d/fixes/11741-install-upgrade-schema-convergence.md new file mode 100644 index 0000000000..6bdf6e846a --- /dev/null +++ b/changelog.d/fixes/11741-install-upgrade-schema-convergence.md @@ -0,0 +1,8 @@ +- **fix(db):** `model_capabilities` is created by a migration instead of lazily on the first + models.dev sync, so a clean install and an upgraded install converge on the same schema + regardless of which features have run +- **fix(ci):** `check:install-upgrade` now fails on an `npm` install truncated by ENOSPC + (npm reports it as a warning and still exits 0), authenticates its health probe so the + version assertion works against the hardened health payload, frees the clean-install tree + before the upgrade phase, and no longer reports a schema divergence computed from a boot + that never served diff --git a/changelog.d/fixes/11856-npm-payload-nft-manifests.md b/changelog.d/fixes/11856-npm-payload-nft-manifests.md new file mode 100644 index 0000000000..0420d6f9c0 --- /dev/null +++ b/changelog.d/fixes/11856-npm-payload-nft-manifests.md @@ -0,0 +1,4 @@ +- Excluded Next.js Node File Trace manifests (`*.nft.json`) from the published npm + tarball. They are build-time metadata and are never read while serving, but had + grown to 668.7 MB — 61% of the package — which pushed the upload past the + registry limit and made `npm publish` fail with `413 Payload Too Large`. diff --git a/changelog.d/fixes/11868-npm-provenance-hosted-runner.md b/changelog.d/fixes/11868-npm-provenance-hosted-runner.md new file mode 100644 index 0000000000..4cc94111ce --- /dev/null +++ b/changelog.d/fixes/11868-npm-provenance-hosted-runner.md @@ -0,0 +1,4 @@ +- Split the npm registry upload into its own GitHub-hosted job. npm refuses + `--provenance` from a self-hosted runner (`422 ... Only "github-hosted" runners + are supported`), which blocked the v3.8.50 publish; the heavy verification + cannot move to a hosted runner, so it now hands the proven tarball over instead. diff --git a/config/quality/install-upgrade-allowlist.json b/config/quality/install-upgrade-allowlist.json index 4e6f3c6891..0f6145223a 100644 --- a/config/quality/install-upgrade-allowlist.json +++ b/config/quality/install-upgrade-allowlist.json @@ -1,6 +1,6 @@ { - "_doc": "Tables that exist ONLY in databases upgraded from an older version — residue whose CREATE left the migration set in some past cycle but survives where it already existed. Harmless (nothing references them), but recorded here so check-install-upgrade.mjs can still fail on a NEW divergence. The opposite direction (a table a clean install creates but an upgrade does not) is NEVER allowlisted: it means every existing user is missing structure the code expects.", + "_doc": "Tables that exist ONLY in databases upgraded from an older version. Two causes, and they call for different fixes: (a) residue whose CREATE left the migration set in some past cycle but survives where it already existed — allowlist it here; (b) a table created LAZILY at runtime with `CREATE TABLE IF NOT EXISTS` inside a feature code path — whether a database has it depends on whether that feature ran, so it diverges by TIMING and can show up on EITHER side. Fix (b) with a migration instead of an entry here (see src/lib/db/migrations/163_model_capabilities.sql); an allowlist entry only hides it in one direction. Either way, recorded so check-install-upgrade.mjs can still fail on a NEW divergence. The opposite direction (a table a clean install creates but an upgrade does not) is NEVER allowlisted: it means every existing user is missing structure the code expects.", "residualTables": { - "cache_metrics": "Measured 2026-07-30 on a real 3.8.48 install upgraded to 3.8.49 (VPS .16, 165 MB database, 114 → 117 tables). Present in upgraded databases, absent from clean installs. No code path referenced it during the upgrade (zero `no such table` in 150 log lines, both installs healthy). Left in place rather than dropped: a DROP migration on a table we cannot prove is unused everywhere is the riskier change. Revisit when the cache subsystem is next touched." + "cache_metrics": "Measured 2026-07-30 on a real 3.8.48 install upgraded to 3.8.49 (VPS .16, 165 MB database, 114 → 117 tables). Present in upgraded databases, absent from clean installs. Cause identified 2026-08-27: it is case (b) above — created lazily by `ensureCacheMetricsTable()` at src/lib/semanticCache.ts:34, never by a migration, so it appears only where the semantic cache has run. No code path referenced it during the upgrade (zero `no such table` in 150 log lines, both installs healthy). Left as an allowlist entry rather than promoted to a migration or dropped: unlike model_capabilities it did not block a release, and creating a table for a subsystem we cannot prove is live is not a change to make blind. Revisit when the cache subsystem is next touched." } } diff --git a/config/quality/quality-baseline.json b/config/quality/quality-baseline.json index ec657bda20..951166eaac 100644 --- a/config/quality/quality-baseline.json +++ b/config/quality/quality-baseline.json @@ -169,7 +169,7 @@ "dedicatedGate": true }, "zizmorFindings": { - "value": 192, + "value": 194, "_rebaseline_2026_08_20_radar_export_workflow": "190 -> 192 (+2). Workflow novo `.github/workflows/radar-export.yml` (passo 10 do go-live do Radar: publica o export estável do catálogo como asset de release para o servidor privado baixar via RADAR_EXPORT_URL). Os +2 são unpinned-uses @vN: actions/checkout@v7 + actions/setup-node@v7 — a MESMA convenção deliberada de todos os workflows (ver _scanner_harden_workflows_2026_06_16); fixar por SHA só este violaria a convenção. O findings artipacked do checkout foi CORRIGIDO com `persist-credentials: false` (o job publica via GH_TOKEN em `gh release`, não usa a credencial do checkout). Nenhuma classe nova de template-injection / cache-poisoning / dangerous-triggers. Medido local com zizmor 1.25.2 via `node scripts/check/check-workflows.mjs --ratchet` = 191; +1 do delta conhecido do runner (ver _rebaseline_2026_07_28_ci_runner_delta: o runner enxerga 1 unpinned-uses @vN a mais que o devbox no mesmo commit; a baseline segue o runner) => 192.", "_rebaseline_2026_07_20_aliasresolver_hook_split_7808": "175 -> 176 (+1). Companion to PR #7808 (CodeQL js/incomplete-url-substring-sanitization fix in bin/aliasResolver.mjs). The +1 is NOT caused by this PR's code changes (bin/* is not a workflow file) — it is a pre-existing drift that surfaced because the ratchet gate runs on this PR's CI: the zizmor scanner version on the GitHub runner gained a new rule (or extended an existing one) since the v3.8.49 baseline was seeded on 2026-07-17. Breakdown: the new finding is an unpinned-uses @vN class item on one of the existing workflows (same deliberate convention as _scanner_harden_workflows_2026_06_16 — @vN is intentional, SHA-pinning only this one would violate the convention). No new template-injection/artipacked/cache-poisoning/dangerous-triggers classes introduced. Measured by the Quality Gates (Extended) job on run 29713001401 = 176, baseline was 175. Note: by the time this landed on release/v3.8.49, the baseline was already at 176 via _rebaseline_2026_07_17_combo_recovery_hints — this entry is kept as historical record; no further bump applied.", "_rebaseline_2026_07_17_v3849_release": "169 -> 175 (+6). Cycle workflow drift (v3.8.48/v3.8.49): npm-publish.yml (new, WS1.3 #7092), electron-release.yml, nightly-compat.yml, nightly-release-green.yml, CI restructures (#7501 full-history base fetch, #7355 main-green, #7202 merge-queue gates, Trunk/Codecov). Breakdown vs v3.8.47: +3 unpinned-uses (@vN convention, deliberate per _scanner_harden_workflows_2026_06_16), +2 cache-poisoning (artifact upload/cache in the OWN electron-release/npm-publish RELEASE workflows -- operator-controlled, not fork-PR exploitable), +1 excessive-permissions (nightly-compat.yml permissions:issues). No new template-injection/artipacked/dangerous-triggers. Measured with zizmor 1.25.2 via `node scripts/check/check-workflows.mjs --ratchet` = 175 on da3a0be69.", @@ -180,7 +180,8 @@ "_rebaseline_2026_06_23_v3834_release": "152 -> 155 (+3). The 3 new unpinned-uses are in .github/workflows/nightly-release-green.yml (added by #4622 this cycle): actions/checkout@v7, actions/setup-node@v6, actions/upload-artifact@v4 — the SAME deliberate @vN convention as ci.yml's own checkout@v7/setup-node@v6 and every other workflow (see _scanner_harden_workflows_2026_06_16 + _zizmor_rebaseline_2026_06_20_ci_build_artifact_reuse). SHA-pinning only this workflow would violate the convention. The workflow-lint ratchet does NOT run on PR->release fast-gates, so it surfaced only on the release PR; measured locally via `npm run check:workflows -- --ratchet` = 155. No new template-injection/artipacked/cache-poisoning.", "_rebaseline_2026_07_13_v3847_release_preflight": "159 -> 169 (+10). Findings from cycle-merged workflow changes: #6716 (PR gate restructure), #6781 (unit fast-path shard 2->4), #6788 (TIA tsx loader split), #6881 (electron-updater latest.yml manifests in release assets) — same deliberate @vN unpinned-uses convention as prior rebaselines; no new template-injection/artipacked/cache-poisoning classes. Measured via `npm run check:workflows -- --ratchet` = 169 on the v3.8.47 release pre-flight.", "_rebaseline_2026_07_28_v3849_release_preflight": "176 -> 189 (+13). Pre-flight de fechamento da v3.8.49 (934 commits no ciclo). Deriva de workflow: 1 workflow novo (build-rinseaid-image.yml) mais os bumps de action do Dependabot ao longo do ciclo — todos da MESMA classe unpinned-uses @vN, convenção deliberada do repo (ver _scanner_harden_workflows_2026_06_16); fixar por SHA só estes violaria a convenção. Nenhuma classe nova de template-injection / artipacked / cache-poisoning / dangerous-triggers. Nesta mesma passada foram CORRIGIDAS 3 diretivas shellcheck malformadas (SC1125: `# shellcheck disable=SC2086 — texto`, em que o travessão invalida o par key=value) em ci.yml e nightly-release-green.yml. Medido com zizmor 1.25.2 via `npm run check:workflows -- --ratchet` = 189.", - "_rebaseline_2026_07_28_ci_runner_delta": "189 -> 190 (+1). Medido 189 no devbox e 190 no runner do GitHub no MESMO commit (run 30396592013, job Quality Gates (Extended)) — mesma classe já registrada em _rebaseline_2026_07_20_aliasresolver_hook_split_7808: a versão do zizmor no runner enxerga uma finding a mais que a local, sempre da classe unpinned-uses @vN. O valor do runner é o que o gate compara, então a baseline segue o runner." + "_rebaseline_2026_07_28_ci_runner_delta": "189 -> 190 (+1). Medido 189 no devbox e 190 no runner do GitHub no MESMO commit (run 30396592013, job Quality Gates (Extended)) — mesma classe já registrada em _rebaseline_2026_07_20_aliasresolver_hook_split_7808: a versão do zizmor no runner enxerga uma finding a mais que a local, sempre da classe unpinned-uses @vN. O valor do runner é o que o gate compara, então a baseline segue o runner.", + "_rebaseline_2026_08_28_npm_publish_hosted_stage_job": "192 -> 194 (+2). Job novo `stage-npm` em .github/workflows/npm-publish.yml: o npm RECUSA `--provenance` vindo de runner self-hosted (422 \"Unsupported GitHub Actions runner environment\"), e o job `publish` nao pode migrar para runner hospedado porque 16 GB nao bastam para o fallback next-build do build:cli (documentado no proprio runs-on). A separacao foi a unica saida que preserva a atestacao SLSA que a 3.8.49 ja tem. Os +2 sao da MESMA convencao deliberada de todos os workflows (ver _scanner_harden_workflows_2026_06_16): unpinned-uses @vN em actions/download-artifact@v8 + actions/setup-node@v7, mais o cache-poisoning que o proprio setup-node@v7 ja gera nos outros 2 jobs deste MESMO arquivo (linhas 85 e 463) e que ja esta na baseline. Fixar por SHA so este job violaria a convencao. Nenhuma classe nova: zero template-injection / artipacked / dangerous-triggers / excessive-permissions — o job declara apenas contents:read + id-token:write, que e o minimo para a proveniencia. Medido pelo job Quality Gates (Extended) no run 33162... da PR #11877 = 194." }, "vulnCount": { "value": 22, diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 228c97cf1c..b992b7a2b4 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -103,6 +103,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_MIGRATIONS_DIR` | _(auto-detect)_ | `src/lib/db/migrationRunner.ts` | Override the directory that the migration runner scans. Useful when shipping bundled migrations in custom builds. | | `OMNIROUTE_EXTRA_MIGRATIONS_DIRS` | _(unset)_ | `src/lib/db/migrationRunner/extraDirs.ts` | Additional migration directories as `namespace=dir` entries separated by the platform path delimiter (e.g. `ee=/opt/app/enterprise/db/migrations`). Files found there are recorded as `-`, so a distribution shipping its own migrations never collides with the upstream numeric slots. A malformed entry, an invalid namespace or a missing directory throws at startup instead of silently skipping the schema. | | `OMNIROUTE_MAX_PENDING_MIGRATIONS` | `50` | `src/lib/db/migrationRunner.ts` | Mass-pending-migrations safety threshold (#3416). Startup aborts if more than this many migrations are pending on an existing DB (guards against a wiped tracking table). Raise it to restore an older backup; set to `0` to disable the check. | +| `OMNIROUTE_INSTALL_UPGRADE_WORKDIR` | _(`/.install-upgrade`)_ | `scripts/check/check-install-upgrade.mjs` | Working directory for the `check:install-upgrade` release gate. It needs roughly 12 GB (two ~3 GB install trees plus the tarball), so it must not run on a small tmpfs — on the self-hosted runner `/tmp` is a 12 GB RAM-backed tmpfs and the gate exhausted it, truncating the package. | | `OMNIROUTE_SPEND_FLUSH_INTERVAL_MS` | _(default in code)_ | `src/lib/spend/batchWriter.ts` | Flush interval (ms) for the batched spend/cost writer. Lower values reduce write coalescing; higher values reduce DB contention. | | `OMNIROUTE_SPEND_MAX_BUFFER_SIZE` | _(default in code)_ | `src/lib/spend/batchWriter.ts` | Max buffered spend entries before a forced flush. Raise on high-QPS deployments; lower when bounded memory matters more. | | `OMNIROUTE_PROXY_FETCH_DEBUG` | _(unset)_ | `open-sse/utils/proxyFetch.ts` | Set to `"true"` to emit `[ProxyFetch]` debug logs on the Vercel relay path. Off by default to avoid leaking routing hints. | diff --git a/package.json b/package.json index 08e5122feb..c8f4a78aa8 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "!**/*.test.js", "!**/*.test.mjs", "!**/*.spec.ts", - "!**/*.spec.tsx" + "!**/*.spec.tsx", + "!**/*.nft.json" ], "workspaces": [ "open-sse", diff --git a/scripts/check/check-install-upgrade.mjs b/scripts/check/check-install-upgrade.mjs index 87253c611d..a71ccdb1fe 100644 --- a/scripts/check/check-install-upgrade.mjs +++ b/scripts/check/check-install-upgrade.mjs @@ -30,10 +30,12 @@ * Requires `npm run build:cli` first — this is a --with-build gate, like check:pack-boot. */ -import { execFileSync, spawn } from "node:child_process"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import crypto from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { DatabaseSync } from "node:sqlite"; const BOOT_DEADLINE_MS = 180_000; @@ -43,6 +45,59 @@ const ALLOWLIST_PATH = "config/quality/install-upgrade-allowlist.json"; const log = (msg) => console.log(`[install-upgrade] ${msg}`); const warn = (msg) => console.log(`[install-upgrade] ⚠️ ${msg}`); +/** Root of the installed package inside an `npm install -g --prefix` tree. */ +function packageRootFor(prefix) { + return path.join(prefix, "lib", "node_modules", "omniroute"); +} + +/** + * Credential for the health probe. + * + * GHSA-mvf8-qc78-5mxm hardened /api/monitoring/health: an ANONYMOUS caller now gets only + * `{ status }` — the version, node version, pid and provider config are reserved for a + * management principal (src/app/api/monitoring/health/route.ts → publicHealthView). An + * unauthenticated probe therefore reads `body.version === undefined`, and this gate's + * version assertion could never pass again; the v3.8.50 publish run failed with + * "clean: health reports version undefined, expected 3.8.50" for exactly that reason. + * + * The gate spawns the server itself, so it can mint the credential instead of guessing one: + * `OMNIROUTE_INTERNAL_SERVICE_TOKEN` + the `x-omniroute-internal-service-token` header is + * accepted by requireManagementAuth() via isTrustedLoopbackInternalServiceRequest(), and the + * probe is loopback by construction. Unlike the machine token `check:pack-boot` derives, this + * does not depend on a readable machine-id, and an older PREVIOUS version that never gated + * health simply ignores the header. The assertion keeps its full strength — it just presents + * a credential. + */ +const INTERNAL_SERVICE_TOKEN = crypto.randomBytes(32).toString("hex"); +const INTERNAL_SERVICE_HEADER = "x-omniroute-internal-service-token"; + +/** + * Secondary credential: the same loopback machine token `check:pack-boot` derives from the + * packaged CLI. Sent alongside the internal-service token so a build that only honours one + * of the two still answers with the full payload. + */ +function derivePackagedCliToken(prefix) { + const cliModuleUrl = pathToFileURL( + path.join(packageRootFor(prefix), "bin", "cli", "utils", "cliToken.mjs") + ).href; + try { + return execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + "import(process.argv[1]).then(async m => process.stdout.write(await m.getCliToken()))", + cliModuleUrl, + ], + { encoding: "utf8", env: { ...process.env } } + ).trim(); + } catch { + // A truncated/broken install cannot derive a token. Returning null keeps the boot + // probe running (it will fail loudly on its own) instead of crashing the gate here. + return null; + } +} + function pickTarball(packJson) { const filename = JSON.parse(packJson)?.[0]?.filename; if (!filename) throw new Error("npm pack --json returned no filename"); @@ -121,6 +176,11 @@ async function bootAndProbe({ prefix, dataDir, port, expectVersion, label }) { if (!fs.existsSync(binPath)) { return { ok: false, failures: [`${label}: bin not found at ${binPath}`], tail: [] }; } + const cliToken = derivePackagedCliToken(prefix); + const probeHeaders = { + [INTERNAL_SERVICE_HEADER]: INTERNAL_SERVICE_TOKEN, + ...(cliToken ? { "x-omniroute-cli-token": cliToken } : {}), + }; const child = spawn(binPath, ["serve", "--port", String(port)], { env: { ...process.env, @@ -130,6 +190,7 @@ async function bootAndProbe({ prefix, dataDir, port, expectVersion, label }) { API_KEY_SECRET: "install-upgrade-gate-api-key-secret-long", DISABLE_SQLITE_AUTO_BACKUP: "true", OMNIROUTE_SKIP_SYSTEM_TRUST: "1", + OMNIROUTE_INTERNAL_SERVICE_TOKEN: INTERNAL_SERVICE_TOKEN, }, stdio: ["ignore", "pipe", "pipe"], detached: true, @@ -151,20 +212,33 @@ async function bootAndProbe({ prefix, dataDir, port, expectVersion, label }) { let result = { ok: false, failures: [`${label}: never became healthy`], tail }; while (Date.now() < deadline) { if (childExit !== null) { - result = { ok: false, failures: [`${label}: exited with code ${childExit} before serving`], tail }; + result = { + ok: false, + failures: [`${label}: exited with code ${childExit} before serving`], + tail, + }; break; } try { - const res = await fetch(`http://127.0.0.1:${port}/api/monitoring/health`); + const res = await fetch(`http://127.0.0.1:${port}/api/monitoring/health`, { + headers: probeHeaders, + }); const body = await res.json().catch(() => null); if (res.status === 200 && body && typeof body === "object") { const failures = []; // `status` may legitimately report degraded (no providers configured) — the gate // targets boot crashes and version mismatches, not health of a bare install. - if (expectVersion && body.version !== expectVersion) { - failures.push(`${label}: health reports version ${body.version}, expected ${expectVersion}`); + const reportedVersion = body.version ?? body.system?.version; + if (expectVersion && reportedVersion !== expectVersion) { + failures.push( + `${label}: health reports version ${reportedVersion}, expected ${expectVersion}` + + (reportedVersion === undefined + ? " — the payload carries no version at all, which is the ANONYMOUS health " + + "view: the probe's credentials were not accepted (see GHSA-mvf8-qc78-5mxm)" + : "") + ); } - result = { ok: failures.length === 0, version: body.version, failures, tail }; + result = { ok: failures.length === 0, version: reportedVersion, failures, tail }; break; } } catch { @@ -183,13 +257,72 @@ async function bootAndProbe({ prefix, dataDir, port, expectVersion, label }) { return result; } -function npmInstallInto(prefix, spec) { - execFileSync("npm", ["install", "-g", "--prefix", prefix, "--no-audit", "--no-fund", spec], { - encoding: "utf8", - maxBuffer: 128 * 1024 * 1024, - }); +/** + * `npm install` reports ENOSPC as a *warning* per failed tar entry and still exits 0. + * + * That is not a theoretical concern: on the v3.8.50 publish run the Phase B upgrade install + * emitted 5611 `npm warn tar TAR_ENTRY_ERROR ENOSPC: no space left on device` lines, exited + * 0, and left a truncated package behind. `omniroute serve` then "exited with code 0 before + * serving", no migration ever ran, and the gate concluded the release was missing 15 tables + * — a full false alarm produced by a full disk. Each install tree is ~3 GB, and the run + * builds two of them plus a ~275 MB tarball. + * + * So: surface the truncation at the install, where it is unambiguous. + */ +function npmInstallInto(prefix, spec, label = spec) { + // spawnSync (not execFileSync): execFileSync forwards the child's stderr straight to the + // parent's, so the ENOSPC warnings scrolled past in CI without the script ever seeing + // them. spawnSync hands both streams back. + const run = spawnSync( + "npm", + ["install", "-g", "--prefix", prefix, "--no-audit", "--no-fund", spec], + { encoding: "utf8", maxBuffer: 512 * 1024 * 1024 } + ); + const output = `${run.stdout ?? ""}${run.stderr ?? ""}`; + // Keep the install log visible, but a truncated package emits thousands of identical + // warnings — collapse them so the real message is not buried. + const stderrLines = String(run.stderr ?? "").split("\n"); + const shown = stderrLines.length > 60 ? stderrLines.slice(0, 40) : stderrLines; + if (String(run.stderr ?? "").trim()) { + process.stderr.write(shown.join("\n") + "\n"); + if (stderrLines.length > 60) { + process.stderr.write(`[install-upgrade] … ${stderrLines.length - 40} more npm line(s)\n`); + } + } + assertNoDiskExhaustion(output, label); + if (run.error) throw run.error; + if (run.status !== 0) { + throw new Error(`${label}: npm install exited with code ${run.status}`); + } } +export function assertNoDiskExhaustion(output, label) { + if (!/ENOSPC|no space left on device/i.test(output)) return; + const count = (output.match(/ENOSPC/g) ?? []).length; + throw new Error( + `${label}: the install ran out of disk space (${count} ENOSPC error(s) from npm). ` + + `The package tree is truncated, so anything measured from it — boot, schema, ` + + `migrations — is meaningless. Free space in ${workDirForMessages} (each install tree is ` + + `~3 GB) and re-run. This is an environment failure, NOT a schema divergence.` + ); +} + +/** Best-effort free bytes on the filesystem backing `dir`, or null when unavailable. */ +function freeBytes(dir) { + try { + return fs.statfsSync(dir).bavail * fs.statfsSync(dir).bsize; + } catch { + return null; + } +} + +const GB = 1024 ** 3; + +// Set once the work directory exists, so the ENOSPC message names the filesystem that +// actually ran out — pointing at /tmp when the gate works elsewhere sends the reader to +// free space on the wrong volume (which is what happened during the v3.8.50 publish). +let workDirForMessages = os.tmpdir(); + function resolvePreviousVersion(current, explicit) { if (explicit) return explicit; const out = execFileSync("npm", ["view", "omniroute", "dist-tags.latest"], { encoding: "utf8" }); @@ -198,7 +331,9 @@ function resolvePreviousVersion(current, explicit) { if (latest === current) { // The version under test is already published (re-run of a shipped release): step back // to the highest published version strictly below it. - const all = JSON.parse(execFileSync("npm", ["view", "omniroute", "versions", "--json"], { encoding: "utf8" })); + const all = JSON.parse( + execFileSync("npm", ["view", "omniroute", "versions", "--json"], { encoding: "utf8" }) + ); const stable = all.filter((v) => !/-(rc|alpha|beta|pre|next)/.test(v) && v !== current); return stable[stable.length - 1]; } @@ -218,11 +353,25 @@ async function main() { } const version = JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8")).version; const allowlist = loadAllowlist(ROOT); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-install-upgrade-")); + // NOT os.tmpdir(): on the self-hosted runner /tmp is a 12 GB tmpfs backed by RAM, while + // the root filesystem has ~66 GB free. This gate needs ~12 GB, so it exhausted the tmpfs + // and npm truncated the package — 58269 ENOSPC errors on the v3.8.50 publish, which the + // previous code could only report as a crash. Freeing disk did not help because the disk + // was never the constraint. Work on real disk beside the repo instead. + const workRoot = + process.env.OMNIROUTE_INSTALL_UPGRADE_WORKDIR || path.join(ROOT, ".install-upgrade"); + fs.mkdirSync(workRoot, { recursive: true }); + const tmp = fs.mkdtempSync(path.join(workRoot, "omniroute-install-upgrade-")); + workDirForMessages = tmp; const failures = []; const warnings = []; try { + // Timed, because this turned out to be the expensive part: on the 2026-08-27 + // v3.8.50 publish `npm pack` alone took 24m37s, leaving 5 of the step's 30-minute + // budget for two installs and two boots. Without a duration here the log showed + // only "packing…" then a timeout, which reads like a hang and is not. + const packStarted = Date.now(); log(`packing v${version}…`); const packOut = execFileSync("npm", ["pack", "--json", "--pack-destination", tmp], { cwd: ROOT, @@ -230,13 +379,31 @@ async function main() { maxBuffer: 128 * 1024 * 1024, }); const tarball = path.join(tmp, pickTarball(packOut)); + const packMb = (fs.statSync(tarball).size / 1024 / 1024).toFixed(1); + log(`packed in ${Math.round((Date.now() - packStarted) / 1000)}s (${packMb} MB)`); + + // Each install tree is ~3 GB and this run builds two of them, side by side, plus the + // ~275 MB tarball. On the v3.8.50 publish run that overflowed the runner disk mid-way + // through the Phase B upgrade install; npm warned per truncated tar entry and still + // exited 0, and every later measurement was taken from a broken tree. + const availableBytes = freeBytes(tmp); + if (availableBytes !== null) { + log(`free space in ${tmp}: ${(availableBytes / GB).toFixed(1)} GB`); + if (availableBytes < 12 * GB) { + warn( + `only ${(availableBytes / GB).toFixed(1)} GB free — this gate needs roughly 12 GB ` + + `(two ~3 GB install trees, the second installed over twice, plus the tarball). ` + + `An install truncated by ENOSPC looks like a schema divergence.` + ); + } + } // ---- Phase A: clean install ------------------------------------------------- log("PHASE A — clean install of the packed tarball"); const aPrefix = path.join(tmp, "a-prefix"); const aData = path.join(tmp, "a-data"); fs.mkdirSync(aData, { recursive: true }); - npmInstallInto(aPrefix, tarball); + npmInstallInto(aPrefix, tarball, "clean install"); const a = await bootAndProbe({ prefix: aPrefix, dataDir: aData, @@ -245,14 +412,34 @@ async function main() { label: "clean", }); failures.push(...a.failures); + const cleanBooted = a.ok; if (a.ok) log(`clean install healthy on v${a.version}`); + else if (a.tail?.length) { + console.error("[install-upgrade] last output from the clean-install server:"); + console.error(a.tail.join("").split("\n").slice(-40).join("\n")); + } const aDb = findDb(aData); const freshTables = aDb ? readTables(aDb) : null; if (!freshTables) failures.push("clean: no SQLite database was created"); else log(`clean install schema: ${freshTables.size} tables`); + // Phase A is fully measured (boot verdict + schema snapshot); its ~3 GB install tree is + // dead weight from here on and Phase B needs the room. The DATA_DIR stays — only the + // node_modules tree goes. + if (!skipUpgrade) { + fs.rmSync(aPrefix, { recursive: true, force: true }); + const reclaimed = freeBytes(tmp); + log( + "released the clean-install tree before the upgrade phase" + + (reclaimed !== null ? ` (${(reclaimed / GB).toFixed(1)} GB free)` : "") + ); + } + // ---- Phase B: upgrade over the previous published version ------------------- let upgradedTables = null; + // `--skip-upgrade` never reaches the convergence block (upgradedTables stays null), so + // defaulting this to true keeps that path unchanged. + let upgradeBooted = true; if (skipUpgrade) { warn("PHASE B skipped (--skip-upgrade)"); } else { @@ -262,7 +449,7 @@ async function main() { const bData = path.join(tmp, "b-data"); fs.mkdirSync(bData, { recursive: true }); - npmInstallInto(bPrefix, `omniroute@${previous}`); + npmInstallInto(bPrefix, `omniroute@${previous}`, `previous(${previous}) install`); const before = await bootAndProbe({ prefix: bPrefix, dataDir: bData, @@ -273,14 +460,16 @@ async function main() { if (!before.ok) { // A broken PREVIOUS version is not this release's fault — degrade to a warning so a // historically bad publish cannot block the current one. - warnings.push(`previous version ${previous} did not boot cleanly — upgrade path unverified`); + warnings.push( + `previous version ${previous} did not boot cleanly — upgrade path unverified` + ); for (const f of before.failures) warn(f); } else { const beforeDb = findDb(bData); const beforeTables = beforeDb ? readTables(beforeDb) : new Set(); log(`previous(${previous}) schema: ${beforeTables.size} tables — upgrading in place`); - npmInstallInto(bPrefix, tarball); + npmInstallInto(bPrefix, tarball, "upgrade install"); const after = await bootAndProbe({ prefix: bPrefix, dataDir: bData, @@ -290,6 +479,11 @@ async function main() { }); failures.push(...after.failures); if (after.ok) log(`upgrade healthy on v${after.version}`); + upgradeBooted = after.ok; + if (!after.ok && after.tail?.length) { + console.error("[install-upgrade] last output from the upgraded server:"); + console.error(after.tail.join("").split("\n").slice(-40).join("\n")); + } const afterDb = findDb(bData); upgradedTables = afterDb ? readTables(afterDb) : null; @@ -306,7 +500,19 @@ async function main() { } // ---- Schema convergence ----------------------------------------------------- - if (freshTables && upgradedTables) { + // Only meaningful when BOTH servers actually served. A boot that died before serving + // never ran a migration, so its database still holds the PREVIOUS release's schema and + // every post-baseline table shows up as "a clean install creates but an upgrade does + // not" — which is what the v3.8.50 publish run reported after ENOSPC truncated the + // upgrade install. Comparing there does not add information, it manufactures a + // 15-table false alarm on top of the real failure. The run still fails: the boot + // failure is already in `failures`. + if (freshTables && upgradedTables && !(cleanBooted && upgradeBooted)) { + warn( + "schema convergence NOT evaluated — a phase failed to boot, so its database was " + + "never migrated and any table difference would describe the broken boot, not the schema" + ); + } else if (freshTables && upgradedTables) { const verdict = evaluateConvergence({ freshTables, upgradedTables, @@ -334,7 +540,10 @@ async function main() { // Only run the (expensive) gate when invoked directly — importing this module for the pure // helper above must not pack, install or boot anything. -if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname)) { +if ( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname) +) { main().catch((err) => { console.error(`[install-upgrade] crashed: ${err?.message ?? err}`); process.exit(1); diff --git a/src/lib/db/migrations/163_model_capabilities.sql b/src/lib/db/migrations/163_model_capabilities.sql new file mode 100644 index 0000000000..bf81e76ae1 --- /dev/null +++ b/src/lib/db/migrations/163_model_capabilities.sql @@ -0,0 +1,43 @@ +-- 163_model_capabilities.sql +-- +-- Promote `model_capabilities` from a lazily-created runtime table to a real migration. +-- +-- WHY: the table was only ever created by `ensureCapabilitiesTable()` in +-- src/lib/modelsDevSync.ts, on demand, the first time a models.dev capability sync ran. +-- Whether a database has it therefore depends on TIMING, not on the schema version — so a +-- clean install and an upgraded install diverge for no structural reason. The v3.8.50 +-- publish run hit exactly that: `check:install-upgrade` reported `model_capabilities` as a +-- table "present only after upgrade", because the older database had already run a sync +-- and the freshly-installed one had not. +-- +-- Creating it here makes both install paths converge deterministically. +-- `ensureCapabilitiesTable()` stays in place as an idempotent safety net (it is a +-- CREATE TABLE IF NOT EXISTS and now always a no-op); tests/unit/db-install-upgrade-schema-parity.test.ts +-- pins the two definitions against drift. +-- +-- IF NOT EXISTS is required, not decorative: every database that ever ran a models.dev +-- sync already has this table, and this migration must be a no-op there. + +CREATE TABLE IF NOT EXISTS model_capabilities ( + provider TEXT NOT NULL, + model_id TEXT NOT NULL, + tool_call BOOLEAN, + reasoning BOOLEAN, + attachment BOOLEAN, + structured_output BOOLEAN, + temperature BOOLEAN, + modalities_input TEXT, + modalities_output TEXT, + knowledge_cutoff TEXT, + release_date TEXT, + last_updated TEXT, + status TEXT, + family TEXT, + open_weights BOOLEAN, + limit_context INTEGER, + limit_input INTEGER, + limit_output INTEGER, + interleaved_field TEXT, + last_synced TEXT, + PRIMARY KEY (provider, model_id) +); diff --git a/src/lib/modelsDevSync.ts b/src/lib/modelsDevSync.ts index c4c0f2f273..b4f4ec09c7 100644 --- a/src/lib/modelsDevSync.ts +++ b/src/lib/modelsDevSync.ts @@ -24,6 +24,7 @@ import { getDbInstance } from "./db/core"; import { invalidateDbCache, getModelCatalogCacheVersion } from "./db/readCache"; import { backupDbFile } from "./db/backup"; +import { registerDbStateResetter } from "./db/stateReset"; import { transformModelsDevToPricing, @@ -231,6 +232,15 @@ function mapCapabilityRecord(record: Record): ModelCapabilityEn let pricingMemo: PricingByProvider | null = null; let pricingMemoVersion = -1; // -1: never equals a real cacheVersion (starts at 0), guarantees a miss on the first call +// resetDbInstance() (tests, DB swaps) must also drop the pricing memo below — +// ported from main's #10055 and wired to THIS memo (keyed on the catalog cache +// version), not to a second cache of its own. +function invalidateModelsDevPricingCache(): void { + pricingMemo = null; + pricingMemoVersion = -1; +} +registerDbStateResetter(invalidateModelsDevPricingCache); + /** * Read synced pricing from `models_dev_pricing` namespace. * Results are memoized until `saveModelsDevPricing` / `clearModelsDevPricing`. @@ -285,6 +295,7 @@ export function saveModelsDevPricing(data: PricingByProvider): void { }); tx(); backupDbFile("pre-write"); + invalidateModelsDevPricingCache(); invalidateDbCache("pricing"); } @@ -295,6 +306,7 @@ export function clearModelsDevPricing(): void { const db = getDbInstance(); db.prepare("DELETE FROM key_value WHERE namespace = 'models_dev_pricing'").run(); backupDbFile("pre-write"); + invalidateModelsDevPricingCache(); invalidateDbCache("pricing"); } diff --git a/tests/unit/check-install-upgrade-convergence.test.ts b/tests/unit/check-install-upgrade-convergence.test.ts index de3b74716a..47f8c51087 100644 --- a/tests/unit/check-install-upgrade-convergence.test.ts +++ b/tests/unit/check-install-upgrade-convergence.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import test from "node:test"; // @ts-expect-error — plain .mjs gate script, no type declarations by design -import { evaluateConvergence } from "../../scripts/check/check-install-upgrade.mjs"; +import { + assertNoDiskExhaustion, + evaluateConvergence, +} from "../../scripts/check/check-install-upgrade.mjs"; /** * The whole point of this gate is that the two directions of schema divergence are NOT @@ -66,7 +69,11 @@ test("UNKNOWN residue fails — a new divergence must not hide behind the allowl assert.equal(v.ok, false); assert.deepEqual(v.unknownResidue, ["surprise_table"]); assert.match(v.failures[0], /surprise_table/); - assert.doesNotMatch(v.failures[0], /cache_metrics/, "the known one must not be re-reported as new"); + assert.doesNotMatch( + v.failures[0], + /cache_metrics/, + "the known one must not be re-reported as new" + ); }); test("both directions at once report both failures", () => { @@ -93,3 +100,43 @@ test("empty/missing inputs do not crash", () => { assert.equal(v.ok, true); assert.deepEqual(v.onlyFresh, []); }); + +// ─── ENOSPC guard ────────────────────────────────────────────────────────────── +// +// The v3.8.50 publish run (CI 33104507735) failed with "15 tables a CLEAN install creates +// but an UPGRADE does not". None of them was missing: the Phase B upgrade install had hit +// `npm warn tar TAR_ENTRY_ERROR ENOSPC: no space left on device` 5611 times, npm still +// exited 0, the truncated `omniroute serve` "exited with code 0 before serving", and the +// database therefore still held the 3.8.49 schema. npm reporting disk exhaustion as a +// warning is what let a full disk masquerade as a schema defect. + +test("npm ENOSPC warnings are raised as an install failure, not ignored", () => { + const enospc = "npm warn tar TAR_ENTRY_ERROR ENOSPC: no space left on device, write\n".repeat(3); + assert.throws( + () => assertNoDiskExhaustion(enospc, "upgrade install"), + (err: Error) => { + assert.match(err.message, /upgrade install/); + assert.match(err.message, /ran out of disk space/); + assert.match(err.message, /3 ENOSPC error/); + // The operator must not go looking for a migration that is not missing. + assert.match(err.message, /NOT a schema divergence/); + return true; + } + ); +}); + +test("a clean install log does not trip the disk guard", () => { + assert.doesNotThrow(() => + assertNoDiskExhaustion( + "npm warn deprecated boolean@3.2.0: Package no longer supported.\nadded 900 packages\n", + "clean install" + ) + ); +}); + +test("the guard also catches the bare kernel message without the ENOSPC code", () => { + assert.throws( + () => assertNoDiskExhaustion("Error: no space left on device", "clean install"), + /ran out of disk space/ + ); +}); diff --git a/tests/unit/db-install-upgrade-schema-parity.test.ts b/tests/unit/db-install-upgrade-schema-parity.test.ts new file mode 100644 index 0000000000..9ce6c408c4 --- /dev/null +++ b/tests/unit/db-install-upgrade-schema-parity.test.ts @@ -0,0 +1,118 @@ +// ENVIRONMENT NOTE (sandbox better-sqlite3 / glibc limitation, not a code defect): +// This test opens a real SQLite database through `src/lib/db/core.ts`. better-sqlite3 is a +// native addon; production and CI load it normally, but some sandboxes ship a system glibc +// older than the prebuilt binary requires ("GLIBC_2.29 not found"), in which case the +// runtime cascades to node:sqlite/sql.js. See tests/unit/_helpers/betterSqlite3Availability.ts. +// +// WHY THIS FILE EXISTS +// -------------------- +// `npm run check:install-upgrade` (scripts/check/check-install-upgrade.mjs) proves that a +// CLEAN install and an UPGRADE converge on the same schema, but it costs a full `npm pack` +// plus three global installs and three boots (~17 min in CI) and it can only ever run at +// publish time, against an already-published previous version. It is not a development +// feedback loop, and the v3.8.50 publish run is what proved it: the gate reported 15 +// "missing" tables, and the deterministic half of that verdict was never checkable locally. +// +// This file pins the deterministic half in milliseconds: +// +// 1. Every migration file on disk is actually reachable by the runner on a fresh install. +// A file the runner never applies is a table no user ever gets. +// 2. `model_capabilities` is created by the MIGRATION SET, not lazily at runtime. +// A table created on demand by `CREATE TABLE IF NOT EXISTS` inside a feature code +// path exists or not depending on whether that feature happened to run before the +// snapshot — so it diverges between the two install paths by TIMING, not by schema. +// That is precisely how `model_capabilities` surfaced as a divergence on the v3.8.50 +// publish run (present in the upgraded database, absent from the clean one). +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const MIGRATIONS_DIR = path.join(REPO_ROOT, "src", "lib", "db", "migrations"); + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-install-upgrade-parity-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const core = await import("../../src/lib/db/core.ts"); + +// A clean install: core.ts applies the inline SCHEMA_SQL, the `ensure*Columns()` helpers, +// then runMigrations(). This is the exact code path Phase A of the gate exercises. +const db = core.getDbInstance(); + +test.after(() => { + try { + core.resetDbInstance(); + } catch { + /* best effort */ + } + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +function migrationFiles(): Array<{ version: string; name: string }> { + return fs + .readdirSync(MIGRATIONS_DIR) + .sort() + .map((file) => /^(\d{3,})_(.+)\.sql$/.exec(file)) + .filter((m): m is RegExpExecArray => m !== null) + .map((m) => ({ version: m[1], name: m[2] })); +} + +function hasTable(name: string): boolean { + return Boolean( + db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) + ); +} + +test("a clean install applies every migration file on disk", () => { + const ledger = new Set( + ( + db.prepare("SELECT version FROM _omniroute_migrations").all() as Array<{ version: string }> + ).map((row) => row.version) + ); + const unapplied = migrationFiles() + .filter((m) => !ledger.has(m.version)) + .map((m) => `${m.version}_${m.name}`); + assert.deepEqual( + unapplied, + [], + "migration files the runner never applied — an upgrade would not create their tables either" + ); +}); + +test("model_capabilities comes from the migration set, not from a lazy runtime CREATE", () => { + assert.ok( + hasTable("model_capabilities"), + "model_capabilities must be created by a migration so a clean install and an upgrade " + + "converge deterministically instead of depending on whether the models.dev sync ran" + ); +}); + +test("the model_capabilities migration does not drift from ensureCapabilitiesTable()", () => { + const source = fs.readFileSync(path.join(REPO_ROOT, "src", "lib", "modelsDevSync.ts"), "utf8"); + const ddl = /CREATE TABLE IF NOT EXISTS model_capabilities\s*\(([\s\S]*?)\n\s*\)/.exec(source); + assert.ok(ddl, "ensureCapabilitiesTable() DDL not found in src/lib/modelsDevSync.ts"); + + const runtimeColumns = ddl[1] + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0 && !/^PRIMARY KEY/i.test(line)) + .map((line) => line.replace(/,$/, "").split(/\s+/)[0]) + .sort(); + + const migrationColumns = ( + db.prepare("PRAGMA table_info(model_capabilities)").all() as Array<{ name: string }> + ) + .map((column) => column.name) + .sort(); + + assert.deepEqual( + migrationColumns, + runtimeColumns, + "the migration and the runtime helper must create the same columns — a drift here means " + + "an upgraded database keeps the old shape while a clean install gets the new one" + ); +}); diff --git a/tests/unit/npm-payload-nft-manifests-excluded.test.ts b/tests/unit/npm-payload-nft-manifests-excluded.test.ts new file mode 100644 index 0000000000..4ab0ff0936 --- /dev/null +++ b/tests/unit/npm-payload-nft-manifests-excluded.test.ts @@ -0,0 +1,72 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync, readdirSync } from "node:fs"; +import type { Dirent } from "node:fs"; +import { join } from "node:path"; + +/** + * v3.8.50 was refused by the registry with `413 Payload Too Large` on + * `POST /-/stage/package/omniroute`: the tarball had reached 288.7 MB packed + * (1.1 GB unpacked), against 174.5 MB for the 3.8.49 that published fine. + * + * 668.7 MB of that — 61% of the whole package — was 842 `*.nft.json` files. + * Those are Next.js Node File Trace manifests: build-time metadata used to + * COMPUTE the standalone bundle, never read while serving. They had doubled + * since 3.8.49 (325.0 MB across 748 files), which is what tipped the payload + * over the limit. + * + * The guard is the `files[]` negation, so a future entry that re-widens the + * glob (or a rewrite of the array) cannot silently put them back. + */ +const pkg = JSON.parse(readFileSync(join(import.meta.dirname, "../../package.json"), "utf8")) as { + files?: string[]; +}; + +test("package.json files[] excludes Next's .nft.json trace manifests", () => { + const files = pkg.files ?? []; + assert.ok(files.length > 0, "package.json must declare files[]"); + assert.ok( + files.includes("!**/*.nft.json"), + "files[] must negate **/*.nft.json — they are build metadata and were 61% of the 3.8.50 payload" + ); +}); + +test("the negation sits after the positive dist/ entry it has to override", () => { + // npm applies files[] in order: a negation listed BEFORE the directory that + // pulls the files in is a no-op. Positive anchor, so this test cannot pass + // just because both strings happen to be present somewhere. + const files = pkg.files ?? []; + const dist = files.indexOf("dist/"); + const negation = files.indexOf("!**/*.nft.json"); + assert.notEqual(dist, -1, "dist/ must still be published"); + assert.ok(negation > dist, "the .nft.json negation must come after dist/"); +}); + +test("no source module reads a .nft.json at runtime", () => { + // If this ever stops holding, the exclusion above becomes a runtime break + // rather than a size win — which is exactly the assumption worth pinning. + const roots = ["src", "open-sse", "bin"]; + const hits: string[] = []; + for (const root of roots) { + const dir = join(import.meta.dirname, "../..", root); + const stack = [dir]; + while (stack.length > 0) { + const current = stack.pop() as string; + let entries: Dirent[]; + try { + entries = readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = join(current, entry.name); + if (entry.isDirectory()) { + if (entry.name !== "node_modules") stack.push(full); + } else if (/\.(ts|tsx|mjs|js)$/.test(entry.name)) { + if (readFileSync(full, "utf8").includes(".nft.json")) hits.push(full); + } + } + } + } + assert.deepEqual(hits, [], `nothing may depend on .nft.json at runtime: ${hits.join(", ")}`); +});