mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-16 20:22:21 +03:00
perf(electron): build the Next standalone once and hydrate natives per leg (#10321 stage 8) (#10390)
The desktop release matrix ran the full Next.js standalone build on all four legs (windows, macos-intel, macos-arm64, linux), duplicating the platform-neutral majority of that work four times and re-exposing every leg to the hosted-runner RAM class of failure that took the linux leg out of v3.8.49. - scripts/build/standaloneTarball.mjs: deterministic, dependency-free tar.gz writer/reader (uid/gid/mtime pinned, sorted entries, symlink + exec-bit preservation; GNU-tar interop covered by tests). - scripts/build/standaloneManifest.mjs: byte-level manifest of .build/next (sha256 + size + symlink target per entry, plus the archive's own digest) catching artifact-transfer corruption before extraction and re-verifying the restored tree byte-for-byte, smuggling included. - scripts/build/standaloneBundle.mjs: pack / restore / hydrate CLI over the two modules above. - scripts/build/hydrateNativeDeps.mjs: swaps install-machine-forked native optionals (@img/sharp-*, @ngrok/ngrok-*, fsevents) from the leg's own npm ci into the restored tree, then verifies the bundled-native closure (koffi triplets, better-sqlite3 prebuilds, wreq-js, onnxruntime with its documented darwin-x64 exemption) services the leg's platform/arch before packaging starts. - .github/workflows/electron-release.yml: new web-build job builds the standalone once on ubuntu with webpack and uploads the bundle; legs download, restore, and hydrate it, skipping the per-leg build. The legacy per-leg build remains as a rollback path via the ELECTRON_SHARED_STANDALONE workflow_dispatch input, and legs fail closed if web-build ran and failed. Regression tests cover archive roundtrip, byte determinism, manifest tamper/smuggle detection, forked-native swaps, and native-closure serviceability.
This commit is contained in:
backryun
106
.github/workflows/electron-release.yml
vendored
106
.github/workflows/electron-release.yml
vendored
@@ -55,9 +55,75 @@ jobs:
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "✓ Valid version: $VERSION"
|
||||
|
||||
web-build:
|
||||
name: Build shared Next standalone
|
||||
needs: validate
|
||||
# Stage 8 (issue #10321): the four desktop legs used to each run the full
|
||||
# `npm run build` (Next standalone) — ~111 runner-minutes per release just to
|
||||
# produce the same platform-independent bundle four times. This job builds it
|
||||
# once on ubuntu; every leg then restores the byte-verified archive and
|
||||
# re-forks its native optionals (scripts/build/standaloneBundle.mjs).
|
||||
#
|
||||
# Rollback lever: set the repo variable ELECTRON_SHARED_STANDALONE=disabled.
|
||||
# This job then skips, every leg falls back to building its own web bundle
|
||||
# (the legacy step below), and the pipeline behaves exactly like pre-Stage 8 —
|
||||
# no revert needed.
|
||||
if: ${{ !cancelled() && needs.validate.result == 'success' && vars.ELECTRON_SHARED_STANDALONE != 'disabled' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
env:
|
||||
NPM_CONFIG_LEGACY_PEER_DEPS: true
|
||||
|
||||
- name: Build Next.js standalone
|
||||
# webpack, not Turbopack, for the same hosted-runner RAM reason as the
|
||||
# linux leg (see the long comment on the fallback step in `build`).
|
||||
env:
|
||||
JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
|
||||
NODE_OPTIONS: "--max_old_space_size=6144"
|
||||
OMNIROUTE_USE_TURBOPACK: "0"
|
||||
run: npm run build
|
||||
|
||||
- name: Pack standalone bundle
|
||||
# Deterministic tar.gz + byte-level manifest; the manifest embeds the
|
||||
# archive's own sha256 so artifact-transfer corruption is caught before
|
||||
# extraction, and every entry is re-verified after extraction.
|
||||
run: node scripts/build/standaloneBundle.mjs pack --out web-bundle.tar.gz
|
||||
|
||||
- name: Upload shared web bundle
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: web-standalone-bundle
|
||||
# compression-level 0: the payload is already a deterministic tar.gz;
|
||||
# re-zipping would only burn runner CPU without shrinking it further.
|
||||
compression-level: 0
|
||||
# Legs consume this within minutes; no reason to retain it like the
|
||||
# installer artifacts (default 90d).
|
||||
retention-days: 3
|
||||
path: |
|
||||
web-bundle.tar.gz
|
||||
web-bundle.tar.gz.manifest.json
|
||||
|
||||
build:
|
||||
name: Build Electron (${{ matrix.platform }})
|
||||
needs: validate
|
||||
needs: [validate, web-build]
|
||||
# `web-build` is skipped when ELECTRON_SHARED_STANDALONE=disabled (rollback
|
||||
# mode); legs then run the legacy per-leg web build below. If it ran and
|
||||
# failed, fail closed: legs cannot package without the bundle, and silently
|
||||
# falling back to four per-leg builds would hide exactly the regression the
|
||||
# shared job exists to surface.
|
||||
if: ${{ !cancelled() && needs.validate.result == 'success' && (needs.web-build.result == 'success' || needs.web-build.result == 'skipped') }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: write # electron-builder may publish artifacts with GH_TOKEN
|
||||
@@ -69,19 +135,27 @@ jobs:
|
||||
runner: windows-latest
|
||||
target: win
|
||||
ext: .exe
|
||||
os: win32
|
||||
arch: x64
|
||||
- platform: macos-intel
|
||||
runner: macos-15-intel
|
||||
target: mac-x64
|
||||
ext: .dmg
|
||||
os: darwin
|
||||
arch: x64
|
||||
- platform: macos-arm64
|
||||
runner: macos-latest
|
||||
target: mac-arm64
|
||||
ext: -arm64.dmg
|
||||
os: darwin
|
||||
arch: arm64
|
||||
- platform: linux
|
||||
runner: ubuntu-latest
|
||||
target: linux
|
||||
ext: .AppImage
|
||||
deb_ext: .deb
|
||||
os: linux
|
||||
arch: x64,arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
@@ -108,7 +182,11 @@ jobs:
|
||||
mkdir -p "$RUNNER_TEMP/home"
|
||||
echo "USERPROFILE=$RUNNER_TEMP/home" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Next.js standalone
|
||||
- name: Build Next.js standalone (legacy per-leg fallback)
|
||||
# Stage 8: only runs in rollback mode (ELECTRON_SHARED_STANDALONE=disabled)
|
||||
# or when the shared web-build job was skipped. Otherwise the leg restores
|
||||
# the shared bundle from the `web-build` job below.
|
||||
if: needs.web-build.result == 'skipped'
|
||||
env:
|
||||
JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
|
||||
NODE_OPTIONS: "--max_old_space_size=6144"
|
||||
@@ -126,6 +204,30 @@ jobs:
|
||||
OMNIROUTE_USE_TURBOPACK: ${{ matrix.platform == 'linux' && '0' || '1' }}
|
||||
run: npm run build
|
||||
|
||||
- name: Download shared web bundle
|
||||
# Stage 8: inverse of the fallback step above — runs exactly when the
|
||||
# shared `web-build` job produced the bundle.
|
||||
if: needs.web-build.result == 'success'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: web-standalone-bundle
|
||||
|
||||
- name: Restore + hydrate shared web bundle
|
||||
if: needs.web-build.result == 'success'
|
||||
shell: bash
|
||||
# restore: verify the archive's sha256 against the manifest, extract, then
|
||||
# re-verify every entry (existence + size + content hash + symlink
|
||||
# targets, and no unlisted files) byte-for-byte.
|
||||
# hydrate: the bundle was built on ubuntu, so install-machine-forked native
|
||||
# optionals (@img/sharp-*, @img/sharp-libvips-*, @ngrok/ngrok-*,
|
||||
# fsevents) carry linux forks. Replace them with the forks this
|
||||
# leg's own `npm ci` resolved, then assert every bundled native
|
||||
# (koffi triplets, better-sqlite3 prebuilds, wreq-js, onnxruntime)
|
||||
# can service this leg's platform/arch before packaging starts.
|
||||
run: |
|
||||
node scripts/build/standaloneBundle.mjs restore --archive web-bundle.tar.gz
|
||||
node scripts/build/standaloneBundle.mjs hydrate --platform ${{ matrix.os }} --arch ${{ matrix.arch }}
|
||||
|
||||
- name: Sync version in electron/package.json
|
||||
shell: bash
|
||||
env:
|
||||
|
||||
137
scripts/build/hydrateNativeDeps.mjs
Normal file
137
scripts/build/hydrateNativeDeps.mjs
Normal file
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Platform hydration for the shared Next standalone web build (issue #10321,
|
||||
* Stage 8).
|
||||
*
|
||||
* The standalone bundle is built ONCE on ubuntu and restored on every desktop
|
||||
* matrix leg. Everything except install-machine-forked optional packages is
|
||||
* platform-independent:
|
||||
*
|
||||
* - Bundled-for-all (verify only): koffi ships every triplet under
|
||||
* `build/koffi/<os>_<arch>`, better-sqlite3 v13 ships Node-API prebuilds for
|
||||
* 8 platforms, wreq-js ships `rust/wreq-js.<plat>-<arch>[-libc].node`, and
|
||||
* onnxruntime-node ships `bin/napi-v6/<os>/<arch>`.
|
||||
* - Install-machine-forked (hydrate): `@img/sharp-*`, `@img/sharp-libvips-*`,
|
||||
* `@ngrok/ngrok-*` and macOS-only `fsevents` resolve to whichever platform
|
||||
* ran `npm ci`. The ubuntu-built tree carries the linux forks; each leg
|
||||
* replaces them with the forks from its OWN `npm ci`d node_modules.
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
/** Scope prefixes whose members are install-machine-forked. */
|
||||
export const HYDRATED_SCOPES = ["@img/sharp-", "@img/sharp-libvips-", "@ngrok/ngrok-"];
|
||||
|
||||
/** Standalone packages that are not forked but must never be platform-forked. */
|
||||
export const HYDRATED_ROOT_PACKAGES = ["fsevents"];
|
||||
|
||||
/**
|
||||
* onnxruntime-node does not publish a darwin-x64 binary for napi-v6 (only
|
||||
* linux/win32 x64 + darwin arm64), so existence cannot be asserted there.
|
||||
*/
|
||||
export const BUNDLED_EXEMPTIONS = new Set(["onnxruntime-node:darwin-x64"]);
|
||||
|
||||
function platformTriple(platform, arch) {
|
||||
// koffi uses underscore triplets; better-sqlite3/wreq-js/onnx use dashes.
|
||||
return { koffi: `${platform}_${arch}`, dash: `${platform}-${arch}` };
|
||||
}
|
||||
|
||||
function rmrf(target) {
|
||||
fs.rmSync(target, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function copyDir(from, to) {
|
||||
fs.cpSync(from, to, { recursive: true, verbatimSymlinks: false, force: true });
|
||||
}
|
||||
|
||||
function directMemberNames(nodeModulesDir, scope) {
|
||||
const scopeDir = path.join(nodeModulesDir, ...scope.split("/").slice(0, -1));
|
||||
const prefix = scope.split("/").pop();
|
||||
try {
|
||||
return fs
|
||||
.readdirSync(scopeDir)
|
||||
.filter((name) => name.startsWith(prefix))
|
||||
.map((name) => `${scope.slice(0, scope.lastIndexOf("/"))}/${name}`);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace install-machine-forked packages inside the restored standalone tree
|
||||
* with the forks resolved by THIS machine's node_modules.
|
||||
*
|
||||
* @param {{standaloneNodeModules: string, sourceNodeModules: string}} opts
|
||||
* @returns {{replaced: string[], removed: string[], copied: string[]}}
|
||||
*/
|
||||
export function hydratePlatformNatives({ standaloneNodeModules, sourceNodeModules }) {
|
||||
const replaced = [];
|
||||
const removed = [];
|
||||
const copied = [];
|
||||
|
||||
const forkedNames = new Set();
|
||||
for (const scope of HYDRATED_SCOPES) {
|
||||
for (const name of directMemberNames(sourceNodeModules, scope)) forkedNames.add(name);
|
||||
for (const name of directMemberNames(standaloneNodeModules, scope)) forkedNames.add(name);
|
||||
}
|
||||
for (const pkg of HYDRATED_ROOT_PACKAGES) {
|
||||
if (fs.existsSync(path.join(sourceNodeModules, pkg))) forkedNames.add(pkg);
|
||||
if (fs.existsSync(path.join(standaloneNodeModules, pkg))) forkedNames.add(pkg);
|
||||
}
|
||||
|
||||
for (const name of forkedNames) {
|
||||
const standalonePath = path.join(standaloneNodeModules, ...name.split("/"));
|
||||
const sourcePath = path.join(sourceNodeModules, ...name.split("/"));
|
||||
const hadIt = fs.existsSync(standalonePath);
|
||||
const hasIt = fs.existsSync(sourcePath);
|
||||
if (hadIt) rmrf(standalonePath);
|
||||
if (!hasIt) {
|
||||
if (hadIt) removed.push(name);
|
||||
continue; // e.g. fsevents on non-darwin legs: simply absent everywhere.
|
||||
}
|
||||
copyDir(sourcePath, standalonePath);
|
||||
copied.push(name);
|
||||
if (hadIt) replaced.push(name);
|
||||
}
|
||||
return { replaced, removed, copied };
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that every bundled native dependency can service `platform`/`arch`.
|
||||
*
|
||||
* @returns {{ok: true} | {ok: false, errors: string[]}}
|
||||
*/
|
||||
export function verifyBundledNatives({ nodeModulesDir, platform, arch }) {
|
||||
const errors = [];
|
||||
const triple = platformTriple(platform, arch);
|
||||
|
||||
const koffiDir = path.join(nodeModulesDir, "koffi", "build", "koffi", triple.koffi);
|
||||
if (!fs.existsSync(koffiDir)) errors.push(`koffi: missing bundled triplet ${triple.koffi}`);
|
||||
|
||||
const sqlitePrebuild = path.join(
|
||||
nodeModulesDir,
|
||||
"better-sqlite3",
|
||||
"prebuilds",
|
||||
`${triple.dash}.node`
|
||||
);
|
||||
if (!fs.existsSync(sqlitePrebuild))
|
||||
errors.push(`better-sqlite3: missing prebuild ${triple.dash}.node`);
|
||||
|
||||
const wreqDir = path.join(nodeModulesDir, "wreq-js", "rust");
|
||||
const wreqNames = fs.existsSync(wreqDir)
|
||||
? fs
|
||||
.readdirSync(wreqDir)
|
||||
.filter((n) => n.startsWith(`wreq-js.${triple.dash}`) && n.endsWith(".node"))
|
||||
: [];
|
||||
if (wreqNames.length === 0) errors.push(`wreq-js: missing rust binary for ${triple.dash}`);
|
||||
|
||||
const exempt = BUNDLED_EXEMPTIONS.has(`onnxruntime-node:${triple.dash}`);
|
||||
if (!exempt) {
|
||||
const onnxDir = path.join(nodeModulesDir, "onnxruntime-node", "bin", "napi-v6", platform, arch);
|
||||
if (!fs.existsSync(onnxDir))
|
||||
errors.push(`onnxruntime-node: missing ${platform}/${arch} binary`);
|
||||
}
|
||||
|
||||
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
||||
}
|
||||
221
scripts/build/standaloneBundle.mjs
Normal file
221
scripts/build/standaloneBundle.mjs
Normal file
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* CLI entry for the shared Next standalone web build (issue #10321, Stage 8).
|
||||
*
|
||||
* One ubuntu `web-build` job runs `pack` once; every desktop matrix leg runs
|
||||
* `restore` (byte-verified against the manifest) and `hydrate` (replaces
|
||||
* install-machine-forked native optionals with this leg's own `npm ci` forks,
|
||||
* then asserts the bundled natives can service the leg's platform/arch).
|
||||
*
|
||||
* Rollback: set repo variable ELECTRON_SHARED_STANDALONE=disabled and the
|
||||
* workflow falls back to the legacy per-leg `npm run build` — no revert needed.
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import {
|
||||
buildStandaloneManifest,
|
||||
verifyStandaloneManifest,
|
||||
MANIFEST_VERSION,
|
||||
} from "./standaloneManifest.mjs";
|
||||
import { createTarGz, extractTarGz } from "./standaloneTarball.mjs";
|
||||
import { hydratePlatformNatives, verifyBundledNatives } from "./hydrateNativeDeps.mjs";
|
||||
|
||||
function sha256File(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash("sha256");
|
||||
const stream = createReadStream(filePath);
|
||||
stream.on("data", (chunk) => hash.update(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", () => resolve(hash.digest("hex")));
|
||||
});
|
||||
}
|
||||
|
||||
function manifestPathFor(archive) {
|
||||
return `${archive}.manifest.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack a web-build tree into a deterministic archive plus a byte-level
|
||||
* manifest (which embeds the archive's own sha256 so transfer corruption is
|
||||
* caught before extraction).
|
||||
*
|
||||
* @param {{dir?: string, out: string, manifest?: string}} opts
|
||||
* @returns {Promise<{archive: string, manifest: string, files: number, archiveBytes: number}>}
|
||||
*/
|
||||
export async function runPack({ dir = ".build/next", out, manifest }) {
|
||||
if (!out) throw new Error("pack requires --out <file.tar.gz>");
|
||||
const rootDir = path.resolve(dir);
|
||||
if (!fs.existsSync(rootDir)) {
|
||||
throw new Error(`web build tree not found: ${rootDir} (did 'npm run build' run?)`);
|
||||
}
|
||||
fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
|
||||
|
||||
const built = await buildStandaloneManifest(rootDir);
|
||||
await createTarGz(rootDir, out);
|
||||
const archiveBytes = fs.statSync(out).size;
|
||||
const archiveSha = await sha256File(out);
|
||||
|
||||
const manifestFile = manifest ?? manifestPathFor(out);
|
||||
const payload = {
|
||||
version: MANIFEST_VERSION,
|
||||
archive: { name: path.basename(out), bytes: archiveBytes, sha256: archiveSha },
|
||||
entries: built.entries,
|
||||
};
|
||||
fs.writeFileSync(manifestFile, `${JSON.stringify(payload, null, 2)}\n`);
|
||||
return { archive: out, manifest: manifestFile, files: built.entries.length, archiveBytes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify + extract a packed archive into `dir`, then prove the restored tree
|
||||
* matches the manifest byte-for-byte.
|
||||
*
|
||||
* @param {{archive: string, manifest?: string, dir?: string}} opts
|
||||
* @returns {Promise<{archive: string, dir: string, files: number}>}
|
||||
*/
|
||||
export async function runRestore({ archive, manifest, dir = ".build/next" }) {
|
||||
if (!archive) throw new Error("restore requires --archive <file.tar.gz>");
|
||||
const manifestFile = manifest ?? manifestPathFor(archive);
|
||||
const raw = JSON.parse(fs.readFileSync(manifestFile, "utf8"));
|
||||
if (raw.version !== MANIFEST_VERSION) {
|
||||
throw new Error(`unsupported manifest version: ${raw.version}`);
|
||||
}
|
||||
|
||||
const archiveBytes = fs.statSync(archive).size;
|
||||
if (archiveBytes !== raw.archive.bytes) {
|
||||
throw new Error(`archive size ${archiveBytes} != manifest ${raw.archive.bytes}`);
|
||||
}
|
||||
const archiveSha = await sha256File(archive);
|
||||
if (archiveSha !== raw.archive.sha256) {
|
||||
throw new Error(`archive sha256 mismatch (expected ${raw.archive.sha256.slice(0, 12)})`);
|
||||
}
|
||||
|
||||
const destDir = path.resolve(dir);
|
||||
fs.rmSync(destDir, { recursive: true, force: true });
|
||||
await extractTarGz(archive, destDir);
|
||||
|
||||
const verdict = await verifyStandaloneManifest(destDir, raw);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`restored tree failed manifest verification:\n ${verdict.errors.join("\n ")}`
|
||||
);
|
||||
}
|
||||
return { archive, dir: destDir, files: raw.entries.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Hydrate the restored tree's node_modules with this machine's forked
|
||||
* optionals and assert bundled natives cover every requested arch.
|
||||
*
|
||||
* @param {{standaloneNodeModules?: string, sourceNodeModules?: string, platform: string, arch: string}} opts
|
||||
* `arch` accepts a comma-separated list (the linux leg ships x64+arm64).
|
||||
* @returns {Promise<{replaced: string[], removed: string[], copied: string[], verified: string[]}>}
|
||||
*/
|
||||
export async function runHydrate({
|
||||
standaloneNodeModules = ".build/next/standalone/node_modules",
|
||||
sourceNodeModules = "node_modules",
|
||||
platform,
|
||||
arch,
|
||||
}) {
|
||||
if (!platform || !arch) throw new Error("hydrate requires --platform <os> --arch <a[,a2...]>");
|
||||
const result = hydratePlatformNatives({
|
||||
standaloneNodeModules: path.resolve(standaloneNodeModules),
|
||||
sourceNodeModules: path.resolve(sourceNodeModules),
|
||||
});
|
||||
const verified = [];
|
||||
for (const one of arch
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)) {
|
||||
const verdict = verifyBundledNatives({
|
||||
nodeModulesDir: path.resolve(standaloneNodeModules),
|
||||
platform,
|
||||
arch: one,
|
||||
});
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`bundled natives cannot service ${platform}/${one}:\n ${verdict.errors.join("\n ")}`
|
||||
);
|
||||
}
|
||||
verified.push(one);
|
||||
}
|
||||
return { ...result, verified };
|
||||
}
|
||||
|
||||
// ─── argv plumbing ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** Minimal `--key value` parser (booleans: `--key` alone → true). */
|
||||
export function parseArgs(argv) {
|
||||
const opts = { _: [] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const token = argv[i];
|
||||
if (!token.startsWith("--")) {
|
||||
opts._.push(token);
|
||||
continue;
|
||||
}
|
||||
const key = token.slice(2);
|
||||
const next = argv[i + 1];
|
||||
if (next !== undefined && !next.startsWith("--")) {
|
||||
opts[key] = next;
|
||||
i++;
|
||||
} else {
|
||||
opts[key] = true;
|
||||
}
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"usage:",
|
||||
" standaloneBundle.mjs pack --out <file.tar.gz> [--dir .build/next] [--manifest <file.json>]",
|
||||
" standaloneBundle.mjs restore --archive <file.tar.gz> [--manifest <file.json>] [--dir .build/next]",
|
||||
" standaloneBundle.mjs hydrate --platform <os> --arch <a[,a2...]>",
|
||||
" [--standalone-node-modules <dir>] [--source-node-modules <dir>]",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const [command = "", ...rest] = argv;
|
||||
const opts = parseArgs(rest);
|
||||
try {
|
||||
if (command === "pack") {
|
||||
const r = await runPack({ dir: opts.dir, out: opts.out, manifest: opts.manifest });
|
||||
console.log(
|
||||
`[standalone-bundle] packed ${r.files} entries -> ${r.archive} ` +
|
||||
`(${(r.archiveBytes / 1e6).toFixed(1)} MB); manifest ${r.manifest}`
|
||||
);
|
||||
} else if (command === "restore") {
|
||||
const r = await runRestore({ archive: opts.archive, manifest: opts.manifest, dir: opts.dir });
|
||||
console.log(
|
||||
`[standalone-bundle] restored ${r.files} entries from ${path.basename(r.archive)} -> ${r.dir}`
|
||||
);
|
||||
} else if (command === "hydrate") {
|
||||
const r = await runHydrate({
|
||||
standaloneNodeModules: opts["standalone-node-modules"],
|
||||
sourceNodeModules: opts["source-node-modules"],
|
||||
platform: opts.platform,
|
||||
arch: opts.arch,
|
||||
});
|
||||
console.log(
|
||||
`[standalone-bundle] hydrated forks: copied=${r.copied.length} replaced=${r.replaced.length} ` +
|
||||
`removed=${r.removed.length}; bundled natives verified for ${r.verified.join("+")}`
|
||||
);
|
||||
} else {
|
||||
console.error(usage());
|
||||
process.exitCode = 2;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[standalone-bundle] ${command || "(no command)"} failed: ${err.message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
import.meta.url === new URL(`file://${path.resolve(process.argv[1])}`).href
|
||||
) {
|
||||
await main(process.argv.slice(2));
|
||||
}
|
||||
132
scripts/build/standaloneManifest.mjs
Normal file
132
scripts/build/standaloneManifest.mjs
Normal file
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Byte-level manifest for the shared Next standalone web build (issue #10321,
|
||||
* Stage 8).
|
||||
*
|
||||
* The desktop pipeline used to rebuild the identical Next standalone bundle
|
||||
* four times (one per electron-release matrix leg). Stage 8 builds it once on
|
||||
* an ubuntu runner and restores it on every leg; this module is the integrity
|
||||
* contract that makes a restored tree provably identical to the built one.
|
||||
*
|
||||
* Deterministic by construction: entries are sorted by path, timestamps are
|
||||
* never recorded, and symlinks are pinned by their target so a restored tree
|
||||
* verifies even though tar extraction rewrites mtimes.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export const MANIFEST_VERSION = 1;
|
||||
|
||||
/** Streamed sha256 for large native payloads (onnxruntime is ~200 MB). */
|
||||
async function sha256File(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash("sha256");
|
||||
const stream = createReadStream(filePath);
|
||||
stream.on("data", (chunk) => hash.update(chunk));
|
||||
stream.on("error", reject);
|
||||
stream.on("end", () => resolve(hash.digest("hex")));
|
||||
});
|
||||
}
|
||||
|
||||
function walkDir(root, current, entries) {
|
||||
const children = fs.readdirSync(current, { withFileTypes: true });
|
||||
// Sort for determinism: manifest of the same tree is byte-identical.
|
||||
children.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const child of children) {
|
||||
const abs = path.join(current, child.name);
|
||||
const rel = path.relative(root, abs).split(path.sep).join("/");
|
||||
if (child.isSymbolicLink()) {
|
||||
entries.push({ path: rel, symlink: fs.readlinkSync(abs) });
|
||||
} else if (child.isDirectory()) {
|
||||
walkDir(root, abs, entries);
|
||||
} else if (child.isFile()) {
|
||||
entries.push({ path: rel, file: abs });
|
||||
}
|
||||
// Other node types (fifo/socket) never appear in build output; ignoring
|
||||
// them keeps the manifest shape minimal.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a manifest of every file and symlink under `rootDir`.
|
||||
*
|
||||
* @returns {Promise<{version: number, entries: {path: string, bytes: number, sha256: string, symlink?: string}[]}>}
|
||||
*/
|
||||
export async function buildStandaloneManifest(rootDir) {
|
||||
const entries = [];
|
||||
walkDir(rootDir, rootDir, entries);
|
||||
const manifestEntries = [];
|
||||
for (const entry of entries) {
|
||||
if (entry.symlink !== undefined) {
|
||||
manifestEntries.push({ path: entry.path, bytes: 0, sha256: "", symlink: entry.symlink });
|
||||
continue;
|
||||
}
|
||||
const stat = fs.statSync(entry.file);
|
||||
manifestEntries.push({
|
||||
path: entry.path,
|
||||
bytes: stat.size,
|
||||
sha256: await sha256File(entry.file),
|
||||
});
|
||||
}
|
||||
manifestEntries.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
||||
return { version: MANIFEST_VERSION, entries: manifestEntries };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify a restored tree against a manifest built by `buildStandaloneManifest`.
|
||||
* Checks existence, size, and content hash of every entry, plus that no
|
||||
* unlisted files were smuggled in.
|
||||
*
|
||||
* @returns {Promise<{ok: true} | {ok: false, errors: string[]}>}
|
||||
*/
|
||||
export async function verifyStandaloneManifest(rootDir, manifest) {
|
||||
const errors = [];
|
||||
if (!manifest || manifest.version !== MANIFEST_VERSION) {
|
||||
return { ok: false, errors: [`unsupported manifest version: ${manifest?.version}`] };
|
||||
}
|
||||
const listed = new Map(manifest.entries.map((e) => [e.path, e]));
|
||||
for (const entry of manifest.entries) {
|
||||
const abs = path.join(rootDir, ...entry.path.split("/"));
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.lstatSync(abs);
|
||||
} catch {
|
||||
errors.push(`${entry.path}: missing`);
|
||||
continue;
|
||||
}
|
||||
if (entry.symlink !== undefined) {
|
||||
if (!stat.isSymbolicLink()) {
|
||||
errors.push(`${entry.path}: expected symlink, found regular entry`);
|
||||
} else {
|
||||
const target = fs.readlinkSync(abs);
|
||||
if (target !== entry.symlink) {
|
||||
errors.push(`${entry.path}: symlink target ${target} != ${entry.symlink}`);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
errors.push(`${entry.path}: expected file, found directory/symlink`);
|
||||
continue;
|
||||
}
|
||||
if (stat.size !== entry.bytes) {
|
||||
errors.push(`${entry.path}: size ${stat.size} != ${entry.bytes}`);
|
||||
continue;
|
||||
}
|
||||
const digest = await sha256File(abs);
|
||||
if (digest !== entry.sha256) {
|
||||
errors.push(`${entry.path}: sha256 mismatch`);
|
||||
}
|
||||
}
|
||||
const actual = [];
|
||||
walkDir(rootDir, rootDir, actual);
|
||||
const actualPaths = new Set(actual.map((e) => e.path));
|
||||
for (const p of listed.keys()) actualPaths.delete(p);
|
||||
if (actualPaths.size > 0) {
|
||||
errors.push(`unlisted files: ${[...actualPaths].sort().slice(0, 5).join(", ")}`);
|
||||
}
|
||||
return errors.length === 0 ? { ok: true } : { ok: false, errors };
|
||||
}
|
||||
381
scripts/build/standaloneTarball.mjs
Normal file
381
scripts/build/standaloneTarball.mjs
Normal file
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Deterministic tar.gz primitives for the shared web build (issue #10321,
|
||||
* Stage 8).
|
||||
*
|
||||
* Why not shell out to system tar: the restore step runs on every desktop
|
||||
* matrix leg including Windows, where bsdtar's long-path behavior on deep
|
||||
* node_modules trees is not guaranteed. Node's fs layer already proves it can
|
||||
* produce and consume this exact tree on Windows today (the legacy per-leg
|
||||
* `npm run build` writes it with the same fs), so a pure-Node reader keeps the
|
||||
* extraction on the one path layer we know works.
|
||||
*
|
||||
* Format: ustar with GNU LongLink ('L') entries for paths > 100 chars,
|
||||
* typeflag '2' for symlinks, mtime/uid/gid zeroed and modes normalized to
|
||||
* 0644/0755 (exec bit only) so the archive of a given tree is byte-identical
|
||||
* on every machine.
|
||||
*/
|
||||
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { once } from "node:events";
|
||||
import { createGunzip, createGzip } from "node:zlib";
|
||||
|
||||
const BLOCK = 512;
|
||||
|
||||
function octal(value, length) {
|
||||
return value.toString(8).padStart(length - 1, "0") + "\0";
|
||||
}
|
||||
|
||||
function headerFor(name, size, typeflag, linkname = "", prefix = "", mode = 0o644) {
|
||||
const buf = Buffer.alloc(BLOCK, 0);
|
||||
buf.write(name.slice(0, 100), 0, 100, "utf8");
|
||||
buf.write(octal(typeflag === "5" ? 0o755 : mode, 8), 100);
|
||||
buf.write(octal(0, 8), 108); // uid
|
||||
buf.write(octal(0, 8), 116); // gid
|
||||
buf.write(octal(size, 12), 124);
|
||||
buf.write(octal(0, 12), 136); // mtime = 0 for determinism
|
||||
buf.write(" ", 148); // checksum placeholder: spaces
|
||||
buf.write(typeflag, 156);
|
||||
buf.write(linkname.slice(0, 100), 157, 100, "utf8");
|
||||
buf.write("ustar\0", 257, 6, "utf8");
|
||||
buf.write("00", 263, 2, "utf8");
|
||||
buf.write(prefix.slice(0, 155), 345, 155, "utf8");
|
||||
let sum = 0;
|
||||
for (const byte of buf) sum += byte;
|
||||
buf.write(sum.toString(8).padStart(6, "0") + "\0 ", 148);
|
||||
return buf;
|
||||
}
|
||||
|
||||
function dataPad(size) {
|
||||
const pad = (BLOCK - (size % BLOCK)) % BLOCK;
|
||||
return Buffer.alloc(pad, 0);
|
||||
}
|
||||
|
||||
function longLinkEntry(name) {
|
||||
const payload = Buffer.from(name + "\0", "utf8");
|
||||
return Buffer.concat([
|
||||
headerFor("././@LongLink", payload.length, "L"),
|
||||
payload,
|
||||
dataPad(payload.length),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Emit header (with LongLink/prefix handling) for one entry. */
|
||||
function entryHeader(relPath, size, typeflag, linkname, mode) {
|
||||
const out = [];
|
||||
if (relPath.length > 100) {
|
||||
const slash = relPath.slice(0, 155).lastIndexOf("/");
|
||||
const prefix = slash > 0 ? relPath.slice(0, slash) : "";
|
||||
const name = prefix ? relPath.slice(slash + 1) : relPath;
|
||||
if (name.length > 100) {
|
||||
out.push(longLinkEntry(relPath));
|
||||
name = relPath.slice(0, 100);
|
||||
}
|
||||
out.push(headerFor(name, size, typeflag, linkname, prefix, mode));
|
||||
} else {
|
||||
out.push(headerFor(relPath, size, typeflag, linkname, undefined, mode));
|
||||
}
|
||||
return Buffer.concat(out);
|
||||
}
|
||||
|
||||
function* walkFiles(root, current = root) {
|
||||
const children = fs
|
||||
.readdirSync(current, { withFileTypes: true })
|
||||
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const child of children) {
|
||||
const abs = path.join(current, child.name);
|
||||
const rel = path.relative(root, abs).split(path.sep).join("/");
|
||||
if (child.isSymbolicLink()) {
|
||||
yield { rel, symlink: fs.readlinkSync(abs) };
|
||||
} else if (child.isDirectory()) {
|
||||
yield* walkFiles(root, abs);
|
||||
} else if (child.isFile()) {
|
||||
yield { rel, abs };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Write a buffer, respecting gzip backpressure. */
|
||||
async function writeWithBackpressure(stream, buf) {
|
||||
if (!stream.write(buf)) await once(stream, "drain");
|
||||
}
|
||||
|
||||
/** Stream one file's bytes into the archive (no whole-file buffering). */
|
||||
function pipeFileInto(gz, failure, abs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const stream = createReadStream(abs, { autoClose: true });
|
||||
const onDrain = () => stream.resume();
|
||||
const detach = () => gz.removeListener("drain", onDrain);
|
||||
stream.on("error", (err) => {
|
||||
detach();
|
||||
reject(err);
|
||||
});
|
||||
stream.on("data", (chunk) => {
|
||||
if (!gz.write(chunk)) stream.pause();
|
||||
});
|
||||
gz.on("drain", onDrain);
|
||||
stream.on("end", () => {
|
||||
detach();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Pack `srcDir` into a deterministic gzipped tarball at `outFile`. */
|
||||
export async function createTarGz(srcDir, outFile) {
|
||||
const out = createWriteStream(outFile);
|
||||
const gz = createGzip({ level: 1 });
|
||||
gz.pipe(out);
|
||||
|
||||
const failure = new Promise((_, reject) => {
|
||||
gz.on("error", reject);
|
||||
out.on("error", reject);
|
||||
});
|
||||
|
||||
try {
|
||||
for (const entry of walkFiles(srcDir)) {
|
||||
if (entry.symlink !== undefined) {
|
||||
if (entry.symlink.length > 100) {
|
||||
throw new Error(`symlink target too long for ustar: ${entry.rel} -> ${entry.symlink}`);
|
||||
}
|
||||
await writeWithBackpressure(gz, entryHeader(entry.rel, 0, "2", entry.symlink));
|
||||
continue;
|
||||
}
|
||||
const st = fs.statSync(entry.abs);
|
||||
const size = st.size;
|
||||
const mode = st.mode & 0o111 ? 0o755 : 0o644;
|
||||
await writeWithBackpressure(gz, entryHeader(entry.rel, size, "0", undefined, mode));
|
||||
if (size > 0) await Promise.race([pipeFileInto(gz, failure, entry.abs), failure]);
|
||||
const pad = (BLOCK - (size % BLOCK)) % BLOCK;
|
||||
if (pad > 0) await writeWithBackpressure(gz, Buffer.alloc(pad, 0));
|
||||
}
|
||||
await writeWithBackpressure(gz, Buffer.alloc(BLOCK * 2, 0)); // terminator
|
||||
await Promise.race([
|
||||
new Promise((resolve, reject) => {
|
||||
out.on("finish", resolve);
|
||||
out.on("error", reject);
|
||||
gz.end();
|
||||
}),
|
||||
failure,
|
||||
]);
|
||||
} catch (err) {
|
||||
gz.destroy();
|
||||
out.destroy();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── extraction ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Promise-based byte source over a gunzip stream. `read(n)` waits until `n`
|
||||
* bytes are buffered (or EOF); `readSome()` returns whatever is available, for
|
||||
* streaming large payloads into files without whole-file buffering.
|
||||
*/
|
||||
class BlockSource {
|
||||
constructor(stream) {
|
||||
this.buffer = Buffer.alloc(0);
|
||||
this.error = null;
|
||||
this.ended = false;
|
||||
this.waiter = null;
|
||||
stream.on("data", (chunk) => {
|
||||
this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk]);
|
||||
this.notify();
|
||||
});
|
||||
stream.on("end", () => {
|
||||
this.ended = true;
|
||||
this.notify();
|
||||
});
|
||||
stream.on("error", (err) => {
|
||||
this.error = err;
|
||||
this.notify();
|
||||
});
|
||||
}
|
||||
|
||||
notify() {
|
||||
if (this.waiter) {
|
||||
const waiter = this.waiter;
|
||||
this.waiter = null;
|
||||
waiter();
|
||||
}
|
||||
}
|
||||
|
||||
readSome() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const attempt = () => {
|
||||
if (this.error) return reject(this.error);
|
||||
if (this.buffer.length > 0) {
|
||||
const out = this.buffer;
|
||||
this.buffer = Buffer.alloc(0);
|
||||
return resolve(out);
|
||||
}
|
||||
if (this.ended) return resolve(null);
|
||||
this.waiter = attempt;
|
||||
};
|
||||
attempt();
|
||||
});
|
||||
}
|
||||
|
||||
unshift(buf) {
|
||||
if (buf && buf.length > 0) this.buffer = Buffer.concat([buf, this.buffer]);
|
||||
}
|
||||
|
||||
async read(n) {
|
||||
let acc = null;
|
||||
let remaining = n;
|
||||
while (remaining > 0) {
|
||||
const chunk = await this.readSome();
|
||||
if (chunk === null) return null; // EOF before n bytes
|
||||
if (chunk.length > remaining) {
|
||||
acc = acc
|
||||
? Buffer.concat([acc, chunk.subarray(0, remaining)])
|
||||
: chunk.subarray(0, remaining);
|
||||
this.unshift(chunk.subarray(remaining));
|
||||
remaining = 0;
|
||||
} else {
|
||||
acc = acc ? Buffer.concat([acc, chunk]) : chunk;
|
||||
remaining -= chunk.length;
|
||||
}
|
||||
}
|
||||
return acc ?? Buffer.alloc(0);
|
||||
}
|
||||
}
|
||||
|
||||
function parseOctal(header, offset, length) {
|
||||
const raw = header.toString("utf8", offset, offset + length).replace(/[\0 ]+$/, "");
|
||||
return raw.length === 0 ? 0 : Number.parseInt(raw, 8);
|
||||
}
|
||||
|
||||
function cstring(header, offset, length) {
|
||||
const raw = header.toString("utf8", offset, offset + length);
|
||||
const nul = raw.indexOf("\0");
|
||||
return nul === -1 ? raw : raw.slice(0, nul);
|
||||
}
|
||||
|
||||
function checksumMatches(header) {
|
||||
const stored = parseOctal(header, 148, 8);
|
||||
const probe = Buffer.from(header);
|
||||
probe.fill(" ", 148, 156); // checksum field counts as spaces while summing
|
||||
let sum = 0;
|
||||
for (const byte of probe) sum += byte;
|
||||
return sum === stored;
|
||||
}
|
||||
|
||||
/** Stream exactly `size` bytes from the reader into `outStream`. */
|
||||
async function copyN(reader, size, outStream) {
|
||||
let remaining = size;
|
||||
while (remaining > 0) {
|
||||
const chunk = await reader.readSome();
|
||||
if (chunk === null) {
|
||||
throw new Error(`unexpected EOF after ${size - remaining} of ${size} bytes`);
|
||||
}
|
||||
const take = chunk.length > remaining ? chunk.subarray(0, remaining) : chunk;
|
||||
if (chunk.length > remaining) reader.unshift(chunk.subarray(remaining));
|
||||
remaining -= take.length;
|
||||
if (!outStream.write(take)) await once(outStream, "drain");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a tarball written by `createTarGz` (ustar + GNU LongLink) into
|
||||
* `destDir`. Returns the number of entries written.
|
||||
*/
|
||||
export async function extractTarGz(archiveFile, destDir) {
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
const src = createReadStream(archiveFile);
|
||||
const gunzip = createGunzip();
|
||||
src.pipe(gunzip);
|
||||
const reader = new BlockSource(gunzip);
|
||||
|
||||
const zeros = Buffer.alloc(BLOCK);
|
||||
let longName = null;
|
||||
let longLink = null;
|
||||
let entries = 0;
|
||||
|
||||
for (;;) {
|
||||
const header = await reader.read(BLOCK);
|
||||
if (header === null) break; // tolerate archives missing the final zero blocks
|
||||
if (header.equals(zeros)) {
|
||||
const second = await reader.read(BLOCK);
|
||||
if (second !== null && !second.equals(zeros)) {
|
||||
throw new Error("corrupt archive: data after terminator block");
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!checksumMatches(header)) {
|
||||
throw new Error(`tar header checksum mismatch at entry #${entries + 1}`);
|
||||
}
|
||||
|
||||
let name = cstring(header, 0, 100);
|
||||
const size = parseOctal(header, 124, 12);
|
||||
const typeflag = String.fromCharCode(header[156] || 0x30);
|
||||
let linkname = cstring(header, 157, 100);
|
||||
const prefix = cstring(header, 345, 155);
|
||||
if (prefix) name = `${prefix}/${name}`;
|
||||
if (longName !== null) {
|
||||
name = longName;
|
||||
longName = null;
|
||||
}
|
||||
if (longLink !== null) {
|
||||
linkname = longLink;
|
||||
longLink = null;
|
||||
}
|
||||
|
||||
const pad = (BLOCK - (size % BLOCK)) % BLOCK;
|
||||
|
||||
if (typeflag === "L" || typeflag === "K") {
|
||||
const payload = await reader.read(size);
|
||||
if (payload === null) throw new Error("unexpected EOF in LongLink payload");
|
||||
const value = cstring(payload, 0, payload.length);
|
||||
if (typeflag === "L") longName = value;
|
||||
else longLink = value;
|
||||
if (pad > 0) await reader.read(pad);
|
||||
continue;
|
||||
}
|
||||
|
||||
const target = safeJoin(destDir, name);
|
||||
|
||||
if (typeflag === "5") {
|
||||
fs.mkdirSync(target, { recursive: true });
|
||||
} else if (typeflag === "2") {
|
||||
if (linkname.length === 0) throw new Error(`symlink entry ${name} has empty target`);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.rmSync(target, { force: true });
|
||||
fs.symlinkSync(linkname, target);
|
||||
} else if (typeflag === "1") {
|
||||
const sourceAbs = safeJoin(destDir, linkname);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.copyFileSync(sourceAbs, target);
|
||||
} else {
|
||||
// Regular file ("0" or "\0"). The packer never stores directory entries,
|
||||
// so parent directories are materialized here.
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
const sink = createWriteStream(target, { flags: "w" });
|
||||
const finished = once(sink, "finish");
|
||||
sink.on("error", (err) => gunzip.destroy(err));
|
||||
await copyN(reader, size, sink);
|
||||
sink.end();
|
||||
await finished;
|
||||
const storedMode = parseOctal(header, 100, 8);
|
||||
if (storedMode) fs.chmodSync(target, storedMode);
|
||||
}
|
||||
if (pad > 0) {
|
||||
const skip = await reader.read(pad);
|
||||
if (skip === null) throw new Error(`unexpected EOF in padding of ${name}`);
|
||||
}
|
||||
entries += 1;
|
||||
}
|
||||
|
||||
src.destroy();
|
||||
return { entries };
|
||||
}
|
||||
|
||||
function safeJoin(destDir, name) {
|
||||
const normalized = path.normalize(name).split(path.sep).join("/");
|
||||
if (normalized.startsWith("/") || normalized.split("/").includes("..")) {
|
||||
throw new Error(`unsafe tar entry path: ${name}`);
|
||||
}
|
||||
return path.join(destDir, ...normalized.split("/"));
|
||||
}
|
||||
304
tests/unit/build/standalone-bundle.test.ts
Normal file
304
tests/unit/build/standalone-bundle.test.ts
Normal file
@@ -0,0 +1,304 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/**
|
||||
* Stage 8 (issue #10321) — shared standalone web bundle.
|
||||
*
|
||||
* One ubuntu `web-build` job packs `.build/next` into a deterministic
|
||||
* archive + byte-level manifest; every desktop leg restores it (verifying
|
||||
* every entry) and re-forks install-machine-forked native optionals for its
|
||||
* platform. These tests pin the integrity chain on temp trees: pack/restore
|
||||
* roundtrip, byte determinism, tamper detection (archive and restored tree),
|
||||
* manifest-version gating, native fork hydration, and the bundled-native
|
||||
* serviceability assertion (including the onnxruntime darwin-x64 exemption).
|
||||
*/
|
||||
|
||||
const bundleMod = await import("../../../scripts/build/standaloneBundle.mjs");
|
||||
const manifestMod = await import("../../../scripts/build/standaloneManifest.mjs");
|
||||
const hydrateMod = await import("../../../scripts/build/hydrateNativeDeps.mjs");
|
||||
|
||||
const { runPack, runRestore } = bundleMod as typeof bundleMod & {
|
||||
runPack: (opts: { dir?: string; out: string; manifest?: string }) => Promise<{
|
||||
archive: string;
|
||||
manifest: string;
|
||||
files: number;
|
||||
archiveBytes: number;
|
||||
}>;
|
||||
runRestore: (opts: { archive: string; manifest?: string; dir?: string }) => Promise<{
|
||||
archive: string;
|
||||
dir: string;
|
||||
files: number;
|
||||
}>;
|
||||
};
|
||||
const { verifyStandaloneManifest, MANIFEST_VERSION } = manifestMod as typeof manifestMod & {
|
||||
MANIFEST_VERSION: number;
|
||||
verifyStandaloneManifest: (
|
||||
rootDir: string,
|
||||
manifest: unknown
|
||||
) => Promise<{ ok: true } | { ok: false; errors: string[] }>;
|
||||
};
|
||||
const { hydratePlatformNatives, verifyBundledNatives } = hydrateMod as typeof hydrateMod & {
|
||||
hydratePlatformNatives: (opts: { standaloneNodeModules: string; sourceNodeModules: string }) => {
|
||||
replaced: string[];
|
||||
removed: string[];
|
||||
copied: string[];
|
||||
};
|
||||
verifyBundledNatives: (opts: { nodeModulesDir: string; platform: string; arch: string }) => {
|
||||
ok: boolean;
|
||||
errors: string[];
|
||||
};
|
||||
};
|
||||
|
||||
const IS_WINDOWS = process.platform === "win32";
|
||||
|
||||
function tmpDir(prefix: string): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function sha256File(filePath: string): string {
|
||||
return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
|
||||
}
|
||||
|
||||
/** Minimal fake `.build/next` tree: nested files, exec bit, and a symlink. */
|
||||
function buildWebTree(root: string): void {
|
||||
const standalone = path.join(root, "standalone");
|
||||
fs.mkdirSync(path.join(standalone, "node_modules", "left-pad"), { recursive: true });
|
||||
fs.writeFileSync(path.join(standalone, "server.js"), "console.log('omniroute');\n");
|
||||
fs.writeFileSync(
|
||||
path.join(standalone, "node_modules", "left-pad", "index.js"),
|
||||
"module.exports = (s, n) => String(s).padStart(n);\n"
|
||||
);
|
||||
fs.writeFileSync(path.join(standalone, "node_modules", "left-pad", "package.json"), "{}\n");
|
||||
const bin = path.join(standalone, "server-cli.js");
|
||||
fs.writeFileSync(bin, "#!/usr/bin/env node\n");
|
||||
fs.chmodSync(bin, 0o755);
|
||||
fs.mkdirSync(path.join(root, "static"), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, "static", "app.css"), "body{margin:0}\n");
|
||||
if (!IS_WINDOWS) {
|
||||
fs.symlinkSync("../standalone/server.js", path.join(root, "static", "server-link.js"));
|
||||
}
|
||||
}
|
||||
|
||||
function writeNative(root: string, relPath: string, content: string): void {
|
||||
const target = path.join(root, ...relPath.split("/"));
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.writeFileSync(target, content);
|
||||
}
|
||||
|
||||
test("pack → restore roundtrip restores the tree byte-for-byte", async () => {
|
||||
const src = tmpDir("s8-src-");
|
||||
const out = path.join(tmpDir("s8-out-"), "web-bundle.tar.gz");
|
||||
const dst = tmpDir("s8-dst-");
|
||||
try {
|
||||
buildWebTree(src);
|
||||
const packed = await runPack({ dir: src, out });
|
||||
assert.ok(packed.files > 0, "manifest must list entries");
|
||||
assert.ok(fs.existsSync(`${out}.manifest.json`), "manifest written next to archive");
|
||||
|
||||
const restored = await runRestore({ archive: out, dir: dst });
|
||||
assert.equal(restored.files, packed.files);
|
||||
|
||||
assert.equal(
|
||||
fs.readFileSync(path.join(dst, "standalone", "server.js"), "utf8"),
|
||||
"console.log('omniroute');\n"
|
||||
);
|
||||
// The restored tree satisfies the manifest (sizes + hashes + symlink targets).
|
||||
const manifest = JSON.parse(fs.readFileSync(`${out}.manifest.json`, "utf8"));
|
||||
const verdict = await verifyStandaloneManifest(dst, manifest);
|
||||
assert.equal(
|
||||
verdict.ok,
|
||||
true,
|
||||
`restored tree must verify: ${verdict.ok ? "" : (verdict as { errors: string[] }).errors.join("; ")}`
|
||||
);
|
||||
if (!IS_WINDOWS) {
|
||||
assert.equal(
|
||||
fs.readlinkSync(path.join(dst, "static", "server-link.js")),
|
||||
"../standalone/server.js",
|
||||
"symlink target preserved"
|
||||
);
|
||||
assert.equal(
|
||||
fs.statSync(path.join(dst, "standalone", "server-cli.js")).mode & 0o111,
|
||||
0o111,
|
||||
"exec bit preserved"
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(src, { recursive: true, force: true });
|
||||
fs.rmSync(path.dirname(out), { recursive: true, force: true });
|
||||
fs.rmSync(dst, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("packing is byte-deterministic across runs", async () => {
|
||||
const src = tmpDir("s8-det-");
|
||||
const outDir = tmpDir("s8-det-out-");
|
||||
try {
|
||||
buildWebTree(src);
|
||||
const a = path.join(outDir, "a.tar.gz");
|
||||
const b = path.join(outDir, "b.tar.gz");
|
||||
await runPack({ dir: src, out: a });
|
||||
await runPack({ dir: src, out: b });
|
||||
assert.equal(sha256File(a), sha256File(b), "two packs of the same tree must be identical");
|
||||
} finally {
|
||||
fs.rmSync(src, { recursive: true, force: true });
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("restore rejects a corrupted archive before extraction", async () => {
|
||||
const src = tmpDir("s8-tamper-");
|
||||
const outDir = tmpDir("s8-tamper-out-");
|
||||
try {
|
||||
buildWebTree(src);
|
||||
const out = path.join(outDir, "web-bundle.tar.gz");
|
||||
await runPack({ dir: src, out });
|
||||
const raw = fs.readFileSync(out);
|
||||
raw[raw.length - 10] ^= 0xff; // flip one byte in the gzip trailer region
|
||||
fs.writeFileSync(out, raw);
|
||||
await assert.rejects(() => runRestore({ archive: out, dir: path.join(outDir, "dst") }), /sha/);
|
||||
} finally {
|
||||
fs.rmSync(src, { recursive: true, force: true });
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("manifest verification flags modified and smuggled files in a restored tree", async () => {
|
||||
const src = tmpDir("s8-verify-");
|
||||
const outDir = tmpDir("s8-verify-out-");
|
||||
const dst = tmpDir("s8-verify-dst-");
|
||||
try {
|
||||
buildWebTree(src);
|
||||
const out = path.join(outDir, "web-bundle.tar.gz");
|
||||
await runPack({ dir: src, out });
|
||||
await runRestore({ archive: out, dir: dst });
|
||||
|
||||
fs.appendFileSync(path.join(dst, "standalone", "server.js"), "// tampered\n");
|
||||
fs.writeFileSync(path.join(dst, "static", "smuggled.js"), "evil();\n");
|
||||
|
||||
const manifest = JSON.parse(fs.readFileSync(`${out}.manifest.json`, "utf8"));
|
||||
const verdict = await verifyStandaloneManifest(dst, manifest);
|
||||
assert.equal(verdict.ok, false);
|
||||
assert.ok(
|
||||
verdict.errors.some((e) => e.includes("standalone/server.js")),
|
||||
`content tampering detected: ${verdict.errors.join("; ")}`
|
||||
);
|
||||
assert.ok(
|
||||
verdict.errors.some((e) => e.includes("unlisted files") && e.includes("static/smuggled.js")),
|
||||
`smuggled file detected: ${verdict.errors.join("; ")}`
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(src, { recursive: true, force: true });
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
fs.rmSync(dst, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("manifest verification rejects an unsupported manifest version", async () => {
|
||||
const dst = tmpDir("s8-ver-");
|
||||
try {
|
||||
const verdict = await verifyStandaloneManifest(dst, {
|
||||
version: MANIFEST_VERSION + 1,
|
||||
entries: [],
|
||||
});
|
||||
assert.equal(verdict.ok, false);
|
||||
assert.match(verdict.errors[0] ?? "", /unsupported manifest version/);
|
||||
} finally {
|
||||
fs.rmSync(dst, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("hydratePlatformNatives swaps install-machine-forked packages for this leg", () => {
|
||||
const standalone = tmpDir("s8-hydrate-sa-");
|
||||
const source = tmpDir("s8-hydrate-src-");
|
||||
try {
|
||||
// The ubuntu-built standalone carries linux sharp + darwin-only fsevents.
|
||||
writeNative(
|
||||
standalone,
|
||||
"node_modules/@img/sharp-linux-x64/package.json",
|
||||
'{"name":"@img/sharp-linux-x64"}'
|
||||
);
|
||||
writeNative(standalone, "node_modules/@img/sharp-linux-x64/lib/index.js", "linux fork");
|
||||
writeNative(standalone, "node_modules/fsevents/fsevents.js", "mac only");
|
||||
// This leg (darwin-arm64) resolved its own forks: different sharp, no fsevents.
|
||||
writeNative(
|
||||
source,
|
||||
"node_modules/@img/sharp-darwin-arm64/package.json",
|
||||
'{"name":"@img/sharp-darwin-arm64"}'
|
||||
);
|
||||
writeNative(source, "node_modules/@img/sharp-darwin-arm64/lib/index.js", "darwin fork");
|
||||
|
||||
const result = hydratePlatformNatives({
|
||||
standaloneNodeModules: path.join(standalone, "node_modules"),
|
||||
sourceNodeModules: path.join(source, "node_modules"),
|
||||
});
|
||||
|
||||
// Platform forks ship under different package names, so hydration is
|
||||
// remove(standalone fork) + copy(this leg's fork); `replaced` stays empty
|
||||
// unless the exact same name exists on both sides.
|
||||
assert.deepEqual(result.copied.sort(), ["@img/sharp-darwin-arm64"]);
|
||||
assert.deepEqual(result.replaced, []);
|
||||
assert.deepEqual(result.removed.sort(), ["@img/sharp-linux-x64", "fsevents"]);
|
||||
assert.ok(
|
||||
fs.existsSync(
|
||||
path.join(standalone, "node_modules", "@img", "sharp-darwin-arm64", "lib", "index.js")
|
||||
),
|
||||
"darwin fork copied in"
|
||||
);
|
||||
assert.ok(
|
||||
!fs.existsSync(path.join(standalone, "node_modules", "@img", "sharp-linux-x64")),
|
||||
"linux fork removed"
|
||||
);
|
||||
assert.ok(
|
||||
!fs.existsSync(path.join(standalone, "node_modules", "fsevents")),
|
||||
"fsevents dropped on non-matching leg"
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(standalone, { recursive: true, force: true });
|
||||
fs.rmSync(source, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyBundledNatives asserts serviceability and honors the onnx darwin-x64 exemption", () => {
|
||||
const root = tmpDir("s8-natives-");
|
||||
try {
|
||||
const nm = path.join(root, "node_modules");
|
||||
writeNative(nm, "koffi/build/koffi/linux_x64/koffi.node", "elf");
|
||||
writeNative(nm, "better-sqlite3/prebuilds/linux-x64.node", "napi");
|
||||
writeNative(nm, "wreq-js/rust/wreq-js.linux-x64-gnu.node", "rust");
|
||||
writeNative(nm, "onnxruntime-node/bin/napi-v6/linux/x64/libonnxruntime.so", "ort");
|
||||
|
||||
const good = verifyBundledNatives({ nodeModulesDir: nm, platform: "linux", arch: "x64" });
|
||||
assert.equal(
|
||||
good.ok,
|
||||
true,
|
||||
`expected serviceable: ${(good as { errors?: string[] }).errors?.join("; ")}`
|
||||
);
|
||||
|
||||
const missingKoffi = verifyBundledNatives({
|
||||
nodeModulesDir: nm,
|
||||
platform: "darwin",
|
||||
arch: "arm64",
|
||||
});
|
||||
assert.equal(missingKoffi.ok, false);
|
||||
assert.ok((missingKoffi as { errors: string[] }).errors.some((e) => e.startsWith("koffi:")));
|
||||
|
||||
// darwin-x64 has no onnxruntime-node prebuild at all — the exemption must keep it green
|
||||
// as long as the other bundled natives service that triple.
|
||||
const nm2 = path.join(root, "node_modules2");
|
||||
writeNative(nm2, "koffi/build/koffi/darwin_x64/koffi.node", "macho");
|
||||
writeNative(nm2, "better-sqlite3/prebuilds/darwin-x64.node", "napi");
|
||||
writeNative(nm2, "wreq-js/rust/wreq-js.darwin-x64.node", "rust");
|
||||
const exempted = verifyBundledNatives({ nodeModulesDir: nm2, platform: "darwin", arch: "x64" });
|
||||
assert.equal(
|
||||
exempted.ok,
|
||||
true,
|
||||
`darwin-x64 must pass via exemption: ${(exempted as { errors?: string[] }).errors?.join("; ")}`
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user