#!/usr/bin/env node /** * Deterministic tar.gz primitives for the shared web build (issue #10321, * Stage 8). * * Why not shell out to system tar: the restore step runs on every desktop * matrix leg including Windows, where bsdtar's long-path behavior on deep * node_modules trees is not guaranteed. Node's fs layer already proves it can * produce and consume this exact tree on Windows today (the legacy per-leg * `npm run build` writes it with the same fs), so a pure-Node reader keeps the * extraction on the one path layer we know works. * * Format: ustar with GNU LongLink ('L') entries for paths > 100 chars, * typeflag '2' for symlinks, mtime/uid/gid zeroed and modes normalized to * 0644/0755 (exec bit only) so the archive of a given tree is byte-identical * on every machine. */ import { createReadStream, createWriteStream } from "node:fs"; import fs from "node:fs"; import path from "node:path"; import { once } from "node:events"; import { createGunzip, createGzip } from "node:zlib"; const BLOCK = 512; function octal(value, length) { return value.toString(8).padStart(length - 1, "0") + "\0"; } function headerFor(name, size, typeflag, linkname = "", prefix = "", mode = 0o644) { const buf = Buffer.alloc(BLOCK, 0); buf.write(name.slice(0, 100), 0, 100, "utf8"); buf.write(octal(typeflag === "5" ? 0o755 : mode, 8), 100); buf.write(octal(0, 8), 108); // uid buf.write(octal(0, 8), 116); // gid buf.write(octal(size, 12), 124); buf.write(octal(0, 12), 136); // mtime = 0 for determinism buf.write(" ", 148); // checksum placeholder: spaces buf.write(typeflag, 156); buf.write(linkname.slice(0, 100), 157, 100, "utf8"); buf.write("ustar\0", 257, 6, "utf8"); buf.write("00", 263, 2, "utf8"); buf.write(prefix.slice(0, 155), 345, 155, "utf8"); let sum = 0; for (const byte of buf) sum += byte; buf.write(sum.toString(8).padStart(6, "0") + "\0 ", 148); return buf; } function dataPad(size) { const pad = (BLOCK - (size % BLOCK)) % BLOCK; return Buffer.alloc(pad, 0); } function longLinkEntry(name) { const payload = Buffer.from(name + "\0", "utf8"); return Buffer.concat([ headerFor("././@LongLink", payload.length, "L"), payload, dataPad(payload.length), ]); } /** Emit header (with LongLink/prefix handling) for one entry. */ function entryHeader(relPath, size, typeflag, linkname, mode) { const out = []; if (relPath.length > 100) { const slash = relPath.slice(0, 155).lastIndexOf("/"); const prefix = slash > 0 ? relPath.slice(0, slash) : ""; const name = prefix ? relPath.slice(slash + 1) : relPath; if (name.length > 100) { out.push(longLinkEntry(relPath)); name = relPath.slice(0, 100); } out.push(headerFor(name, size, typeflag, linkname, prefix, mode)); } else { out.push(headerFor(relPath, size, typeflag, linkname, undefined, mode)); } return Buffer.concat(out); } function* walkFiles(root, current = root) { const children = fs .readdirSync(current, { withFileTypes: true }) .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); for (const child of children) { const abs = path.join(current, child.name); const rel = path.relative(root, abs).split(path.sep).join("/"); if (child.isSymbolicLink()) { yield { rel, symlink: fs.readlinkSync(abs) }; } else if (child.isDirectory()) { yield* walkFiles(root, abs); } else if (child.isFile()) { yield { rel, abs }; } } } /** Write a buffer, respecting gzip backpressure. */ async function writeWithBackpressure(stream, buf) { if (!stream.write(buf)) await once(stream, "drain"); } /** Stream one file's bytes into the archive (no whole-file buffering). */ function pipeFileInto(gz, failure, abs) { return new Promise((resolve, reject) => { const stream = createReadStream(abs, { autoClose: true }); const onDrain = () => stream.resume(); const detach = () => gz.removeListener("drain", onDrain); stream.on("error", (err) => { detach(); reject(err); }); stream.on("data", (chunk) => { if (!gz.write(chunk)) stream.pause(); }); gz.on("drain", onDrain); stream.on("end", () => { detach(); resolve(); }); }); } /** Pack `srcDir` into a deterministic gzipped tarball at `outFile`. */ export async function createTarGz(srcDir, outFile) { const out = createWriteStream(outFile); const gz = createGzip({ level: 1 }); gz.pipe(out); const failure = new Promise((_, reject) => { gz.on("error", reject); out.on("error", reject); }); try { for (const entry of walkFiles(srcDir)) { if (entry.symlink !== undefined) { if (entry.symlink.length > 100) { throw new Error(`symlink target too long for ustar: ${entry.rel} -> ${entry.symlink}`); } await writeWithBackpressure(gz, entryHeader(entry.rel, 0, "2", entry.symlink)); continue; } const st = fs.statSync(entry.abs); const size = st.size; const mode = st.mode & 0o111 ? 0o755 : 0o644; await writeWithBackpressure(gz, entryHeader(entry.rel, size, "0", undefined, mode)); if (size > 0) await Promise.race([pipeFileInto(gz, failure, entry.abs), failure]); const pad = (BLOCK - (size % BLOCK)) % BLOCK; if (pad > 0) await writeWithBackpressure(gz, Buffer.alloc(pad, 0)); } await writeWithBackpressure(gz, Buffer.alloc(BLOCK * 2, 0)); // terminator await Promise.race([ new Promise((resolve, reject) => { out.on("finish", resolve); out.on("error", reject); gz.end(); }), failure, ]); } catch (err) { gz.destroy(); out.destroy(); throw err; } } // ─── extraction ────────────────────────────────────────────────────────────────── /** * Promise-based byte source over a gunzip stream. `read(n)` waits until `n` * bytes are buffered (or EOF); `readSome()` returns whatever is available, for * streaming large payloads into files without whole-file buffering. */ class BlockSource { constructor(stream) { this.buffer = Buffer.alloc(0); this.error = null; this.ended = false; this.waiter = null; stream.on("data", (chunk) => { this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk]); this.notify(); }); stream.on("end", () => { this.ended = true; this.notify(); }); stream.on("error", (err) => { this.error = err; this.notify(); }); } notify() { if (this.waiter) { const waiter = this.waiter; this.waiter = null; waiter(); } } readSome() { return new Promise((resolve, reject) => { const attempt = () => { if (this.error) return reject(this.error); if (this.buffer.length > 0) { const out = this.buffer; this.buffer = Buffer.alloc(0); return resolve(out); } if (this.ended) return resolve(null); this.waiter = attempt; }; attempt(); }); } unshift(buf) { if (buf && buf.length > 0) this.buffer = Buffer.concat([buf, this.buffer]); } async read(n) { let acc = null; let remaining = n; while (remaining > 0) { const chunk = await this.readSome(); if (chunk === null) return null; // EOF before n bytes if (chunk.length > remaining) { acc = acc ? Buffer.concat([acc, chunk.subarray(0, remaining)]) : chunk.subarray(0, remaining); this.unshift(chunk.subarray(remaining)); remaining = 0; } else { acc = acc ? Buffer.concat([acc, chunk]) : chunk; remaining -= chunk.length; } } return acc ?? Buffer.alloc(0); } } function parseOctal(header, offset, length) { const raw = header.toString("utf8", offset, offset + length).replace(/[\0 ]+$/, ""); return raw.length === 0 ? 0 : Number.parseInt(raw, 8); } function cstring(header, offset, length) { const raw = header.toString("utf8", offset, offset + length); const nul = raw.indexOf("\0"); return nul === -1 ? raw : raw.slice(0, nul); } function checksumMatches(header) { const stored = parseOctal(header, 148, 8); const probe = Buffer.from(header); probe.fill(" ", 148, 156); // checksum field counts as spaces while summing let sum = 0; for (const byte of probe) sum += byte; return sum === stored; } /** Stream exactly `size` bytes from the reader into `outStream`. */ async function copyN(reader, size, outStream) { let remaining = size; while (remaining > 0) { const chunk = await reader.readSome(); if (chunk === null) { throw new Error(`unexpected EOF after ${size - remaining} of ${size} bytes`); } const take = chunk.length > remaining ? chunk.subarray(0, remaining) : chunk; if (chunk.length > remaining) reader.unshift(chunk.subarray(remaining)); remaining -= take.length; if (!outStream.write(take)) await once(outStream, "drain"); } } /** * Extract a tarball written by `createTarGz` (ustar + GNU LongLink) into * `destDir`. Returns the number of entries written. */ export async function extractTarGz(archiveFile, destDir) { fs.mkdirSync(destDir, { recursive: true }); const src = createReadStream(archiveFile); const gunzip = createGunzip(); src.pipe(gunzip); const reader = new BlockSource(gunzip); const zeros = Buffer.alloc(BLOCK); let longName = null; let longLink = null; let entries = 0; for (;;) { const header = await reader.read(BLOCK); if (header === null) break; // tolerate archives missing the final zero blocks if (header.equals(zeros)) { const second = await reader.read(BLOCK); if (second !== null && !second.equals(zeros)) { throw new Error("corrupt archive: data after terminator block"); } break; } if (!checksumMatches(header)) { throw new Error(`tar header checksum mismatch at entry #${entries + 1}`); } let name = cstring(header, 0, 100); const size = parseOctal(header, 124, 12); const typeflag = String.fromCharCode(header[156] || 0x30); let linkname = cstring(header, 157, 100); const prefix = cstring(header, 345, 155); if (prefix) name = `${prefix}/${name}`; if (longName !== null) { name = longName; longName = null; } if (longLink !== null) { linkname = longLink; longLink = null; } const pad = (BLOCK - (size % BLOCK)) % BLOCK; if (typeflag === "L" || typeflag === "K") { const payload = await reader.read(size); if (payload === null) throw new Error("unexpected EOF in LongLink payload"); const value = cstring(payload, 0, payload.length); if (typeflag === "L") longName = value; else longLink = value; if (pad > 0) await reader.read(pad); continue; } const target = safeJoin(destDir, name); if (typeflag === "5") { fs.mkdirSync(target, { recursive: true }); } else if (typeflag === "2") { if (linkname.length === 0) throw new Error(`symlink entry ${name} has empty target`); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.rmSync(target, { force: true }); fs.symlinkSync(linkname, target); } else if (typeflag === "1") { const sourceAbs = safeJoin(destDir, linkname); fs.mkdirSync(path.dirname(target), { recursive: true }); fs.copyFileSync(sourceAbs, target); } else { // Regular file ("0" or "\0"). The packer never stores directory entries, // so parent directories are materialized here. fs.mkdirSync(path.dirname(target), { recursive: true }); const sink = createWriteStream(target, { flags: "w" }); const finished = once(sink, "finish"); sink.on("error", (err) => gunzip.destroy(err)); await copyN(reader, size, sink); sink.end(); await finished; const storedMode = parseOctal(header, 100, 8); if (storedMode) fs.chmodSync(target, storedMode); } if (pad > 0) { const skip = await reader.read(pad); if (skip === null) throw new Error(`unexpected EOF in padding of ${name}`); } entries += 1; } src.destroy(); return { entries }; } function safeJoin(destDir, name) { const normalized = path.normalize(name).split(path.sep).join("/"); if (normalized.startsWith("/") || normalized.split("/").includes("..")) { throw new Error(`unsafe tar entry path: ${name}`); } return path.join(destDir, ...normalized.split("/")); }