diff --git a/changelog.d/fixes/6794-electron-turbopack-symlinks.md b/changelog.d/fixes/6794-electron-turbopack-symlinks.md new file mode 100644 index 0000000000..963e35e6bc --- /dev/null +++ b/changelog.d/fixes/6794-electron-turbopack-symlinks.md @@ -0,0 +1 @@ +- **fix(electron): materialize Turbopack hashed-module symlinks during packaging (#6724, #6594)** (#6794 — thanks @huohua-dev). diff --git a/scripts/build/assembleStandalone.mjs b/scripts/build/assembleStandalone.mjs index 7afe168b29..bdaed25aea 100644 --- a/scripts/build/assembleStandalone.mjs +++ b/scripts/build/assembleStandalone.mjs @@ -39,7 +39,8 @@ * prune + validate (pack-artifact-policy) - Y - UNIQUE (prepublish) * data/ dir creation - Y - UNIQUE (prepublish) * --- electron-UNIQUE --- - * better-sqlite3 + keytar native strip (ABI rebuild) - - Y UNIQUE (electron) + * better-sqlite3 native strip + Electron-ABI rebuild - - Y UNIQUE (electron) + * Turbopack hashed-module symlink materialize (node_modules) - - Y SHARED (opt-in: materializeSymlinks) * symlink guard (assertBundleIsPackagable) - - Y UNIQUE (electron) * removeGeneratedElectronArtifacts - - Y UNIQUE (electron) */ @@ -469,6 +470,140 @@ function copyNativeAssetsAndExtraModules(projectRoot, resolvedOutDir) { } } +/** + * Materialize Turbopack "hashed external module" symlinks inside a bundled + * node_modules dir into real, self-contained directories. + * + * Next.js/Turbopack standalone output emits entries like + * better-sqlite3-90e2652d1716b047 -> /node_modules/better-sqlite3 + * as ABSOLUTE symlinks into the build machine's tree. cpSync preserves symlinks and + * electron-builder preserves extraResources symlinks verbatim, so the packaged app + * ships dangling links pointing at e.g. /Users/runner/work/... On the end-user machine + * those targets don't exist → the instrumentation hook throws + * ERR_MODULE_NOT_FOUND: Cannot find package 'ws-' → server boot fails. + * (issues #6724, #6594). Windows is doubly broken because it can't follow POSIX + * symlinks at all. + * + * The fix: for every symlink under the given node_modules (top level + one level of + * scoped @scope/ dirs), replace it with a REAL directory copy of its dereferenced + * target — a dereference is the only option that is correct on every OS (Windows + * included) and survives the machine that built it. If the link is already dangling + * (target absent), fall back to copying a sibling real package whose name is the + * hashed name with its trailing `-` suffix stripped; if none exists, drop the + * dangling link so it cannot poison module resolution. + * + * @param {string} nodeModulesDir - absolute path to a bundled node_modules directory + * @returns {{ materialized: number, relinked: number, removed: number }} + */ +export function materializeBundledSymlinks(nodeModulesDir) { + const summary = { materialized: 0, relinked: 0, removed: 0 }; + if (!fsSync.existsSync(nodeModulesDir)) return summary; + + const entries = []; + for (const name of fsSync.readdirSync(nodeModulesDir)) { + const entryPath = path.join(nodeModulesDir, name); + if (name.startsWith("@") && fsSync.lstatSync(entryPath).isDirectory()) { + // Scoped packages live one level deeper (@scope/pkg). + for (const scoped of fsSync.readdirSync(entryPath)) { + entries.push(path.join(entryPath, scoped)); + } + continue; + } + entries.push(entryPath); + } + + for (const entryPath of entries) { + let stat; + try { + stat = fsSync.lstatSync(entryPath); + } catch { + continue; + } + if (!stat.isSymbolicLink()) continue; + + let realTarget = null; + try { + realTarget = fsSync.realpathSync(entryPath); + } catch { + realTarget = null; + } + + if (realTarget && fsSync.existsSync(realTarget)) { + // Dereference: copy the resolved real files in place of the link. + fsSync.rmSync(entryPath, { recursive: true, force: true }); + fsSync.cpSync(realTarget, entryPath, { recursive: true, dereference: true }); + summary.materialized += 1; + continue; + } + + // Dangling link (e.g. absolute path into the build machine that no longer + // exists). Try a sibling real package named without the trailing - hash. + const baseName = path.basename(entryPath).replace(/-[0-9a-f]{8,}$/i, ""); + const sibling = path.join(path.dirname(entryPath), baseName); + if (baseName !== path.basename(entryPath) && fsSync.existsSync(sibling)) { + let siblingStat = null; + try { + siblingStat = fsSync.lstatSync(sibling); + } catch { + siblingStat = null; + } + if (siblingStat && siblingStat.isDirectory()) { + fsSync.rmSync(entryPath, { recursive: true, force: true }); + fsSync.cpSync(sibling, entryPath, { recursive: true, dereference: true }); + summary.relinked += 1; + continue; + } + } + + // Nothing to resolve to — drop the dangling link so it cannot shadow resolution. + console.warn( + `[assembleStandalone] Dropping dangling module symlink (target missing): ${entryPath}` + ); + fsSync.rmSync(entryPath, { recursive: true, force: true }); + summary.removed += 1; + } + + return summary; +} + +/** + * Sync an Electron-ABI-rebuilt native module into any hashed/plain copies of + * that module already materialized inside a nested node_modules dir. + * + * materializeBundledSymlinks() turns Turbopack hashed-module symlinks (e.g. + * `better-sqlite3-90e2652d1716b047`) into real directory copies of the + * Node-ABI build. A later step in prepare-electron-standalone.mjs rebuilds + * better-sqlite3 against the Electron ABI at the bundle root — but the + * hashed copy under the nested node_modules still holds the stale Node-ABI + * build, and the server's hashed `require("better-sqlite3-")` resolves + * to it, not the rebuilt root module. Previously that hashed copy was simply + * deleted, which caused MODULE_NOT_FOUND and a silent fallback to the sql.js + * WASM driver in the packaged app (issue #6794 follow-up). Overwriting each + * matching entry with the rebuilt root module keeps the hashed require + * resolving to a working, ABI-correct native driver instead. + * + * @param {string} rootModuleDir - absolute path to the already-rebuilt module (e.g. /node_modules/better-sqlite3) + * @param {string} nodeModulesDir - absolute path to the nested node_modules dir to scan + * @returns {{ synced: number }} + */ +export function syncRebuiltNativeModuleIntoHashedEntries(rootModuleDir, nodeModulesDir) { + const summary = { synced: 0 }; + if (!fsSync.existsSync(rootModuleDir) || !fsSync.existsSync(nodeModulesDir)) return summary; + + const baseName = path.basename(rootModuleDir); + const pattern = new RegExp(`^${baseName}(-[0-9a-f]{8,})?$`, "i"); + + for (const name of fsSync.readdirSync(nodeModulesDir)) { + if (!pattern.test(name)) continue; + const entryPath = path.join(nodeModulesDir, name); + fsSync.rmSync(entryPath, { recursive: true, force: true }); + fsSync.cpSync(rootModuleDir, entryPath, { recursive: true, dereference: true }); + summary.synced += 1; + } + + return summary; +} + /** * Assemble the Next.js standalone bundle into outDir. * @@ -485,6 +620,7 @@ function copyNativeAssetsAndExtraModules(projectRoot, resolvedOutDir) { * @param {boolean} [opts.sanitizePaths] - replace build-machine abs paths with "." (default false) * @param {boolean} [opts.patchTurbopackChunks] - strip hashed externals from .next/server js files (default false) * @param {boolean} [opts.copyNatives] - copy native assets + extra modules (default true) + * @param {boolean} [opts.materializeSymlinks] - dereference Turbopack hashed-module symlinks in node_modules (default false) * @returns {void} */ export function assembleStandalone({ @@ -494,6 +630,7 @@ export function assembleStandalone({ sanitizePaths = false, patchTurbopackChunks: doPatchChunks = false, copyNatives = true, + materializeSymlinks = false, }) { if (!distDir) throw new Error("[assembleStandalone] distDir is required"); if (!outDir) throw new Error("[assembleStandalone] outDir is required"); @@ -550,4 +687,23 @@ export function assembleStandalone({ if (copyNatives) { copyNativeAssetsAndExtraModules(projectRoot, resolvedOutDir); } + + // 7. Optionally dereference Turbopack hashed-module symlinks so the bundle is + // self-contained (no absolute links into the build machine). Runs AFTER the + // native/extra-module copy so the sibling-package relink fallback can find + // real packages. See materializeBundledSymlinks + issues #6724, #6594. + if (materializeSymlinks) { + for (const nmDir of [ + path.join(resolvedOutDir, "node_modules"), + path.join(resolvedOutDir, relDistDir, "node_modules"), + ]) { + const s = materializeBundledSymlinks(nmDir); + if (s.materialized || s.relinked || s.removed) { + console.log( + `[assembleStandalone] Materialized module symlinks in ${path.relative(resolvedOutDir, nmDir) || "."}: ` + + `${s.materialized} dereferenced, ${s.relinked} relinked, ${s.removed} dropped` + ); + } + } + } } diff --git a/scripts/build/prepare-electron-standalone.mjs b/scripts/build/prepare-electron-standalone.mjs index f48e1556a8..da04fca00b 100644 --- a/scripts/build/prepare-electron-standalone.mjs +++ b/scripts/build/prepare-electron-standalone.mjs @@ -178,6 +178,9 @@ assembleStandalone({ projectRoot: ROOT, sanitizePaths: true, copyNatives: true, + // #6724/#6594: dereference Turbopack hashed-module symlinks — inside the packaged + // app they would point at the build machine's absolute paths and break on install. + materializeSymlinks: true, }); // Electron-UNIQUE post-assembly steps diff --git a/tests/unit/materialize-bundled-symlinks.test.ts b/tests/unit/materialize-bundled-symlinks.test.ts new file mode 100644 index 0000000000..a83dc980bd --- /dev/null +++ b/tests/unit/materialize-bundled-symlinks.test.ts @@ -0,0 +1,225 @@ +import assert from "node:assert/strict"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +// @ts-expect-error - JS build helper without type declarations +import { + materializeBundledSymlinks, + syncRebuiltNativeModuleIntoHashedEntries, +} from "../../scripts/build/assembleStandalone.mjs"; + +function makePkg(dir: string, name: string, marker: string) { + const pkgDir = join(dir, name); + mkdirSync(pkgDir, { recursive: true }); + writeFileSync(join(pkgDir, "package.json"), JSON.stringify({ name, marker })); + return pkgDir; +} + +test("materializeBundledSymlinks dereferences a live symlink into a real directory", () => { + const root = mkdtempSync(join(tmpdir(), "mbs-live-")); + try { + const realPkgHome = join(root, "external"); + mkdirSync(realPkgHome, { recursive: true }); + makePkg(realPkgHome, "ws", "real-ws"); + + const nm = join(root, "bundle", "node_modules"); + mkdirSync(nm, { recursive: true }); + symlinkSync(join(realPkgHome, "ws"), join(nm, "ws-a972e7ffa40ff725"), "dir"); + + const summary = materializeBundledSymlinks(nm); + + assert.equal(summary.materialized, 1); + const target = join(nm, "ws-a972e7ffa40ff725"); + assert.equal(lstatSync(target).isSymbolicLink(), false); + assert.equal(lstatSync(target).isDirectory(), true); + assert.equal(JSON.parse(readFileSync(join(target, "package.json"), "utf8")).marker, "real-ws"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("materializeBundledSymlinks relinks a dangling hashed symlink to its sibling real package", () => { + const root = mkdtempSync(join(tmpdir(), "mbs-dangle-")); + try { + const nm = join(root, "node_modules"); + mkdirSync(nm, { recursive: true }); + // Sibling real package (as copied by copyNativeAssetsAndExtraModules). + makePkg(nm, "better-sqlite3", "real-bsq"); + // Dangling absolute link into a build machine that does not exist here. + symlinkSync( + "/Users/runner/work/OmniRoute/OmniRoute/.build/next/standalone/node_modules/better-sqlite3", + join(nm, "better-sqlite3-90e2652d1716b047"), + "dir" + ); + + const summary = materializeBundledSymlinks(nm); + + assert.equal(summary.relinked, 1); + const target = join(nm, "better-sqlite3-90e2652d1716b047"); + assert.equal(lstatSync(target).isSymbolicLink(), false); + assert.equal(JSON.parse(readFileSync(join(target, "package.json"), "utf8")).marker, "real-bsq"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("materializeBundledSymlinks drops a dangling link with no resolvable sibling", () => { + const root = mkdtempSync(join(tmpdir(), "mbs-drop-")); + try { + const nm = join(root, "node_modules"); + mkdirSync(nm, { recursive: true }); + symlinkSync( + "/nonexistent/build/machine/path/mystery", + join(nm, "mystery-deadbeefcafe0001"), + "dir" + ); + + const summary = materializeBundledSymlinks(nm); + + assert.equal(summary.removed, 1); + assert.equal(existsSync(join(nm, "mystery-deadbeefcafe0001")), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("materializeBundledSymlinks handles scoped-package symlinks", () => { + const root = mkdtempSync(join(tmpdir(), "mbs-scope-")); + try { + const realPkgHome = join(root, "external"); + mkdirSync(join(realPkgHome, "@huggingface"), { recursive: true }); + makePkg(join(realPkgHome, "@huggingface"), "transformers", "real-hf"); + + const nm = join(root, "node_modules"); + mkdirSync(join(nm, "@huggingface"), { recursive: true }); + symlinkSync( + join(realPkgHome, "@huggingface", "transformers"), + join(nm, "@huggingface", "transformers-abc1234567890def"), + "dir" + ); + + const summary = materializeBundledSymlinks(nm); + + assert.equal(summary.materialized, 1); + const target = join(nm, "@huggingface", "transformers-abc1234567890def"); + assert.equal(lstatSync(target).isSymbolicLink(), false); + assert.equal(JSON.parse(readFileSync(join(target, "package.json"), "utf8")).marker, "real-hf"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("materializeBundledSymlinks leaves real directories untouched and no-ops on missing dir", () => { + const root = mkdtempSync(join(tmpdir(), "mbs-noop-")); + try { + const nm = join(root, "node_modules"); + mkdirSync(nm, { recursive: true }); + makePkg(nm, "pino", "real-pino"); + + const summary = materializeBundledSymlinks(nm); + assert.deepEqual(summary, { materialized: 0, relinked: 0, removed: 0 }); + assert.equal(existsSync(join(nm, "pino", "package.json")), true); + + const missing = materializeBundledSymlinks(join(root, "does-not-exist")); + assert.deepEqual(missing, { materialized: 0, relinked: 0, removed: 0 }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("syncRebuiltNativeModuleIntoHashedEntries overwrites a hashed entry with the rebuilt root module", () => { + const root = mkdtempSync(join(tmpdir(), "sync-hashed-")); + try { + const rootModule = makePkg(root, "better-sqlite3", "electron-abi-rebuilt"); + + const nm = join(root, "nested", "node_modules"); + makePkg(nm, "better-sqlite3-90e2652d1716b047", "stale-node-abi"); + + const summary = syncRebuiltNativeModuleIntoHashedEntries(rootModule, nm); + + assert.equal(summary.synced, 1); + const target = join(nm, "better-sqlite3-90e2652d1716b047"); + assert.equal( + JSON.parse(readFileSync(join(target, "package.json"), "utf8")).marker, + "electron-abi-rebuilt" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("syncRebuiltNativeModuleIntoHashedEntries overwrites a plain-named entry too", () => { + const root = mkdtempSync(join(tmpdir(), "sync-plain-")); + try { + const rootModule = makePkg(root, "better-sqlite3", "electron-abi-rebuilt"); + + const nm = join(root, "nested", "node_modules"); + makePkg(nm, "better-sqlite3", "stale-node-abi"); + + const summary = syncRebuiltNativeModuleIntoHashedEntries(rootModule, nm); + + assert.equal(summary.synced, 1); + assert.equal( + JSON.parse(readFileSync(join(nm, "better-sqlite3", "package.json"), "utf8")).marker, + "electron-abi-rebuilt" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("syncRebuiltNativeModuleIntoHashedEntries no-ops when root module or nested node_modules is missing", () => { + const root = mkdtempSync(join(tmpdir(), "sync-noop-")); + try { + const rootModule = join(root, "does-not-exist", "better-sqlite3"); + const nm = join(root, "nested", "node_modules"); + makePkg(nm, "better-sqlite3", "stale-node-abi"); + + const missingRoot = syncRebuiltNativeModuleIntoHashedEntries(rootModule, nm); + assert.deepEqual(missingRoot, { synced: 0 }); + + const realRootModule = makePkg(root, "better-sqlite3", "electron-abi-rebuilt"); + const missingNm = syncRebuiltNativeModuleIntoHashedEntries( + realRootModule, + join(root, "does-not-exist-nm") + ); + assert.deepEqual(missingNm, { synced: 0 }); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("syncRebuiltNativeModuleIntoHashedEntries leaves unrelated entries untouched", () => { + const root = mkdtempSync(join(tmpdir(), "sync-unrelated-")); + try { + const rootModule = makePkg(root, "better-sqlite3", "electron-abi-rebuilt"); + + const nm = join(root, "nested", "node_modules"); + makePkg(nm, "pino", "real-pino"); + makePkg(nm, "better-sqlite3-helper", "unrelated-package"); + + const summary = syncRebuiltNativeModuleIntoHashedEntries(rootModule, nm); + + assert.deepEqual(summary, { synced: 0 }); + assert.equal( + JSON.parse(readFileSync(join(nm, "pino", "package.json"), "utf8")).marker, + "real-pino" + ); + assert.equal( + JSON.parse(readFileSync(join(nm, "better-sqlite3-helper", "package.json"), "utf8")).marker, + "unrelated-package" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +});