diff --git a/scripts/build/colocateOptionals.mjs b/scripts/build/colocateOptionals.mjs index 91548954b0..367a95c112 100644 --- a/scripts/build/colocateOptionals.mjs +++ b/scripts/build/colocateOptionals.mjs @@ -47,7 +47,8 @@ */ import { cpSync, existsSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { createRequire } from "node:module"; +import { dirname, join, sep } from "node:path"; /** * Entry packages of the SLM optional stack (the closure roots). `@huggingface/transformers` is @@ -96,6 +97,33 @@ export function computeDependencyClosure(nodeModulesDir, seeds = SEED_PACKAGES) return closure; } +/** + * A package in the target tree counts as PRESENT only when its entrypoint + * resolves from inside that tree — the same contract the Dockerfile's + * post-build guard enforces. Next's file tracing can materialize a package + * PARTIALLY (the package.json lands, the files its `main` points at do not), + * and a directory-level `existsSync` check then skips the package forever + * while the runtime dies with "Cannot find module /dist/index.js". + * + * @param {string} targetNodeModulesDir + * @param {string} name + * @returns {boolean} + */ +function isPackageIntact(targetNodeModulesDir, name) { + if (!existsSync(join(targetNodeModulesDir, name))) return false; + try { + const probe = createRequire( + join(targetNodeModulesDir, "__colocate_probe__.js") + ); + const resolved = probe.resolve(name); + // A resolution that walked past the target into an ancestor tree does not + // prove the target copy is usable. + return resolved.startsWith(targetNodeModulesDir + sep); + } catch { + return false; + } +} + /** * Co-locate the SLM optional dependency closure from `/node_modules` * into a standalone bundle's `node_modules`. @@ -142,11 +170,12 @@ export function colocateLlmlinguaOptionals({ const closure = computeDependencyClosure(rootNm, seeds); - // Check the complete closure rather than only the entry package. A partially - // populated bundle must still receive any missing transitive dependencies. + // Check the complete closure rather than only the entry package, and judge + // presence by entrypoint integrity — a partially traced directory (see + // isPackageIntact) must still receive its missing files. if ( closure.length > 0 && - closure.every((name) => existsSync(join(targetNm, name))) + closure.every((name) => isPackageIntact(targetNm, name)) ) { return { skipped: true, reason: "already co-located" }; } @@ -155,11 +184,18 @@ export function colocateLlmlinguaOptionals({ for (const name of closure) { const dest = join(targetNm, name); - if (existsSync(dest)) continue; + if (isPackageIntact(targetNm, name)) continue; try { mkdirSync(dirname(dest), { recursive: true }); - cpSync(join(rootNm, name), dest, { recursive: true }); + // force:false merges into a partially traced directory: files the trace + // already materialized are kept, missing ones (the package payload) are + // filled in from the root tree. + cpSync(join(rootNm, name), dest, { + recursive: true, + force: false, + errorOnExist: false, + }); copied++; } catch (err) { log( diff --git a/tests/unit/docker-llmlingua-optionals-9166.test.ts b/tests/unit/docker-llmlingua-optionals-9166.test.ts index ba1f245743..da994ed5c4 100644 --- a/tests/unit/docker-llmlingua-optionals-9166.test.ts +++ b/tests/unit/docker-llmlingua-optionals-9166.test.ts @@ -55,6 +55,7 @@ function buildLlmlinguaRoot( rootNm, "@atjsh/llmlingua-2", { + main: "dist/index.js", dependencies: { "es-toolkit": "^1.38.0", }, @@ -227,6 +228,103 @@ test("#9166 standalone assembly never overwrites an already pinned transformers } }); +test("#9166 co-location completes a partially traced package (package.json without its main)", () => { + const root = mkdtempSync( + join(tmpdir(), "omniroute-docker-llmlingua-partial-9166-") + ); + + try { + buildLlmlinguaRoot(root); + const { distDir, standaloneDir } = createStandalone(root); + + // Next's file tracing materializes @atjsh/llmlingua-2 PARTIALLY in the + // standalone: the package.json lands (its "main" points at dist/index.js) + // but the dist/ payload does not — the exact state the Docker guard hits + // ("Cannot find module .../dist/index.js"). A directory-level no-clobber + // sees the dir and skips the package forever. + mkPkg(join(standaloneDir, "node_modules"), "@atjsh/llmlingua-2", { + main: "dist/index.js", + }); + + assembleStandalone({ + distDir, + outDir: standaloneDir, + projectRoot: root, + copyNatives: true, + }); + + assert.ok( + existsSync( + join( + standaloneDir, + "node_modules", + "@atjsh", + "llmlingua-2", + "dist", + "index.js" + ) + ), + "a partially traced package must be completed, not skipped as already present" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("#9166 co-location is not skipped when every closure dir exists but one is partial", () => { + const root = mkdtempSync( + join(tmpdir(), "omniroute-docker-llmlingua-partial-all-9166-") + ); + + try { + buildLlmlinguaRoot(root); + const { distDir, standaloneDir } = createStandalone(root); + const standaloneNm = join(standaloneDir, "node_modules"); + + // Every closure package already has a directory in the standalone (so a + // directory-level "already co-located" early-exit would fire), but the + // llmlingua-2 one is the partial NFT-trace shell without its main. + for (const packageName of [ + "es-toolkit", + "@tensorflow/tfjs", + "@tensorflow/tfjs-core", + "long", + "js-tiktoken", + "base64-js", + "@huggingface/transformers", + "onnxruntime-node", + ]) { + mkPkg(standaloneNm, packageName, { main: "index.js" }, { + "index.js": "export {};\n", + }); + } + mkPkg(standaloneNm, "@atjsh/llmlingua-2", { main: "dist/index.js" }); + + assembleStandalone({ + distDir, + outDir: standaloneDir, + projectRoot: root, + copyNatives: true, + }); + + assert.ok( + existsSync( + join( + standaloneDir, + "node_modules", + "@atjsh", + "llmlingua-2", + "dist", + "index.js" + ) + ), + "the closure-wide early-exit must not fire while any member is partial" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("#9166 Docker explicitly installs and validates LLMLingua optionals", () => { const dockerfile = readFileSync( new URL("../../Dockerfile", import.meta.url),