diff --git a/docs/ops/RELEASE_CHECKLIST.md b/docs/ops/RELEASE_CHECKLIST.md index 80990f8317..c0e7546997 100644 --- a/docs/ops/RELEASE_CHECKLIST.md +++ b/docs/ops/RELEASE_CHECKLIST.md @@ -272,7 +272,11 @@ Before shipping any v3.8.x release, verify these additional items: `@tensorflow/tfjs`, `js-tiktoken`) survive an update. The ultra `modelPath` SLM tier additionally needs `@huggingface/transformers@3.5.2` (pinned — llmlingua-2 uses the 3.x tokenizer API) and the tinybert model, auto-downloaded to `${DATA_DIR}/models/llmlingua` - on first use. + on first use. Postinstall (`scripts/build/colocateOptionals.mjs`) then co-locates the SLM + optional closure into `dist/node_modules` so the worker resolves a SINGLE + `@huggingface/transformers` 3.5.2 instance — the standalone trace bundles only transformers, + not the dynamically-imported optionals, so without this the worker would load llmlingua-2 + against the root's transformers and the SLM tier would silently fail-open. - [ ] `omniroute status` works with no `.env` (CLI token path, loopback only) - [ ] `curl http://localhost:20128/api/shutdown` returns 401 (always-protected route) - [ ] `curl -H "host: evil.com" http://localhost:20128/api/mcp/sse` returns 401 (loopback guard) diff --git a/package.json b/package.json index aace26853b..67146c5532 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "bin/cli/runtime/", "scripts/postinstall.mjs", "scripts/build/postinstallSupport.mjs", + "scripts/build/colocateOptionals.mjs", "scripts/build/sync-env.mjs", "scripts/dev/responses-ws-proxy.mjs", "scripts/check/check-supported-node-runtime.ts", diff --git a/scripts/build/colocateOptionals.mjs b/scripts/build/colocateOptionals.mjs new file mode 100644 index 0000000000..5317d2fe8a --- /dev/null +++ b/scripts/build/colocateOptionals.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node + +/** + * OmniRoute — Co-locate the LLMLingua-2 optional dependency closure into the standalone bundle. + * + * The compression "ultra" SLM tier (PR #4257) runs `@atjsh/llmlingua-2` + + * `@huggingface/transformers` + `@tensorflow/tfjs` + `js-tiktoken` inside a worker thread + * (`open-sse/services/compression/engines/llmlingua/onnxWorker.js`, shipped under `dist/`). These + * are `optionalDependencies`: npm installs them into the ROOT `node_modules` on + * `--include=optional`, but the Next.js standalone trace bundles ONLY `@huggingface/transformers` + * (3.5.2, pinned) into `dist/node_modules` — it does NOT trace the optional, dynamically-imported + * SLM packages. + * + * ## Why this matters (the instance-split bug) + * + * The worker lives under `dist/`, so its `import("@huggingface/transformers")` resolves + * `dist/node_modules/@huggingface/transformers` (3.5.2) and the worker sets the model `cacheDir` + * on THAT instance's `env`. But its `import("@atjsh/llmlingua-2")` walks past `dist/node_modules` + * (no `@atjsh` there) up to the ROOT `node_modules`, and llmlingua-2's own + * `import("@huggingface/transformers")` then resolves the ROOT transformers — a DIFFERENT instance. + * The `cacheDir`/`localModelPath` config the worker set never reaches the instance llmlingua-2 + * actually uses, so the local model under `DATA_DIR/models/llmlingua` is never found and the SLM + * tier silently fails-open (no compression). Worse, if the root transformers is a 4.x line, + * llmlingua-2 throws on a tokenizer-API change (`decoder.decode` is undefined). + * + * ## The fix + * + * Co-locate the SLM optional dependency CLOSURE from the root `node_modules` into + * `dist/node_modules` (NO-CLOBBER, so the pinned `dist` transformers 3.5.2 / onnxruntime / sharp + * stay). Then the worker resolves `@atjsh/llmlingua-2` AND `@huggingface/transformers` from the + * SAME `dist/node_modules` — a single 3.5.2 instance — so the env config applies and the local + * model loads. + * + * `@huggingface/transformers` is intentionally NOT a closure seed: it is a PEER of + * `@atjsh/llmlingua-2` (not a regular dependency) and is already bundled in `dist/node_modules`, + * so the closure walk never reaches it and the no-clobber guard would skip it anyway. + * + * ## Validation (Hard Rule #18) + * + * Manual co-location of this exact closure on the production VPS produced real 54.8% compression + * (11520 → 5203 chars) via real ONNX inference — both the default and the `modelPath` (PR #4257) + * code paths. See the unit test for the closure-walk + no-clobber contract. + * + * Idempotent + fail-soft: skips when the optionals are absent (the common case — they are OPTIONAL) + * or already co-located; a per-package copy failure only disables the SLM tier, which is itself + * fail-open, so this never throws into the install. + */ + +import { cpSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +/** + * Entry packages of the SLM optional stack (the closure roots). `@huggingface/transformers` is + * deliberately absent — it is the pinned instance already present in `dist/node_modules`. + */ +export const SEED_PACKAGES = ["@atjsh/llmlingua-2", "@tensorflow/tfjs", "js-tiktoken"]; + +/** + * Compute the transitive dependency closure of `seeds` by walking each package's `dependencies` + + * `optionalDependencies` from a `node_modules` directory. Packages that are not present in that + * tree (e.g. peers provided elsewhere, like `@huggingface/transformers` in `dist`) are skipped — + * the closure only contains packages that actually exist in `nodeModulesDir`. + * + * @param {string} nodeModulesDir absolute path to the source `node_modules` + * @param {string[]} [seeds] closure roots (defaults to {@link SEED_PACKAGES}) + * @returns {string[]} package names in discovery order, seeds first + */ +export function computeDependencyClosure(nodeModulesDir, seeds = SEED_PACKAGES) { + const closure = []; + const seen = new Set(); + const stack = [...seeds]; + + while (stack.length) { + const name = stack.shift(); + if (seen.has(name)) continue; + seen.add(name); + + const pkgDir = join(nodeModulesDir, name); + if (!existsSync(pkgDir)) continue; // absent in this tree (peer provided elsewhere) — skip + + closure.push(name); + + let manifest; + try { + manifest = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf8")); + } catch { + continue; // unreadable/absent manifest — copy the dir but do not recurse + } + + const deps = { ...manifest.dependencies, ...manifest.optionalDependencies }; + for (const dep of Object.keys(deps)) { + if (!seen.has(dep)) stack.push(dep); + } + } + + return closure; +} + +/** + * Co-locate the SLM optional closure from `/node_modules` into + * `/dist/node_modules`. No-op when the standalone `dist` bundle or the optional seeds are + * absent, and idempotent once co-located. Never throws. + * + * @param {{ rootDir: string, log?: (message: string) => void }} opts + * @returns {{ skipped: true, reason: string } + * | { skipped: false, copied: number, closure: number }} + */ +export function colocateLlmlinguaOptionals({ rootDir, log = () => {} }) { + const rootNm = join(rootDir, "node_modules"); + const distNm = join(rootDir, "dist", "node_modules"); + + if (!existsSync(distNm)) { + return { skipped: true, reason: "no standalone dist/node_modules" }; + } + // Gate: only run when the optional stack was actually installed (`npm install --include=optional`). + if (!SEED_PACKAGES.every((seed) => existsSync(join(rootNm, seed)))) { + return { skipped: true, reason: "SLM optionals not installed at root" }; + } + // Idempotent: the entry package is already co-located → nothing to do. + if (existsSync(join(distNm, "@atjsh", "llmlingua-2"))) { + return { skipped: true, reason: "already co-located" }; + } + + const closure = computeDependencyClosure(rootNm); + let copied = 0; + + for (const name of closure) { + const dest = join(distNm, name); + if (existsSync(dest)) continue; // no-clobber: keep dist's pinned copy (transformers 3.5.2, …) + try { + mkdirSync(dirname(dest), { recursive: true }); + cpSync(join(rootNm, name), dest, { recursive: true }); + copied++; + } catch (err) { + log(` ⚠️ LLMLingua optional co-location failed for ${name}: ${err.message}`); + } + } + + if (copied > 0) { + log(` ✅ Co-located ${copied} LLMLingua SLM optional package(s) into dist/node_modules.\n`); + } + + return { skipped: false, copied, closure: closure.length }; +} diff --git a/scripts/build/pack-artifact-policy.ts b/scripts/build/pack-artifact-policy.ts index 5b7c66106f..7c02c86245 100644 --- a/scripts/build/pack-artifact-policy.ts +++ b/scripts/build/pack-artifact-policy.ts @@ -95,6 +95,7 @@ export const PACK_ARTIFACT_ROOT_ALLOWED_EXACT_PATHS: string[] = [ "scripts/build/native-binary-compat.mjs", "scripts/build/postinstall.mjs", "scripts/build/postinstallSupport.mjs", + "scripts/build/colocateOptionals.mjs", "scripts/build/sync-env.mjs", "scripts/dev/responses-ws-proxy.mjs", "scripts/dev/sync-env.mjs", @@ -135,6 +136,7 @@ export const PACK_ARTIFACT_REQUIRED_PATHS: string[] = [ "scripts/build/native-binary-compat.mjs", "scripts/build/postinstall.mjs", "scripts/build/postinstallSupport.mjs", + "scripts/build/colocateOptionals.mjs", "src/shared/utils/nodeRuntimeSupport.ts", ]; diff --git a/scripts/build/postinstall.mjs b/scripts/build/postinstall.mjs index deecca2a12..9ce512643a 100644 --- a/scripts/build/postinstall.mjs +++ b/scripts/build/postinstall.mjs @@ -28,6 +28,7 @@ import { fileURLToPath } from "node:url"; import { PUBLISHED_BUILD_ARCH, PUBLISHED_BUILD_PLATFORM } from "./native-binary-compat.mjs"; import { hasStandaloneAppBundle, isTermux } from "./postinstallSupport.mjs"; +import { colocateLlmlinguaOptionals } from "./colocateOptionals.mjs"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -327,9 +328,24 @@ async function syncProjectEnv() { } } +/** + * Co-locate the LLMLingua-2 SLM optional dependency closure into dist/node_modules so the + * compression "ultra" SLM tier (PR #4257) resolves a single @huggingface/transformers instance at + * runtime. No-op unless the optionals were installed (`--include=optional`). See colocateOptionals.mjs. + */ +async function ensureLlmlinguaOptionals() { + try { + colocateLlmlinguaOptionals({ rootDir: ROOT, log: (m) => console.log(m) }); + } catch (err) { + // Best-effort: the SLM tier is itself fail-open, so a co-location hiccup never fails the install. + console.warn(` ⚠️ LLMLingua optional co-location skipped: ${err.message}`); + } +} + await fixBetterSqliteBinary(); await fixWreqJsBinary(); await ensureSwcHelpers(); +await ensureLlmlinguaOptionals(); await syncProjectEnv(); // Warm up native runtimes (better-sqlite3 in ~/.omniroute/runtime/). diff --git a/tests/unit/colocate-optionals.test.ts b/tests/unit/colocate-optionals.test.ts new file mode 100644 index 0000000000..b9832c580b --- /dev/null +++ b/tests/unit/colocate-optionals.test.ts @@ -0,0 +1,181 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { tmpdir } from "node:os"; + +import { + computeDependencyClosure, + colocateLlmlinguaOptionals, + SEED_PACKAGES, +} from "../../scripts/build/colocateOptionals.mjs"; + +/** Create a fake installed package with a manifest and optional extra files. */ +function mkPkg( + nmDir: string, + name: string, + manifest: Record = {}, + files: Record = {} +): void { + const dir = join(nmDir, name); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "package.json"), JSON.stringify({ name, version: "1.0.0", ...manifest })); + for (const [rel, content] of Object.entries(files)) { + const fp = join(dir, rel); + mkdirSync(dirname(fp), { recursive: true }); + writeFileSync(fp, content); + } +} + +/** + * Build a root tree mirroring the real SLM optional shape: + * @atjsh/llmlingua-2 → dep es-toolkit, PEER @huggingface/transformers (+ tfjs, js-tiktoken) + * @tensorflow/tfjs → dep @tensorflow/tfjs-core → dep long + * js-tiktoken → dep base64-js + * @huggingface/transformers present at root as a (stale) 4.2.0 + */ +function buildRoot(rootDir: string): void { + const rootNm = join(rootDir, "node_modules"); + mkPkg( + rootNm, + "@atjsh/llmlingua-2", + { + dependencies: { "es-toolkit": "^1.38.0" }, + peerDependencies: { + "@huggingface/transformers": "*", + "@tensorflow/tfjs": "*", + "js-tiktoken": "*", + }, + }, + { "dist/index.js": "export const llmlingua = true;\n" } + ); + mkPkg(rootNm, "es-toolkit", {}); + mkPkg(rootNm, "@tensorflow/tfjs", { dependencies: { "@tensorflow/tfjs-core": "4.22.0" } }); + mkPkg(rootNm, "@tensorflow/tfjs-core", { dependencies: { long: "^5.0.0" } }); + mkPkg(rootNm, "long", {}); + mkPkg(rootNm, "js-tiktoken", { dependencies: { "base64-js": "^1.5.1" } }); + mkPkg(rootNm, "base64-js", {}); + // Root transformers is the STALE 4.x line — the bug we must not propagate into dist. + mkPkg(rootNm, "@huggingface/transformers", { version: "4.2.0" }); +} + +test("computeDependencyClosure walks deps transitively and skips peers (transformers)", () => { + const root = mkdtempSync(join(tmpdir(), "omniroute-colocate-closure-")); + try { + buildRoot(root); + const closure = computeDependencyClosure(join(root, "node_modules")); + + for (const expected of [ + "@atjsh/llmlingua-2", + "@tensorflow/tfjs", + "js-tiktoken", + "es-toolkit", + "@tensorflow/tfjs-core", + "long", + "base64-js", + ]) { + assert.ok(closure.includes(expected), `closure should include ${expected}`); + } + // The peer (declared via peerDependencies, NOT dependencies) must NOT be pulled in. + assert.ok( + !closure.includes("@huggingface/transformers"), + "closure must NOT include the transformers peer" + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("colocateLlmlinguaOptionals copies the closure into dist and never clobbers dist transformers", () => { + const root = mkdtempSync(join(tmpdir(), "omniroute-colocate-copy-")); + try { + buildRoot(root); + // dist already ships the PINNED transformers (3.5.2) — must survive untouched. + const distNm = join(root, "dist", "node_modules"); + mkPkg(distNm, "@huggingface/transformers", { version: "3.5.2" }); + + const result = colocateLlmlinguaOptionals({ rootDir: root }); + assert.equal(result.skipped, false); + if (result.skipped === false) { + assert.ok(result.copied >= 6, `expected >=6 packages copied, got ${result.copied}`); + } + + // Full closure landed in dist/node_modules. + for (const name of [ + "@atjsh/llmlingua-2", + "es-toolkit", + "@tensorflow/tfjs", + "@tensorflow/tfjs-core", + "long", + "js-tiktoken", + "base64-js", + ]) { + assert.ok(existsSync(join(distNm, name)), `${name} should be co-located into dist`); + } + // The package payload came along (not just the manifest). + assert.ok(existsSync(join(distNm, "@atjsh", "llmlingua-2", "dist", "index.js"))); + + // CRITICAL: dist's pinned transformers is preserved — root's 4.2.0 must NOT win. + const distTransformers = JSON.parse( + readFileSync(join(distNm, "@huggingface", "transformers", "package.json"), "utf8") + ); + assert.equal(distTransformers.version, "3.5.2", "dist transformers must remain 3.5.2"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("colocateLlmlinguaOptionals is idempotent (second run is a no-op)", () => { + const root = mkdtempSync(join(tmpdir(), "omniroute-colocate-idem-")); + try { + buildRoot(root); + mkPkg(join(root, "dist", "node_modules"), "@huggingface/transformers", { version: "3.5.2" }); + + const first = colocateLlmlinguaOptionals({ rootDir: root }); + assert.equal(first.skipped, false); + + const second = colocateLlmlinguaOptionals({ rootDir: root }); + assert.equal(second.skipped, true); + if (second.skipped === true) { + assert.equal(second.reason, "already co-located"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("colocateLlmlinguaOptionals skips when SLM optionals are not installed", () => { + const root = mkdtempSync(join(tmpdir(), "omniroute-colocate-noopt-")); + try { + // dist bundle exists, but the optional seeds were never installed at root. + mkPkg(join(root, "dist", "node_modules"), "@huggingface/transformers", { version: "3.5.2" }); + mkdirSync(join(root, "node_modules"), { recursive: true }); + + const result = colocateLlmlinguaOptionals({ rootDir: root }); + assert.equal(result.skipped, true); + if (result.skipped === true) { + assert.equal(result.reason, "SLM optionals not installed at root"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("colocateLlmlinguaOptionals skips when there is no standalone dist bundle", () => { + const root = mkdtempSync(join(tmpdir(), "omniroute-colocate-nodist-")); + try { + buildRoot(root); // optionals present, but no dist/node_modules + const result = colocateLlmlinguaOptionals({ rootDir: root }); + assert.equal(result.skipped, true); + if (result.skipped === true) { + assert.equal(result.reason, "no standalone dist/node_modules"); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("SEED_PACKAGES excludes transformers (it is a dist-pinned peer, not a seed)", () => { + assert.ok(!SEED_PACKAGES.includes("@huggingface/transformers")); + assert.deepEqual(SEED_PACKAGES, ["@atjsh/llmlingua-2", "@tensorflow/tfjs", "js-tiktoken"]); +}); diff --git a/tests/unit/pack-artifact-policy.test.ts b/tests/unit/pack-artifact-policy.test.ts index 2316f5361f..08b78d4a5b 100644 --- a/tests/unit/pack-artifact-policy.test.ts +++ b/tests/unit/pack-artifact-policy.test.ts @@ -99,6 +99,7 @@ test("findMissingArtifactPaths flags missing root runtime files in the tarball", "dist/responses-ws-proxy.mjs", "dist/server-ws.mjs", "dist/webdav-handler.mjs", + "scripts/build/colocateOptionals.mjs", "scripts/build/native-binary-compat.mjs", "src/shared/utils/nodeRuntimeSupport.ts", ]);