From fc3ce839feefece134c15f22d44300f1f5dc629d Mon Sep 17 00:00:00 2001 From: "Bob.Hou" Date: Fri, 11 Sep 2026 18:27:52 -0400 Subject: [PATCH] fix(db): resolve sql-wasm.wasm across global and hoisted layouts (#12960) (#13035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct root cause chain, and it explains why this only bites global installs: npm 11 skips `optionalDependencies` install scripts, the server child runs with `cwd: /dist`, and `sql.js` sits at `/node_modules`. Probing the parent directory is the missing rung. --- Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017). - `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓ - complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline - 531 of 532 focused assertions green across the batch's 46 test files - `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied. ⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here). Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit. --- .../fixes/12960-sqljs-wasm-global-path.md | 1 + docs/reference/ENVIRONMENT.md | 1 + src/lib/db/adapters/sqljsAdapter.ts | 113 ++++++++- .../unit/sqljs-wasm-resolution-12960.test.ts | 235 ++++++++++++++++++ 4 files changed, 340 insertions(+), 10 deletions(-) create mode 100644 changelog.d/fixes/12960-sqljs-wasm-global-path.md create mode 100644 tests/unit/sqljs-wasm-resolution-12960.test.ts diff --git a/changelog.d/fixes/12960-sqljs-wasm-global-path.md b/changelog.d/fixes/12960-sqljs-wasm-global-path.md new file mode 100644 index 0000000000..a77ef9a9bf --- /dev/null +++ b/changelog.d/fixes/12960-sqljs-wasm-global-path.md @@ -0,0 +1 @@ +- **fix(db):** resolve `sql-wasm.wasm` across global npm install and hoisted layouts, ensuring OmniRoute can boot cleanly on Node 24 when native `better-sqlite3` is uncompiled. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index bce25352e8..9a66d935f9 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -102,6 +102,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari | `OMNIROUTE_API_KEY_BASE64` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** (Base64-encoded form) accepted as a fallback. Decoded automatically before use. | | `OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS` | _(unset)_ | `src/lib/db/core.ts` | Override the periodic SQLite healthcheck interval (ms). When unset, defaults are derived from `NODE_ENV`. | | `OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS` | `21600000` (6h) | `src/lib/db/walMaintenance.ts` | Override the periodic `wal_checkpoint(TRUNCATE)` interval (ms). Auto-checkpoint never shrinks the WAL file itself, and a long-running server never closes its DB. `0` disables. | +| `OMNIROUTE_SQLJS_WASM_PATH` | _(auto-detect)_ | `src/lib/db/adapters/sqljsAdapter.ts` | Explicit path (absolute or relative to cwd) to `sql-wasm.wasm` when using the `sql.js` WASM fallback adapter. Auto-detected via package dependencies and candidate layouts when unset. | | `OMNIROUTE_SKIP_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts`, `src/lib/db/healthCheck.ts` | Set to `1` to skip the DB healthcheck entirely on startup. Useful for short-lived tasks and integration tests. | | `OMNIROUTE_FORCE_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts` | Set to `1` to force the DB healthcheck loop on, even when it would normally be skipped (e.g., short-lived tasks). | | `OMNIROUTE_SKIP_POSTINSTALL` | `0` | `scripts/postinstall.mjs` | Set to `1` to skip the native-runtime warm-up during `npm install`. Useful in CI/headless installs where sqlite is already built. | diff --git a/src/lib/db/adapters/sqljsAdapter.ts b/src/lib/db/adapters/sqljsAdapter.ts index c908c7eaaf..f60024033f 100644 --- a/src/lib/db/adapters/sqljsAdapter.ts +++ b/src/lib/db/adapters/sqljsAdapter.ts @@ -1,6 +1,7 @@ // src/lib/db/adapters/sqljsAdapter.ts import fs from "node:fs"; import path from "node:path"; +import * as nodeModule from "node:module"; import type { SqliteAdapter, PreparedStatement, RunResult } from "./types"; const SAVE_DEBOUNCE_MS = 100; @@ -21,14 +22,67 @@ function toPlainRow(row: T): T { let _sqlJsLib: Awaited> | null = null; -function resolveSqlJsWasmPath(): string { - // The standalone assembler copies the complete sql.js package into - // /node_modules/sql.js. Every packaged server launcher sets cwd to that - // bundle directory, so the JavaScript entrypoint and its sibling WASM share one - // explicit runtime contract instead of relying on a require.resolve call that - // webpack can rewrite. The second path retains direct-source compatibility. - const candidatePaths = [ +/** + * Resolves the absolute on-disk path to `sql-wasm.wasm`. + * + * Precedence order: + * 0. `OMNIROUTE_SQLJS_WASM_PATH` env override (validated to be a non-empty, non-directory file, + * and resolved to an absolute path). + * 1. Layout candidate paths checked relative to `process.cwd()`: + * - `/node_modules/sql.js/dist/sql-wasm.wasm` (standard standalone layout) + * - `/../node_modules/sql.js/dist/sql-wasm.wasm` (global npm install CLI layout, where + * child process cwd is `/dist` while dependencies are under `/node_modules`) + * - `/.next/standalone/node_modules/sql.js/dist/sql-wasm.wasm` (direct source / legacy) + * 2. Dynamic resolution via `createRequire` anchored at `process.cwd()` and `process.argv[1]` + * (handles hoisted, symlinked, pnpm, or non-standard node_modules topologies). + * + * Throws an actionable Error explaining how to rebuild better-sqlite3 or provide the WASM binary + * if none of the above locate a valid file. + */ +export function resolveSqlJsWasmPath(): string { + // 0. Explicit environment variable override + if (process.env.OMNIROUTE_SQLJS_WASM_PATH != null) { + const raw = process.env.OMNIROUTE_SQLJS_WASM_PATH; + const trimmed = raw.trim(); + if (trimmed.length === 0) { + throw new Error( + `[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to an empty or whitespace-only string.\n` + + `Unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection, or set it to the path of a valid sql-wasm.wasm file.` + ); + } + const resolvedPath = path.resolve(trimmed); + let stat: fs.Stats; + try { + stat = fs.statSync(resolvedPath); + } catch (err) { + throw new Error( + `[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the file cannot be accessed: ${(err as Error).message}\n` + + `Verify the path or unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection.` + ); + } + if (stat.isDirectory()) { + throw new Error( + `[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the path points to a directory, not a file.\n` + + `Set it to the full path of sql-wasm.wasm or unset the variable to allow auto-detection.` + ); + } + if (!stat.isFile() || stat.size === 0) { + throw new Error( + `[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the file is empty (size=0) or not a regular file.\n` + + `Verify the path or unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection.` + ); + } + return resolvedPath; + } + + // 1. Explicit layout candidate paths checked first against process.cwd() + const candidatePaths: string[] = [ + // Standard standalone layout (/node_modules/sql.js/...) path.join(process.cwd(), "node_modules", "sql.js", "dist", "sql-wasm.wasm"), + // Global CLI install (#12960): `omniroute serve` child process sets cwd + // to /dist, while npm installs dependencies at /node_modules + path.join(process.cwd(), "..", "node_modules", "sql.js", "dist", "sql-wasm.wasm"), + // Direct source / legacy standalone layouts path.join( process.cwd(), ".next", @@ -46,10 +100,49 @@ function resolveSqlJsWasmPath(): string { } } + // 2. Dynamic module resolution via createRequire across standard anchors. + // sql.js package.json declares exports: { "./dist/*": "./dist/*" }, so + // resolving "sql.js/dist/sql-wasm.wasm" is officially supported and handles + // any hoisted, symlinked, pnpm, or non-standard node_modules layout. + // Note: process.argv[1] can be undefined in embedded Node or worker contexts; + // the `|| ""` fallback ensures safe string handling, filtered by !anchor. + const anchors = [process.cwd(), process.argv[1] || ""]; + + for (const anchor of anchors) { + if (!anchor) continue; + try { + const runtimeRequire = nodeModule.createRequire(anchor); + const resolved = runtimeRequire.resolve("sql.js/dist/sql-wasm.wasm"); + if (resolved && fs.existsSync(resolved)) { + return resolved; + } + } catch (err: unknown) { + // Swallowing MODULE_NOT_FOUND / ERR_MODULE_NOT_FOUND is expected when sql.js is not + // resolvable from this specific anchor. Unexpected errors (e.g. EACCES, corrupted + // package metadata) should be rethrown so operators see the real failure. + const code = (err as { code?: string })?.code; + const msg = (err as Error)?.message || ""; + const isNotFound = + code === "MODULE_NOT_FOUND" || + code === "ERR_MODULE_NOT_FOUND" || + msg.includes("Cannot find module"); + if (!isNotFound) { + throw err; + } + } + } + throw new Error( - `[sqljsAdapter] Packaged sql.js runtime is incomplete: sql-wasm.wasm was not found. Checked:\n${candidatePaths.join( - "\n" - )}` + `[sqljsAdapter] Packaged sql.js runtime is incomplete: sql-wasm.wasm was not found.\n` + + `The fallback WASM runtime could not locate sql-wasm.wasm at any checked location.\n` + + `Checked locations:\n${candidatePaths.map((p) => ` - ${p}`).join("\n")}\n\n` + + `Remedy:\n` + + ` * If running a global npm install without native SQLite (better-sqlite3), rebuild it:\n` + + ` cd $(npm root -g)/omniroute && npm rebuild better-sqlite3\n` + + ` * If running locally, rebuild better-sqlite3:\n` + + ` npm rebuild better-sqlite3\n` + + ` * Or set OMNIROUTE_SQLJS_WASM_PATH to the path of sql-wasm.wasm.\n` + + ` * See docs/guides/TROUBLESHOOTING.md for details.` ); } diff --git a/tests/unit/sqljs-wasm-resolution-12960.test.ts b/tests/unit/sqljs-wasm-resolution-12960.test.ts new file mode 100644 index 0000000000..86f4752cb8 --- /dev/null +++ b/tests/unit/sqljs-wasm-resolution-12960.test.ts @@ -0,0 +1,235 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import os from "node:os"; + +import { resolveSqlJsWasmPath } from "../../src/lib/db/adapters/sqljsAdapter.ts"; + +describe("sql.js WASM path resolution (#12960)", () => { + test("resolves an existing sql-wasm.wasm in the current environment", (t) => { + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + t.after(() => { + if (origEnv !== undefined) { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + }); + + const wasmPath = resolveSqlJsWasmPath(); + assert.ok(typeof wasmPath === "string" && wasmPath.length > 0); + assert.ok(fs.existsSync(wasmPath), `Resolved path must exist: ${wasmPath}`); + assert.ok(wasmPath.endsWith("sql-wasm.wasm")); + }); + + test("honors OMNIROUTE_SQLJS_WASM_PATH when set to a valid file", (t) => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-override-")); + const fakeWasm = path.join(tmpDir, "custom-sql-wasm.wasm"); + fs.writeFileSync(fakeWasm, "mock wasm binary"); + + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = fakeWasm; + + t.after(() => { + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const resolved = resolveSqlJsWasmPath(); + assert.equal(resolved, fakeWasm); + }); + + test("resolves relative OMNIROUTE_SQLJS_WASM_PATH to an absolute path", (t) => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-rel-override-")); + const fakeWasm = path.join(tmpDir, "rel-sql-wasm.wasm"); + fs.writeFileSync(fakeWasm, "mock wasm binary"); + + const origCwd = process.cwd(); + process.chdir(tmpDir); + + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = "./rel-sql-wasm.wasm"; + + t.after(() => { + process.chdir(origCwd); + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const resolved = resolveSqlJsWasmPath(); + assert.ok(path.isAbsolute(resolved)); + assert.equal(fs.realpathSync(resolved), fs.realpathSync(fakeWasm)); + }); + + test("throws when OMNIROUTE_SQLJS_WASM_PATH points to non-existent file", (t) => { + const nonExistent = path.join(os.tmpdir(), `non-existent-wasm-${Date.now()}.wasm`); + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = nonExistent; + + t.after(() => { + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + }); + + assert.throws( + () => resolveSqlJsWasmPath(), + /OMNIROUTE_SQLJS_WASM_PATH is set to .* but the file cannot be accessed/ + ); + }); + + test("throws when OMNIROUTE_SQLJS_WASM_PATH is set to an empty string", (t) => { + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = " "; + + t.after(() => { + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + }); + + assert.throws( + () => resolveSqlJsWasmPath(), + /OMNIROUTE_SQLJS_WASM_PATH is set to an empty or whitespace-only string/ + ); + }); + + test("throws when OMNIROUTE_SQLJS_WASM_PATH points to a directory", (t) => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-dir-wasm-")); + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = tmpDir; + + t.after(() => { + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + assert.throws(() => resolveSqlJsWasmPath(), /points to a directory, not a file/); + }); + + test("throws when OMNIROUTE_SQLJS_WASM_PATH points to an empty (0-byte) file", (t) => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-empty-wasm-")); + const emptyWasm = path.join(tmpDir, "empty.wasm"); + fs.writeFileSync(emptyWasm, ""); + + const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH; + process.env.OMNIROUTE_SQLJS_WASM_PATH = emptyWasm; + + t.after(() => { + if (origEnv === undefined) { + delete process.env.OMNIROUTE_SQLJS_WASM_PATH; + } else { + process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + assert.throws(() => resolveSqlJsWasmPath(), /file is empty \(size=0\)/); + }); + + test("resolves from parent node_modules when cwd is /dist (global install layout)", (t) => { + // Simulate: + // /lib/node_modules/omniroute/dist/ <-- cwd + // /lib/node_modules/omniroute/node_modules/sql.js/dist/sql-wasm.wasm + const tmpBase = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-global-layout-")); + const pkgRoot = path.join(tmpBase, "lib", "node_modules", "omniroute"); + const distDir = path.join(pkgRoot, "dist"); + const sqlJsDist = path.join(pkgRoot, "node_modules", "sql.js", "dist"); + + fs.mkdirSync(distDir, { recursive: true }); + fs.mkdirSync(sqlJsDist, { recursive: true }); + + const targetWasm = path.join(sqlJsDist, "sql-wasm.wasm"); + fs.writeFileSync(targetWasm, "mock wasm"); + + const origCwd = process.cwd(); + process.chdir(distDir); + + t.after(() => { + process.chdir(origCwd); + fs.rmSync(tmpBase, { recursive: true, force: true }); + }); + + const resolved = resolveSqlJsWasmPath(); + assert.equal(fs.realpathSync(resolved), fs.realpathSync(targetWasm)); + }); + + test("throws an actionable error naming the remedy when WASM cannot be found", (t) => { + const tmpEmpty = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-empty-")); + const origCwd = process.cwd(); + const origArgv1 = process.argv[1]; + + process.chdir(tmpEmpty); + // Point argv[1] to a non-existent location inside tmpEmpty so require.resolve cannot escape + process.argv[1] = path.join(tmpEmpty, "dummy-server.js"); + + t.after(() => { + process.chdir(origCwd); + process.argv[1] = origArgv1; + fs.rmSync(tmpEmpty, { recursive: true, force: true }); + }); + + let thrownError: Error | null = null; + try { + resolveSqlJsWasmPath(); + } catch (err) { + thrownError = err as Error; + } + + assert.ok(thrownError, "Expected resolveSqlJsWasmPath to throw"); + const msg = thrownError.message; + + // Must name the packaged sql.js problem + assert.match(msg, /\[sqljsAdapter\] Packaged sql\.js runtime is incomplete/); + // Must explain that the fallback WASM runtime could not locate the binary + assert.match(msg, /fallback WASM runtime could not locate sql-wasm\.wasm/); + // Must provide the actionable remedy for global and local installs (#12960) + assert.match(msg, /npm rebuild better-sqlite3/); + assert.match(msg, /docs\/guides\/TROUBLESHOOTING\.md/); + assert.match(msg, /OMNIROUTE_SQLJS_WASM_PATH/); + }); + + test("rethrows non-MODULE_NOT_FOUND unexpected errors during require resolution", (t) => { + const origCwd = process.cwd(); + const origArgv1 = process.argv[1]; + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-rethrow-")); + process.chdir(tmpDir); + + process.argv[1] = "\0invalid_null_byte_path"; + + t.after(() => { + process.chdir(origCwd); + process.argv[1] = origArgv1; + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + assert.throws( + () => resolveSqlJsWasmPath(), + (err: unknown) => { + const error = err as Error; + return ( + error.name === "TypeError" || + (error as { code?: string }).code === "ERR_INVALID_ARG_VALUE" || + error.message.includes("null byte") + ); + } + ); + }); +});