fix(db): resolve sql-wasm.wasm across global and hoisted layouts (#12960) (#13035)

Correct root cause chain, and it explains why this only bites global installs: npm 11 skips `optionalDependencies` install scripts, the server child runs with `cwd: <packageRoot>/dist`, and `sql.js` sits at `<packageRoot>/node_modules`. Probing the parent directory is the missing rung.

---

Validated in one consolidated worktree cut from `release/v3.8.51`, boarded with the other 19 PRs of this batch. Two in-batch conflicts, both additive and resolved by keeping each side: the `ENVIRONMENT.md` table (#13035 + #13011) and the `chatHelpers.ts` import block (#12975 on the tip + #13017).

- `typecheck:core` clean; `check:dashboard-typecheck` OK (206 pre-existing, within baseline); `check:changelog-integrity` OK; `check:docs-counts` migrations ✓
- complexity 2816 / baseline 3218 and cognitive-complexity 1271 / baseline 1437 — both under baseline
- 531 of 532 focused assertions green across the batch's 46 test files
- `check-file-size` rebaselined for the batch's real growth (annotation `_rebaseline_2026_09_11_mergebatch_v3851_houminxi`, landed on #13038), attributed per PR

The single red is **not this batch**: `tests/unit/combo/quota-weighted-strategy.test.ts` → "A/B isolation: 7 hard-empty + 2 at 0.5% + 1 at 40%, floor=1" asserts an order between two connections of identical weight and flakes on the pure tip too — 2 failures in 4 runs at `origin/release/v3.8.51` with nothing from this batch applied.

⚠️ base-red inherited: #12732 — `Docs Gates`, `Merge integrity`, `No new ESLint warnings`, `Unit Tests fast-path` and `Fast Quality Gates` reproduce on the pure tip (provider count 356 vs the 358 the modules define, SKILL.md drift, and `open-sse/utils/stream.ts` at 3115 > frozen 3098, untouched here).

Thanks @HouMinXi — the live evidence on these (X500 logs, `storage.sqlite` state, real `/v1/models` probes, the 36-minute outage write-up) is what let a 20-PR batch be reviewed as a unit.
This commit is contained in:
Bob.Hou
2026-09-11 18:27:52 -04:00
committed by GitHub
parent 541a6481a9
commit fc3ce839fe
4 changed files with 340 additions and 10 deletions

View File

@@ -0,0 +1 @@
- **fix(db):** resolve `sql-wasm.wasm` across global npm install and hoisted layouts, ensuring OmniRoute can boot cleanly on Node 24 when native `better-sqlite3` is uncompiled.

View File

@@ -102,6 +102,7 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari
| `OMNIROUTE_API_KEY_BASE64` | _(unset)_ | `src/lib/db/encryption.ts` | **Legacy alias** (Base64-encoded form) accepted as a fallback. Decoded automatically before use. |
| `OMNIROUTE_DB_HEALTHCHECK_INTERVAL_MS` | _(unset)_ | `src/lib/db/core.ts` | Override the periodic SQLite healthcheck interval (ms). When unset, defaults are derived from `NODE_ENV`. |
| `OMNIROUTE_WAL_TRUNCATE_INTERVAL_MS` | `21600000` (6h) | `src/lib/db/walMaintenance.ts` | Override the periodic `wal_checkpoint(TRUNCATE)` interval (ms). Auto-checkpoint never shrinks the WAL file itself, and a long-running server never closes its DB. `0` disables. |
| `OMNIROUTE_SQLJS_WASM_PATH` | _(auto-detect)_ | `src/lib/db/adapters/sqljsAdapter.ts` | Explicit path (absolute or relative to cwd) to `sql-wasm.wasm` when using the `sql.js` WASM fallback adapter. Auto-detected via package dependencies and candidate layouts when unset. |
| `OMNIROUTE_SKIP_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts`, `src/lib/db/healthCheck.ts` | Set to `1` to skip the DB healthcheck entirely on startup. Useful for short-lived tasks and integration tests. |
| `OMNIROUTE_FORCE_DB_HEALTHCHECK` | `0` | `src/lib/db/core.ts` | Set to `1` to force the DB healthcheck loop on, even when it would normally be skipped (e.g., short-lived tasks). |
| `OMNIROUTE_SKIP_POSTINSTALL` | `0` | `scripts/postinstall.mjs` | Set to `1` to skip the native-runtime warm-up during `npm install`. Useful in CI/headless installs where sqlite is already built. |

View File

@@ -1,6 +1,7 @@
// src/lib/db/adapters/sqljsAdapter.ts
import fs from "node:fs";
import path from "node:path";
import * as nodeModule from "node:module";
import type { SqliteAdapter, PreparedStatement, RunResult } from "./types";
const SAVE_DEBOUNCE_MS = 100;
@@ -21,14 +22,67 @@ function toPlainRow<T>(row: T): T {
let _sqlJsLib: Awaited<ReturnType<(typeof import("sql.js"))["default"]>> | null = null;
function resolveSqlJsWasmPath(): string {
// The standalone assembler copies the complete sql.js package into
// <bundle>/node_modules/sql.js. Every packaged server launcher sets cwd to that
// bundle directory, so the JavaScript entrypoint and its sibling WASM share one
// explicit runtime contract instead of relying on a require.resolve call that
// webpack can rewrite. The second path retains direct-source compatibility.
const candidatePaths = [
/**
* Resolves the absolute on-disk path to `sql-wasm.wasm`.
*
* Precedence order:
* 0. `OMNIROUTE_SQLJS_WASM_PATH` env override (validated to be a non-empty, non-directory file,
* and resolved to an absolute path).
* 1. Layout candidate paths checked relative to `process.cwd()`:
* - `<cwd>/node_modules/sql.js/dist/sql-wasm.wasm` (standard standalone layout)
* - `<cwd>/../node_modules/sql.js/dist/sql-wasm.wasm` (global npm install CLI layout, where
* child process cwd is `<pkgRoot>/dist` while dependencies are under `<pkgRoot>/node_modules`)
* - `<cwd>/.next/standalone/node_modules/sql.js/dist/sql-wasm.wasm` (direct source / legacy)
* 2. Dynamic resolution via `createRequire` anchored at `process.cwd()` and `process.argv[1]`
* (handles hoisted, symlinked, pnpm, or non-standard node_modules topologies).
*
* Throws an actionable Error explaining how to rebuild better-sqlite3 or provide the WASM binary
* if none of the above locate a valid file.
*/
export function resolveSqlJsWasmPath(): string {
// 0. Explicit environment variable override
if (process.env.OMNIROUTE_SQLJS_WASM_PATH != null) {
const raw = process.env.OMNIROUTE_SQLJS_WASM_PATH;
const trimmed = raw.trim();
if (trimmed.length === 0) {
throw new Error(
`[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to an empty or whitespace-only string.\n` +
`Unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection, or set it to the path of a valid sql-wasm.wasm file.`
);
}
const resolvedPath = path.resolve(trimmed);
let stat: fs.Stats;
try {
stat = fs.statSync(resolvedPath);
} catch (err) {
throw new Error(
`[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the file cannot be accessed: ${(err as Error).message}\n` +
`Verify the path or unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection.`
);
}
if (stat.isDirectory()) {
throw new Error(
`[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the path points to a directory, not a file.\n` +
`Set it to the full path of sql-wasm.wasm or unset the variable to allow auto-detection.`
);
}
if (!stat.isFile() || stat.size === 0) {
throw new Error(
`[sqljsAdapter] OMNIROUTE_SQLJS_WASM_PATH is set to "${trimmed}", but the file is empty (size=0) or not a regular file.\n` +
`Verify the path or unset OMNIROUTE_SQLJS_WASM_PATH to allow auto-detection.`
);
}
return resolvedPath;
}
// 1. Explicit layout candidate paths checked first against process.cwd()
const candidatePaths: string[] = [
// Standard standalone layout (<bundle>/node_modules/sql.js/...)
path.join(process.cwd(), "node_modules", "sql.js", "dist", "sql-wasm.wasm"),
// Global CLI install (#12960): `omniroute serve` child process sets cwd
// to <packageRoot>/dist, while npm installs dependencies at <packageRoot>/node_modules
path.join(process.cwd(), "..", "node_modules", "sql.js", "dist", "sql-wasm.wasm"),
// Direct source / legacy standalone layouts
path.join(
process.cwd(),
".next",
@@ -46,10 +100,49 @@ function resolveSqlJsWasmPath(): string {
}
}
// 2. Dynamic module resolution via createRequire across standard anchors.
// sql.js package.json declares exports: { "./dist/*": "./dist/*" }, so
// resolving "sql.js/dist/sql-wasm.wasm" is officially supported and handles
// any hoisted, symlinked, pnpm, or non-standard node_modules layout.
// Note: process.argv[1] can be undefined in embedded Node or worker contexts;
// the `|| ""` fallback ensures safe string handling, filtered by !anchor.
const anchors = [process.cwd(), process.argv[1] || ""];
for (const anchor of anchors) {
if (!anchor) continue;
try {
const runtimeRequire = nodeModule.createRequire(anchor);
const resolved = runtimeRequire.resolve("sql.js/dist/sql-wasm.wasm");
if (resolved && fs.existsSync(resolved)) {
return resolved;
}
} catch (err: unknown) {
// Swallowing MODULE_NOT_FOUND / ERR_MODULE_NOT_FOUND is expected when sql.js is not
// resolvable from this specific anchor. Unexpected errors (e.g. EACCES, corrupted
// package metadata) should be rethrown so operators see the real failure.
const code = (err as { code?: string })?.code;
const msg = (err as Error)?.message || "";
const isNotFound =
code === "MODULE_NOT_FOUND" ||
code === "ERR_MODULE_NOT_FOUND" ||
msg.includes("Cannot find module");
if (!isNotFound) {
throw err;
}
}
}
throw new Error(
`[sqljsAdapter] Packaged sql.js runtime is incomplete: sql-wasm.wasm was not found. Checked:\n${candidatePaths.join(
"\n"
)}`
`[sqljsAdapter] Packaged sql.js runtime is incomplete: sql-wasm.wasm was not found.\n` +
`The fallback WASM runtime could not locate sql-wasm.wasm at any checked location.\n` +
`Checked locations:\n${candidatePaths.map((p) => ` - ${p}`).join("\n")}\n\n` +
`Remedy:\n` +
` * If running a global npm install without native SQLite (better-sqlite3), rebuild it:\n` +
` cd $(npm root -g)/omniroute && npm rebuild better-sqlite3\n` +
` * If running locally, rebuild better-sqlite3:\n` +
` npm rebuild better-sqlite3\n` +
` * Or set OMNIROUTE_SQLJS_WASM_PATH to the path of sql-wasm.wasm.\n` +
` * See docs/guides/TROUBLESHOOTING.md for details.`
);
}

View File

@@ -0,0 +1,235 @@
import { test, describe } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import os from "node:os";
import { resolveSqlJsWasmPath } from "../../src/lib/db/adapters/sqljsAdapter.ts";
describe("sql.js WASM path resolution (#12960)", () => {
test("resolves an existing sql-wasm.wasm in the current environment", (t) => {
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
t.after(() => {
if (origEnv !== undefined) {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
});
const wasmPath = resolveSqlJsWasmPath();
assert.ok(typeof wasmPath === "string" && wasmPath.length > 0);
assert.ok(fs.existsSync(wasmPath), `Resolved path must exist: ${wasmPath}`);
assert.ok(wasmPath.endsWith("sql-wasm.wasm"));
});
test("honors OMNIROUTE_SQLJS_WASM_PATH when set to a valid file", (t) => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-override-"));
const fakeWasm = path.join(tmpDir, "custom-sql-wasm.wasm");
fs.writeFileSync(fakeWasm, "mock wasm binary");
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = fakeWasm;
t.after(() => {
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
fs.rmSync(tmpDir, { recursive: true, force: true });
});
const resolved = resolveSqlJsWasmPath();
assert.equal(resolved, fakeWasm);
});
test("resolves relative OMNIROUTE_SQLJS_WASM_PATH to an absolute path", (t) => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-rel-override-"));
const fakeWasm = path.join(tmpDir, "rel-sql-wasm.wasm");
fs.writeFileSync(fakeWasm, "mock wasm binary");
const origCwd = process.cwd();
process.chdir(tmpDir);
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = "./rel-sql-wasm.wasm";
t.after(() => {
process.chdir(origCwd);
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
fs.rmSync(tmpDir, { recursive: true, force: true });
});
const resolved = resolveSqlJsWasmPath();
assert.ok(path.isAbsolute(resolved));
assert.equal(fs.realpathSync(resolved), fs.realpathSync(fakeWasm));
});
test("throws when OMNIROUTE_SQLJS_WASM_PATH points to non-existent file", (t) => {
const nonExistent = path.join(os.tmpdir(), `non-existent-wasm-${Date.now()}.wasm`);
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = nonExistent;
t.after(() => {
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
});
assert.throws(
() => resolveSqlJsWasmPath(),
/OMNIROUTE_SQLJS_WASM_PATH is set to .* but the file cannot be accessed/
);
});
test("throws when OMNIROUTE_SQLJS_WASM_PATH is set to an empty string", (t) => {
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = " ";
t.after(() => {
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
});
assert.throws(
() => resolveSqlJsWasmPath(),
/OMNIROUTE_SQLJS_WASM_PATH is set to an empty or whitespace-only string/
);
});
test("throws when OMNIROUTE_SQLJS_WASM_PATH points to a directory", (t) => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-dir-wasm-"));
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = tmpDir;
t.after(() => {
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
fs.rmSync(tmpDir, { recursive: true, force: true });
});
assert.throws(() => resolveSqlJsWasmPath(), /points to a directory, not a file/);
});
test("throws when OMNIROUTE_SQLJS_WASM_PATH points to an empty (0-byte) file", (t) => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-empty-wasm-"));
const emptyWasm = path.join(tmpDir, "empty.wasm");
fs.writeFileSync(emptyWasm, "");
const origEnv = process.env.OMNIROUTE_SQLJS_WASM_PATH;
process.env.OMNIROUTE_SQLJS_WASM_PATH = emptyWasm;
t.after(() => {
if (origEnv === undefined) {
delete process.env.OMNIROUTE_SQLJS_WASM_PATH;
} else {
process.env.OMNIROUTE_SQLJS_WASM_PATH = origEnv;
}
fs.rmSync(tmpDir, { recursive: true, force: true });
});
assert.throws(() => resolveSqlJsWasmPath(), /file is empty \(size=0\)/);
});
test("resolves from parent node_modules when cwd is <pkg>/dist (global install layout)", (t) => {
// Simulate:
// <tmpRoot>/lib/node_modules/omniroute/dist/ <-- cwd
// <tmpRoot>/lib/node_modules/omniroute/node_modules/sql.js/dist/sql-wasm.wasm
const tmpBase = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-global-layout-"));
const pkgRoot = path.join(tmpBase, "lib", "node_modules", "omniroute");
const distDir = path.join(pkgRoot, "dist");
const sqlJsDist = path.join(pkgRoot, "node_modules", "sql.js", "dist");
fs.mkdirSync(distDir, { recursive: true });
fs.mkdirSync(sqlJsDist, { recursive: true });
const targetWasm = path.join(sqlJsDist, "sql-wasm.wasm");
fs.writeFileSync(targetWasm, "mock wasm");
const origCwd = process.cwd();
process.chdir(distDir);
t.after(() => {
process.chdir(origCwd);
fs.rmSync(tmpBase, { recursive: true, force: true });
});
const resolved = resolveSqlJsWasmPath();
assert.equal(fs.realpathSync(resolved), fs.realpathSync(targetWasm));
});
test("throws an actionable error naming the remedy when WASM cannot be found", (t) => {
const tmpEmpty = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-empty-"));
const origCwd = process.cwd();
const origArgv1 = process.argv[1];
process.chdir(tmpEmpty);
// Point argv[1] to a non-existent location inside tmpEmpty so require.resolve cannot escape
process.argv[1] = path.join(tmpEmpty, "dummy-server.js");
t.after(() => {
process.chdir(origCwd);
process.argv[1] = origArgv1;
fs.rmSync(tmpEmpty, { recursive: true, force: true });
});
let thrownError: Error | null = null;
try {
resolveSqlJsWasmPath();
} catch (err) {
thrownError = err as Error;
}
assert.ok(thrownError, "Expected resolveSqlJsWasmPath to throw");
const msg = thrownError.message;
// Must name the packaged sql.js problem
assert.match(msg, /\[sqljsAdapter\] Packaged sql\.js runtime is incomplete/);
// Must explain that the fallback WASM runtime could not locate the binary
assert.match(msg, /fallback WASM runtime could not locate sql-wasm\.wasm/);
// Must provide the actionable remedy for global and local installs (#12960)
assert.match(msg, /npm rebuild better-sqlite3/);
assert.match(msg, /docs\/guides\/TROUBLESHOOTING\.md/);
assert.match(msg, /OMNIROUTE_SQLJS_WASM_PATH/);
});
test("rethrows non-MODULE_NOT_FOUND unexpected errors during require resolution", (t) => {
const origCwd = process.cwd();
const origArgv1 = process.argv[1];
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sqljs-rethrow-"));
process.chdir(tmpDir);
process.argv[1] = "\0invalid_null_byte_path";
t.after(() => {
process.chdir(origCwd);
process.argv[1] = origArgv1;
fs.rmSync(tmpDir, { recursive: true, force: true });
});
assert.throws(
() => resolveSqlJsWasmPath(),
(err: unknown) => {
const error = err as Error;
return (
error.name === "TypeError" ||
(error as { code?: string }).code === "ERR_INVALID_ARG_VALUE" ||
error.message.includes("null byte")
);
}
);
});
});