diff --git a/changelog.d/fixes/6835-db-oom-probe-cyclebreaker.md b/changelog.d/fixes/6835-db-oom-probe-cyclebreaker.md new file mode 100644 index 0000000000..3208171d3f --- /dev/null +++ b/changelog.d/fixes/6835-db-oom-probe-cyclebreaker.md @@ -0,0 +1 @@ +- **fix(db):** cap the sql.js OOM-during-probe path in `getDbInstance()` at 3 attempts with a terminal diagnostic — previously only the generic-corruption probe-failure path had a cycle-breaker (#6632), so a persistently OOMing `storage.sqlite` probe re-threw the identical error forever on every call from every background poller, hanging the app with "Internal Server Error" and no self-recovery (#6835). diff --git a/src/lib/db/core.ts b/src/lib/db/core.ts index 0fa67ac01e..c9004262f8 100644 --- a/src/lib/db/core.ts +++ b/src/lib/db/core.ts @@ -466,6 +466,14 @@ declare global { // Next.js HMR re-evaluations so concurrent subsystems all see the same // count and we abort with a clear error instead of looping forever. var __omnirouteDbProbeRestoreCount: number | undefined; + // Cycle-breaker counter for the OOM-during-probe path (#6835). Unlike the + // generic corruption path above, an OOM probe failure never renames the + // file away (intentional — the DB may be perfectly fine, just too large + // for the current heap), so the restore-count cap above is structurally + // unreachable here. Without an independent cap, every background poller + // (BATCH, HealthCheck, ProviderLimitsSync, ModelSync) re-throws the same + // OOM error forever with no terminal diagnostic. + var __omnirouteDbOomFailureCount: number | undefined; } function getDb(): SqliteDatabase | null { @@ -1076,6 +1084,22 @@ export function getDbInstance(): SqliteDatabase { // immediately gives the user a clear "increase --max-old-space-size" // signal instead of silently renaming a perfectly good DB. if (/out of memory|allocation failure|Array buffer allocation failed|allocation failed/i.test(message)) { + // Cycle-breaker (#6835): the OOM path never renames the file away, + // so it never trips the generic probe-failed/restore cap above. Cap + // it independently after 3 consecutive OOM failures (same threshold + // as the generic path) so repeated polling doesn't hang forever with + // no actionable terminal diagnostic. + if ( + (globalThis.__omnirouteDbOomFailureCount = + (globalThis.__omnirouteDbOomFailureCount || 0) + 1) > 3 + ) { + throw new Error( + `[DB] Aborting startup: persistent out-of-memory probing ${sqliteFile} after 3 attempts. ` + + `Increase the V8 heap with NODE_OPTIONS=--max-old-space-size=4096 (or higher) — the ` + + `current heap is insufficient for this database — and restart, or shrink/restore the ` + + `database from a backup. Original error: ${message}` + ); + } throw new Error( `[DB] Out of memory while probing ${sqliteFile}. ` + `The bundled sql.js driver loads the entire file into WASM memory; ` + diff --git a/tests/unit/probe-6835-cyclebreaker.test.ts b/tests/unit/probe-6835-cyclebreaker.test.ts new file mode 100644 index 0000000000..fda653124d --- /dev/null +++ b/tests/unit/probe-6835-cyclebreaker.test.ts @@ -0,0 +1,28 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +test("getDbInstance() caps the probe-failed/restore cycle at 3 attempts (#6835)", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-6835-")); + process.env.DATA_DIR = tmpDir; + const sqliteFile = path.join(tmpDir, "storage.sqlite"); + const backupFile = `${sqliteFile}.probe-failed-1000000000000`; + fs.writeFileSync(backupFile, Buffer.from("not a real sqlite file, always fails to open")); + const core = await import("../../src/lib/db/core.ts"); + const errors: string[] = []; + for (let i = 0; i < 6; i++) { + try { + core.getDbInstance(); + errors.push("(no error)"); + break; + } catch (err: unknown) { + errors.push(err instanceof Error ? err.message : String(err)); + } + } + const abortIndex = errors.findIndex((e) => e.includes("Aborting startup")); + assert.notEqual(abortIndex, -1, "Expected the cap to trip; got: " + errors.join(" | ")); + assert.ok(abortIndex <= 4, "Expected cap by call #4; took until #" + abortIndex); + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); diff --git a/tests/unit/probe-6835-oom-uncapped.test.ts b/tests/unit/probe-6835-oom-uncapped.test.ts new file mode 100644 index 0000000000..9b91101646 --- /dev/null +++ b/tests/unit/probe-6835-oom-uncapped.test.ts @@ -0,0 +1,58 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +test("getDbInstance() eventually caps a persistently-OOMing sql.js probe (#6835)", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-6835-oom-")); + process.env.DATA_DIR = tmpDir; + const sqliteFile = path.join(tmpDir, "storage.sqlite"); + fs.mkdirSync(sqliteFile); // forces better-sqlite3/node:sqlite to fail synchronously (EISDIR-style) + await import("../../src/lib/db/adapters/driverFactory.ts"); + const core = await import("../../src/lib/db/core.ts"); + const fakeAdapter = { + driver: "sql.js" as const, + open: true, + name: sqliteFile, + prepare() { + throw new Error("out of memory"); + }, + exec() { + throw new Error("out of memory"); + }, + pragma() { + throw new Error("out of memory"); + }, + transaction(fn: (...a: unknown[]) => T) { + return fn; + }, + immediate() {}, + async backup() {}, + checkpoint() {}, + close() {}, + raw: null, + }; + ( + globalThis as unknown as { __omnirouteSqlJsAdapters: Map } + ).__omnirouteSqlJsAdapters = new Map([[sqliteFile, fakeAdapter]]); + const errors: string[] = []; + for (let i = 0; i < 8; i++) { + try { + core.getDbInstance(); + errors.push("(no error)"); + break; + } catch (err: unknown) { + errors.push(err instanceof Error ? err.message : String(err)); + } + } + const anyAborted = errors.some((e) => e.includes("Aborting startup")); + assert.ok( + anyAborted, + "Expected getDbInstance() to eventually give up with a terminal " + + "'Aborting startup'-style diagnostic after repeated OOM probe failures, the same way it " + + "already does for generic corruption (#6632). Instead every call re-threw an identical, " + + "uncapped OOM error:\n" + errors.map((e, i) => ` [${i}] ${e}`).join("\n") + ); + fs.rmSync(tmpDir, { recursive: true, force: true }); +});