fix(db): cap OOM probe-failure cycle in getDbInstance() (#6835) (#7186)

When better-sqlite3/node:sqlite are unavailable and the sql.js WASM
fallback OOMs while probing storage.sqlite, getDbInstance() rethrew
an identical 'Out of memory while probing' error on every call,
forever — unlike the generic-corruption probe-failure path (#6632),
which correctly caps at 3 attempts via the restore-count cycle
breaker. Because the OOM path never renames the file away
(intentional — OOM is not corruption), the existing cap is
structurally unreachable for this branch, so every independent
background poller (BATCH, ProviderLimitsSync, HealthCheck,
ModelSync) kept re-triggering the same failure with no terminal
diagnostic, hanging the app forever.

Adds an independent __omnirouteDbOomFailureCount cycle-breaker
mirroring the existing threshold of 3, throwing a distinct terminal
'Aborting startup' diagnostic after repeated OOM failures instead of
looping. Does not touch the rename/backup safety mechanism.

Reported-by: xHmeyer, mostafa-binesh

changelog.d/fixes/6835-db-oom-probe-cyclebreaker.md

@@ -0,0 +1 @@
+- **fix(db):** cap the sql.js OOM-during-probe path in `getDbInstance()` at 3 attempts with a terminal diagnostic — previously only the generic-corruption probe-failure path had a cycle-breaker (#6632), so a persistently OOMing `storage.sqlite` probe re-threw the identical error forever on every call from every background poller, hanging the app with "Internal Server Error" and no self-recovery (#6835).

src/lib/db/core.ts

@@ -466,6 +466,14 @@ declare global {
   // Next.js HMR re-evaluations so concurrent subsystems all see the same
   // count and we abort with a clear error instead of looping forever.
   var __omnirouteDbProbeRestoreCount: number | undefined;
+  // Cycle-breaker counter for the OOM-during-probe path (#6835). Unlike the
+  // generic corruption path above, an OOM probe failure never renames the
+  // file away (intentional — the DB may be perfectly fine, just too large
+  // for the current heap), so the restore-count cap above is structurally
+  // unreachable here. Without an independent cap, every background poller
+  // (BATCH, HealthCheck, ProviderLimitsSync, ModelSync) re-throws the same
+  // OOM error forever with no terminal diagnostic.
+  var __omnirouteDbOomFailureCount: number | undefined;
 }
 
 function getDb(): SqliteDatabase | null {
@@ -1076,6 +1084,22 @@ export function getDbInstance(): SqliteDatabase {
       // immediately gives the user a clear "increase --max-old-space-size"
       // signal instead of silently renaming a perfectly good DB.
       if (/out of memory|allocation failure|Array buffer allocation failed|allocation failed/i.test(message)) {
+        // Cycle-breaker (#6835): the OOM path never renames the file away,
+        // so it never trips the generic probe-failed/restore cap above. Cap
+        // it independently after 3 consecutive OOM failures (same threshold
+        // as the generic path) so repeated polling doesn't hang forever with
+        // no actionable terminal diagnostic.
+        if (
+          (globalThis.__omnirouteDbOomFailureCount =
+            (globalThis.__omnirouteDbOomFailureCount || 0) + 1) > 3
+        ) {
+          throw new Error(
+            `[DB] Aborting startup: persistent out-of-memory probing ${sqliteFile} after 3 attempts. ` +
+              `Increase the V8 heap with NODE_OPTIONS=--max-old-space-size=4096 (or higher) — the ` +
+              `current heap is insufficient for this database — and restart, or shrink/restore the ` +
+              `database from a backup. Original error: ${message}`
+          );
+        }
         throw new Error(
           `[DB] Out of memory while probing ${sqliteFile}. ` +
             `The bundled sql.js driver loads the entire file into WASM memory; ` +

tests/unit/probe-6835-cyclebreaker.test.ts

@@ -0,0 +1,28 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+test("getDbInstance() caps the probe-failed/restore cycle at 3 attempts (#6835)", async () => {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-6835-"));
+  process.env.DATA_DIR = tmpDir;
+  const sqliteFile = path.join(tmpDir, "storage.sqlite");
+  const backupFile = `${sqliteFile}.probe-failed-1000000000000`;
+  fs.writeFileSync(backupFile, Buffer.from("not a real sqlite file, always fails to open"));
+  const core = await import("../../src/lib/db/core.ts");
+  const errors: string[] = [];
+  for (let i = 0; i < 6; i++) {
+    try {
+      core.getDbInstance();
+      errors.push("(no error)");
+      break;
+    } catch (err: unknown) {
+      errors.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+  const abortIndex = errors.findIndex((e) => e.includes("Aborting startup"));
+  assert.notEqual(abortIndex, -1, "Expected the cap to trip; got: " + errors.join(" | "));
+  assert.ok(abortIndex <= 4, "Expected cap by call #4; took until #" + abortIndex);
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+});

tests/unit/probe-6835-oom-uncapped.test.ts

@@ -0,0 +1,58 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+test("getDbInstance() eventually caps a persistently-OOMing sql.js probe (#6835)", async () => {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-6835-oom-"));
+  process.env.DATA_DIR = tmpDir;
+  const sqliteFile = path.join(tmpDir, "storage.sqlite");
+  fs.mkdirSync(sqliteFile); // forces better-sqlite3/node:sqlite to fail synchronously (EISDIR-style)
+  await import("../../src/lib/db/adapters/driverFactory.ts");
+  const core = await import("../../src/lib/db/core.ts");
+  const fakeAdapter = {
+    driver: "sql.js" as const,
+    open: true,
+    name: sqliteFile,
+    prepare() {
+      throw new Error("out of memory");
+    },
+    exec() {
+      throw new Error("out of memory");
+    },
+    pragma() {
+      throw new Error("out of memory");
+    },
+    transaction<T>(fn: (...a: unknown[]) => T) {
+      return fn;
+    },
+    immediate() {},
+    async backup() {},
+    checkpoint() {},
+    close() {},
+    raw: null,
+  };
+  (
+    globalThis as unknown as { __omnirouteSqlJsAdapters: Map<string, unknown> }
+  ).__omnirouteSqlJsAdapters = new Map([[sqliteFile, fakeAdapter]]);
+  const errors: string[] = [];
+  for (let i = 0; i < 8; i++) {
+    try {
+      core.getDbInstance();
+      errors.push("(no error)");
+      break;
+    } catch (err: unknown) {
+      errors.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+  const anyAborted = errors.some((e) => e.includes("Aborting startup"));
+  assert.ok(
+    anyAborted,
+    "Expected getDbInstance() to eventually give up with a terminal " +
+      "'Aborting startup'-style diagnostic after repeated OOM probe failures, the same way it " +
+      "already does for generic corruption (#6632). Instead every call re-threw an identical, " +
+      "uncapped OOM error:\n" + errors.map((e, i) => `  [${i}] ${e}`).join("\n")
+  );
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+});