From 96a33053bcbaa33caec30ec50f576a1fb9a3d6ce Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza <8016841+diegosouzapw@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:16:17 -0300 Subject: [PATCH] fix: page call_logs cleanup queries to avoid startup OOM on large DBs (#5618) (#5675) --- CHANGELOG.md | 2 + src/lib/usage/callLogs.ts | 30 ++--- src/lib/usage/callLogsBoundedQueries.ts | 42 +++++++ .../unit/call-log-oom-unbounded-5618.test.ts | 111 ++++++++++++++++++ 4 files changed, 170 insertions(+), 15 deletions(-) create mode 100644 src/lib/usage/callLogsBoundedQueries.ts create mode 100644 tests/unit/call-log-oom-unbounded-5618.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa73ded58..dd7bbea563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### 🔧 Bug Fixes +- **storage (daemon):** fix a Node.js **out-of-memory crash on startup** when `storage.sqlite` grows large (~170 MB+). The boot-time call-log cleanup (`cleanupExpiredLogs` → `rotateCallLogs`) ran two **unbounded `SELECT … FROM call_logs … .all()`** queries — `listReferencedArtifacts` (every artifact path) and `deleteCallLogsBefore` (every id before the retention cutoff). `node:sqlite`'s `StatementSync.all()` materializes the entire result set as JS objects at once, so on a large table the V8 heap blew up and the process crashed before binding (`FATAL ERROR: … heap out of memory`, native frame `node::sqlite::StatementSync::All`). Both queries now page through `call_logs` in bounded 5 000-row chunks (new `src/lib/usage/callLogsBoundedQueries.ts`), keeping peak memory flat regardless of table size — no more manual `--max-old-space-size` bump required. Regression guard: `tests/unit/call-log-oom-unbounded-5618.test.ts`. ([#5618](https://github.com/diegosouzapw/OmniRoute/issues/5618)) + - **dashboard (provider setup):** fix three provider setup links that pointed at 404 pages. **Ollama Cloud** / **ollama-search** linked to `ollama.com/settings/api-keys` → corrected to `ollama.com/settings/keys` (the page moved; Ollama Cloud is a real keyed service, so the field stays). **SearchAPI** linked to the bare `searchapi.io/docs` (404) → `searchapi.io/docs/google`. **You.com** linked to `you.com/docs/search/overview` (404) → `you.com/business/api/` (the developer portal). All three replacements were verified live. Regression guard: `tests/unit/provider-setup-links-5572.test.ts`. ([#5572](https://github.com/diegosouzapw/OmniRoute/issues/5572), [#5574](https://github.com/diegosouzapw/OmniRoute/issues/5574), [#5576](https://github.com/diegosouzapw/OmniRoute/issues/5576)) - **providers (AI/ML API):** the model-import step now loads the **live** AI/ML API catalog (400+ models) instead of falling back to a stale 6-model seed. The registry had no `modelsUrl`, so the route silently used the bundled catalog with an "API unavailable — using local catalog" warning even when the key was valid. AI/ML API exposes its full catalog at the public, **auth-free** `https://api.aimlapi.com/models` endpoint (a bare array of `{ id, type, info }`, distinct from the OpenAI-compat `/v1/models`); it's now wired into the models route's discovery config, with the bundled catalog kept as the offline fallback. Regression guard: `tests/unit/provider-models-route.test.ts`. ([#5570](https://github.com/diegosouzapw/OmniRoute/issues/5570)) diff --git a/src/lib/usage/callLogs.ts b/src/lib/usage/callLogs.ts index 048f75addd..92d69912cb 100644 --- a/src/lib/usage/callLogs.ts +++ b/src/lib/usage/callLogs.ts @@ -9,6 +9,7 @@ import fs from "node:fs"; import path from "node:path"; import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts"; import { getDbInstance } from "../db/core"; +import { collectReferencedArtifacts, selectCallLogIdsBefore } from "./callLogsBoundedQueries"; import { getRequestDetailLogByCallLogId } from "../db/detailedLogs"; import { shouldPersistToDisk } from "./migrations"; import { @@ -415,14 +416,8 @@ function clearArtifactReference(relativePath: string, nextState: CallLogDetailSt } function listReferencedArtifacts() { - const db = getDbInstance(); - const rows = db - .prepare("SELECT artifact_relpath FROM call_logs WHERE artifact_relpath IS NOT NULL") - .all() as Array<{ artifact_relpath: string | null }>; - - return new Set( - rows.map((row) => row.artifact_relpath).filter((value): value is string => Boolean(value)) - ); + // #5618: paged to avoid an unbounded `.all()` OOM on large call_logs tables. + return collectReferencedArtifacts(); } // #5217: SQLite caps a statement at SQLITE_MAX_VARIABLE_NUMBER bound params @@ -510,13 +505,18 @@ export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries? } export function deleteCallLogsBefore(cutoff: string): DeleteResult { - const db = getDbInstance(); - const ids = db - .prepare("SELECT id FROM call_logs WHERE timestamp < ? ORDER BY timestamp ASC") - .all(cutoff) - .map((row) => String((row as { id: string }).id)); - - return deleteCallLogRowsByIds(ids); + // #5618: page the id selection so a large backlog never loads in one `.all()`. + let deletedRows = 0; + let deletedArtifacts = 0; + for (;;) { + const ids = selectCallLogIdsBefore(cutoff); + if (ids.length === 0) break; + const result = deleteCallLogRowsByIds(ids); + deletedRows += result.deletedRows; + deletedArtifacts += result.deletedArtifacts; + if (result.deletedRows === 0) break; + } + return { deletedRows, deletedArtifacts }; } export function trimCallLogsToMaxRows(maxRows = getCallLogsTableMaxRows()) { diff --git a/src/lib/usage/callLogsBoundedQueries.ts b/src/lib/usage/callLogsBoundedQueries.ts new file mode 100644 index 0000000000..5fa9719178 --- /dev/null +++ b/src/lib/usage/callLogsBoundedQueries.ts @@ -0,0 +1,42 @@ +import { getDbInstance } from "../db/core"; + +// #5618 — node:sqlite's StatementSync.all() materializes the ENTIRE result set as +// JS objects at once. On a large storage.sqlite (~170 MB+) an unbounded +// `SELECT … FROM call_logs` over the whole table blows the V8 heap during the +// startup cleanup pass (`cleanupExpiredLogs` → `rotateCallLogs`), crashing the +// daemon before it binds. These helpers page through call_logs in bounded chunks +// so peak memory stays flat regardless of table size. +const CALL_LOG_QUERY_PAGE = 5000; + +/** + * Collect every non-null `artifact_relpath` referenced by call_logs, paging with + * LIMIT/OFFSET so a huge table never loads into memory in one `.all()`. + */ +export function collectReferencedArtifacts(): Set { + const db = getDbInstance(); + const referenced = new Set(); + const stmt = db.prepare( + "SELECT artifact_relpath FROM call_logs WHERE artifact_relpath IS NOT NULL LIMIT ? OFFSET ?" + ); + for (let offset = 0; ; offset += CALL_LOG_QUERY_PAGE) { + const rows = stmt.all(CALL_LOG_QUERY_PAGE, offset) as Array<{ artifact_relpath: string | null }>; + for (const row of rows) { + if (typeof row.artifact_relpath === "string") referenced.add(row.artifact_relpath); + } + if (rows.length < CALL_LOG_QUERY_PAGE) break; + } + return referenced; +} + +/** + * Select one bounded page of call_log ids older than `cutoff` (oldest first). + * Callers loop until it returns an empty page, deleting each batch, so the id + * list never grows to the full retention backlog at once. + */ +export function selectCallLogIdsBefore(cutoff: string, limit = CALL_LOG_QUERY_PAGE): string[] { + const db = getDbInstance(); + const rows = db + .prepare("SELECT id FROM call_logs WHERE timestamp < ? ORDER BY timestamp ASC LIMIT ?") + .all(cutoff, limit) as Array<{ id: string }>; + return rows.map((row) => String(row.id)); +} diff --git a/tests/unit/call-log-oom-unbounded-5618.test.ts b/tests/unit/call-log-oom-unbounded-5618.test.ts new file mode 100644 index 0000000000..e5ff9757f7 --- /dev/null +++ b/tests/unit/call-log-oom-unbounded-5618.test.ts @@ -0,0 +1,111 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +// #5618 — `cleanupExpiredLogs` → `rotateCallLogs` ran at daemon startup and used +// unbounded `SELECT … FROM call_logs … .all()` calls. node:sqlite's +// StatementSync.all() materializes the whole result set, so on a large +// storage.sqlite (~170 MB+) the JS heap blew up and the process crashed before +// binding. The two startup queries must page with LIMIT instead of loading the +// whole table at once. + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-calllogs-oom-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.CALL_LOG_RETENTION_DAYS = "3650"; + +const core = await import("../../src/lib/db/core.ts"); +const callLogs = await import("../../src/lib/usage/callLogs.ts"); +const bounded = await import("../../src/lib/usage/callLogsBoundedQueries.ts"); + +function insertCallLog(id: string, timestamp: string, artifact: string | null = null) { + const db = core.getDbInstance(); + db.prepare( + `INSERT INTO call_logs (id, timestamp, method, path, status, model, provider, detail_state, artifact_relpath) + VALUES (@id, @timestamp, 'POST', '/v1/chat/completions', 200, 'openai/gpt-4.1', 'openai', 'none', @artifact)` + ).run({ id, timestamp, artifact }); +} + +function seed(total: number, withArtifact: boolean) { + const db = core.getDbInstance(); + const base = Date.parse("2026-01-01T00:00:00.000Z"); + db.transaction(() => { + for (let i = 0; i < total; i++) { + insertCallLog( + `r-${String(i).padStart(6, "0")}`, + new Date(base + i * 1000).toISOString(), + withArtifact ? `2026-01/${i}.json` : null + ); + } + })(); +} + +function captureSql(run: () => void): string[] { + const db = core.getDbInstance(); + const orig = db.prepare.bind(db); + const sqls: string[] = []; + (db as unknown as { prepare: unknown }).prepare = (sql: string) => { + sqls.push(sql); + return orig(sql); + }; + try { + run(); + } finally { + (db as unknown as { prepare: unknown }).prepare = orig; + } + return sqls; +} + +const unboundedSelectsOnCallLogs = (sqls: string[]) => + sqls.filter( + (s) => /SELECT/i.test(s) && /\bFROM\s+call_logs\b/i.test(s) && !/LIMIT/i.test(s) + ); + +test.beforeEach(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +}); + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("#5618 collectReferencedArtifacts pages with LIMIT and collects across pages — no unbounded .all()", () => { + const total = 6000; // > one 5000-row page → exercises the pagination loop + seed(total, true); + + let referenced: Set | undefined; + const sqls = captureSql(() => { + referenced = bounded.collectReferencedArtifacts(); + }); + + assert.equal(referenced!.size, total, "every artifact path is collected across all pages"); + assert.deepEqual( + unboundedSelectsOnCallLogs(sqls), + [], + `unbounded SELECT on call_logs (OOM risk): ${unboundedSelectsOnCallLogs(sqls).join("; ")}` + ); +}); + +test("#5618 deleteCallLogsBefore selects ids with LIMIT (bounded) instead of all at once", () => { + const total = 1200; + seed(total, false); + + let result: { deletedRows: number } | undefined; + const sqls = captureSql(() => { + result = callLogs.deleteCallLogsBefore("2030-01-01T00:00:00.000Z"); + }); + + assert.equal(result!.deletedRows, total, "all rows before the cutoff are deleted (across pages)"); + const unboundedIdSelects = sqls.filter( + (s) => /SELECT\s+id\s+FROM\s+call_logs/i.test(s) && !/LIMIT/i.test(s) + ); + assert.deepEqual( + unboundedIdSelects, + [], + `unbounded id SELECT on call_logs (OOM risk): ${unboundedIdSelects.join("; ")}` + ); +});