fix: page call_logs cleanup queries to avoid startup OOM on large DBs (#5618) (#5675)

src/lib/usage/callLogs.ts

@@ -9,6 +9,7 @@ import fs from "node:fs";
 import path from "node:path";
 import type { RequestPipelinePayloads } from "@omniroute/open-sse/utils/requestLogger.ts";
 import { getDbInstance } from "../db/core";
+import { collectReferencedArtifacts, selectCallLogIdsBefore } from "./callLogsBoundedQueries";
 import { getRequestDetailLogByCallLogId } from "../db/detailedLogs";
 import { shouldPersistToDisk } from "./migrations";
 import {
@@ -415,14 +416,8 @@ function clearArtifactReference(relativePath: string, nextState: CallLogDetailSt
 }
 
 function listReferencedArtifacts() {
-  const db = getDbInstance();
-  const rows = db
-    .prepare("SELECT artifact_relpath FROM call_logs WHERE artifact_relpath IS NOT NULL")
-    .all() as Array<{ artifact_relpath: string | null }>;
-
-  return new Set(
-    rows.map((row) => row.artifact_relpath).filter((value): value is string => Boolean(value))
-  );
+  // #5618: paged to avoid an unbounded `.all()` OOM on large call_logs tables.
+  return collectReferencedArtifacts();
 }
 
 // #5217: SQLite caps a statement at SQLITE_MAX_VARIABLE_NUMBER bound params
@@ -510,13 +505,18 @@ export function cleanupOverflowCallLogFiles(baseDir = CALL_LOGS_DIR, maxEntries?
 }
 
 export function deleteCallLogsBefore(cutoff: string): DeleteResult {
-  const db = getDbInstance();
-  const ids = db
-    .prepare("SELECT id FROM call_logs WHERE timestamp < ? ORDER BY timestamp ASC")
-    .all(cutoff)
-    .map((row) => String((row as { id: string }).id));
-
-  return deleteCallLogRowsByIds(ids);
+  // #5618: page the id selection so a large backlog never loads in one `.all()`.
+  let deletedRows = 0;
+  let deletedArtifacts = 0;
+  for (;;) {
+    const ids = selectCallLogIdsBefore(cutoff);
+    if (ids.length === 0) break;
+    const result = deleteCallLogRowsByIds(ids);
+    deletedRows += result.deletedRows;
+    deletedArtifacts += result.deletedArtifacts;
+    if (result.deletedRows === 0) break;
+  }
+  return { deletedRows, deletedArtifacts };
 }
 
 export function trimCallLogsToMaxRows(maxRows = getCallLogsTableMaxRows()) {

src/lib/usage/callLogsBoundedQueries.ts

@@ -0,0 +1,42 @@
+import { getDbInstance } from "../db/core";
+
+// #5618 — node:sqlite's StatementSync.all() materializes the ENTIRE result set as
+// JS objects at once. On a large storage.sqlite (~170 MB+) an unbounded
+// `SELECT … FROM call_logs` over the whole table blows the V8 heap during the
+// startup cleanup pass (`cleanupExpiredLogs` → `rotateCallLogs`), crashing the
+// daemon before it binds. These helpers page through call_logs in bounded chunks
+// so peak memory stays flat regardless of table size.
+const CALL_LOG_QUERY_PAGE = 5000;
+
+/**
+ * Collect every non-null `artifact_relpath` referenced by call_logs, paging with
+ * LIMIT/OFFSET so a huge table never loads into memory in one `.all()`.
+ */
+export function collectReferencedArtifacts(): Set<string> {
+  const db = getDbInstance();
+  const referenced = new Set<string>();
+  const stmt = db.prepare(
+    "SELECT artifact_relpath FROM call_logs WHERE artifact_relpath IS NOT NULL LIMIT ? OFFSET ?"
+  );
+  for (let offset = 0; ; offset += CALL_LOG_QUERY_PAGE) {
+    const rows = stmt.all(CALL_LOG_QUERY_PAGE, offset) as Array<{ artifact_relpath: string | null }>;
+    for (const row of rows) {
+      if (typeof row.artifact_relpath === "string") referenced.add(row.artifact_relpath);
+    }
+    if (rows.length < CALL_LOG_QUERY_PAGE) break;
+  }
+  return referenced;
+}
+
+/**
+ * Select one bounded page of call_log ids older than `cutoff` (oldest first).
+ * Callers loop until it returns an empty page, deleting each batch, so the id
+ * list never grows to the full retention backlog at once.
+ */
+export function selectCallLogIdsBefore(cutoff: string, limit = CALL_LOG_QUERY_PAGE): string[] {
+  const db = getDbInstance();
+  const rows = db
+    .prepare("SELECT id FROM call_logs WHERE timestamp < ? ORDER BY timestamp ASC LIMIT ?")
+    .all(cutoff, limit) as Array<{ id: string }>;
+  return rows.map((row) => String(row.id));
+}