fix(db): schedule retention cleanup + fix cleanup table/column names (extracted from #4428) (#4691)

Integrated into release/v3.8.34 (cleanup core extracted from #4428, credit @oyi77)

CHANGELOG.md

@@ -16,6 +16,10 @@ _In development — bullets added per PR; finalized at release._
 
 - **chore(quality): release-green pre-flight validator + nightly signal** — new `npm run check:release-green` (`scripts/quality/validate-release-green.mjs`) reproduces the release-equivalent validation (full unit + vitest + ratchets + typecheck + lint, optional `--with-build` package-artifact) against the current working tree and classifies each red as **HARD** (real defect) vs **DRIFT** (ratchet, rebaselined at release) — purely diagnostic, never blocking contributors. A new `nightly-release-green` workflow runs it on the active release branch and opens/updates a tracking issue on hard failures. Closes the gap where the full gate (`ci.yml`) only ran on the release PR, so reds accrued silently on `release/**` and surfaced in layers at release time. (thanks @diegosouzapw)
 
+### 🐛 Fixed
+
+- **fix(db): scheduled cleanup actually runs + queries target the real tables (DB-bloat / OOM)** — `runAutoCleanup` was never scheduled, so retention cleanup never executed and tables (`compression_analytics`, `usage_history`, …) grew unbounded into multi-GB SQLite files driving high RSS. Worse, several cleanup queries referenced wrong table/column names (`call_logs.created_at`→`timestamp`, `compression_analytics.created_at`→`timestamp`, `mcp_audit_log`→`mcp_tool_audit`, `a2a_events`→`a2a_task_events`, `memory_entries`→`memories`), so even a manual run silently no-op'd or errored. Fixed the five queries to match the real schema, added `cleanupProxyLogs`, and wired a `startCleanupScheduler` (startup + every 6h, VACUUM after deletes) into `server-init` alongside the existing budget-reset and reasoning-cache jobs. ([#4428](https://github.com/diegosouzapw/OmniRoute/pull/4428) — thanks @oyi77)
+
 ---
 
 ## [3.8.33] — 2026-06-21

config/quality/file-size-baseline.json

@@ -133,7 +133,8 @@
     "open-sse/services/batchProcessor.ts": 828,
     "open-sse/services/browserBackedChat.ts": 850,
     "open-sse/services/claudeCodeCompatible.ts": 1202,
-    "open-sse/services/combo.ts": 2991,
+    "_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.",
+    "open-sse/services/combo.ts": 3036,
     "open-sse/services/rateLimitManager.ts": 1035,
     "open-sse/services/tokenRefresh.ts": 1997,
     "open-sse/services/usage.ts": 3450,
@@ -241,7 +242,8 @@
     "tests/unit/perplexity-web.test.ts": 959,
     "tests/unit/provider-models-route.test.ts": 1618,
     "tests/unit/provider-validation-specialty.test.ts": 2752,
-    "tests/unit/providers-page-utils.test.ts": 1004,
+    "_rebaseline_pr4613_compatible_provider_groups": "Reconcile #4613 already-merged growth: providers-page-utils.test.ts 1004->1052 (+48, buildCompatibleProviderGroups partition unit test). Fast-gate PR->release does not run check:file-size, so this surfaced post-merge.",
+    "tests/unit/providers-page-utils.test.ts": 1052,
     "tests/unit/reasoning-cache.test.ts": 980,
     "tests/unit/route-edge-coverage.test.ts": 1234,
     "tests/unit/search-handler-extended.test.ts": 1124,

src/lib/db/cleanup.ts

@@ -64,7 +64,7 @@ export async function cleanupCallLogs(): Promise<CleanupResult> {
   const result: CleanupResult = { deleted: 0, errors: 0 };
 
   try {
-    const stmt = db.prepare("DELETE FROM call_logs WHERE created_at < ?");
+    const stmt = db.prepare("DELETE FROM call_logs WHERE timestamp < ?");
     const runResult = stmt.run(cutoffISO);
     result.deleted = runResult.changes;
 
@@ -141,7 +141,7 @@ export async function cleanupCompressionAnalytics(): Promise<CleanupResult> {
   const result: CleanupResult = { deleted: 0, errors: 0 };
 
   try {
-    const stmt = db.prepare("DELETE FROM compression_analytics WHERE created_at < ?");
+    const stmt = db.prepare("DELETE FROM compression_analytics WHERE timestamp < ?");
     const runResult = stmt.run(cutoffISO);
     result.deleted = runResult.changes;
 
@@ -171,7 +171,7 @@ export async function cleanupMcpAudit(): Promise<CleanupResult> {
   const result: CleanupResult = { deleted: 0, errors: 0 };
 
   try {
-    const stmt = db.prepare("DELETE FROM mcp_audit_log WHERE timestamp < ?");
+    const stmt = db.prepare("DELETE FROM mcp_tool_audit WHERE timestamp < ?");
     const runResult = stmt.run(cutoffISO);
     result.deleted = runResult.changes;
 
@@ -201,7 +201,7 @@ export async function cleanupA2aEvents(): Promise<CleanupResult> {
   const result: CleanupResult = { deleted: 0, errors: 0 };
 
   try {
-    const stmt = db.prepare("DELETE FROM a2a_events WHERE timestamp < ?");
+    const stmt = db.prepare("DELETE FROM a2a_task_events WHERE timestamp < ?");
     const runResult = stmt.run(cutoffISO);
     result.deleted = runResult.changes;
 
@@ -229,7 +229,7 @@ export async function cleanupMemoryEntries(): Promise<CleanupResult> {
   const result: CleanupResult = { deleted: 0, errors: 0 };
 
   try {
-    const stmt = db.prepare("DELETE FROM memory_entries WHERE created_at < ?");
+    const stmt = db.prepare("DELETE FROM memories WHERE created_at < ?");
     const runResult = stmt.run(cutoffISO);
     result.deleted = runResult.changes;
 
@@ -270,6 +270,7 @@ export async function runAutoCleanup(): Promise<{
     mcpAudit: await cleanupMcpAudit(),
     a2aEvents: await cleanupA2aEvents(),
     memoryEntries: await cleanupMemoryEntries(),
+    proxyLogs: await cleanupProxyLogs(),
   };
 
   const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0);
@@ -349,3 +350,114 @@ export async function purgeDetailedLogs(): Promise<CleanupResult> {
 
   return result;
 }
+
+/**
+ * Clean up old proxy_logs based on retention settings.
+ * Uses the same retention period as call_logs (30 days default).
+ */
+export async function cleanupProxyLogs(): Promise<CleanupResult> {
+  const db = getDbInstance();
+  const retention = getRetentionSettings();
+
+  const retentionDays = retention.callLogs;
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
+  const cutoffISO = cutoffDate.toISOString();
+
+  const result: CleanupResult = { deleted: 0, errors: 0 };
+
+  try {
+    const stmt = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?");
+    const runResult = stmt.run(cutoffISO);
+    result.deleted = runResult.changes;
+
+    console.log(
+      `[Cleanup] Deleted ${result.deleted} proxy_logs older than ${retentionDays} days`
+    );
+  } catch (err: unknown) {
+    console.error("[Cleanup] Error cleaning proxy_logs:", err);
+    result.errors++;
+  }
+
+  return result;
+}
+
+// ──────────────── Background Cleanup Scheduler ────────────────
+
+const CLEANUP_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
+let _cleanupSchedulerTimer: ReturnType<typeof setInterval> | null = null;
+
+/**
+ * Start the background cleanup scheduler. Runs cleanup on startup
+ * and then every 6 hours. Runs VACUUM after deletes to reclaim disk space.
+ *
+ * Without this, tables grow unboundedly (compression_analytics 600K+ rows,
+ * usage_history 250K+ rows) causing 1.4GB+ SQLite files and 3-8GB RSS
+ * from better-sqlite3 memory mapping.
+ */
+export function startCleanupScheduler(): void {
+  if (_cleanupSchedulerTimer) return;
+
+  // Run cleanup 30s after startup (let the server initialize first).
+  setTimeout(async () => {
+    try {
+      const result = await runAutoCleanup();
+      const proxyResult = await cleanupProxyLogs();
+      const totalDeleted = result.totalDeleted + proxyResult.deleted;
+      if (totalDeleted > 0) {
+        console.log(
+          `[Cleanup] Startup cleanup freed ${totalDeleted} rows. Running VACUUM...`
+        );
+        try {
+          const db = getDbInstance();
+          db.exec("VACUUM");
+          console.log("[Cleanup] VACUUM completed after startup cleanup.");
+        } catch (vacErr) {
+          console.error("[Cleanup] VACUUM after cleanup failed:", vacErr);
+        }
+      }
+    } catch (err) {
+      console.error("[Cleanup] Startup cleanup failed:", err);
+    }
+  }, 30_000);
+
+  // Schedule periodic cleanup every 6 hours.
+  _cleanupSchedulerTimer = setInterval(async () => {
+    try {
+      const result = await runAutoCleanup();
+      const proxyResult = await cleanupProxyLogs();
+      const totalDeleted = result.totalDeleted + proxyResult.deleted;
+      if (totalDeleted > 0) {
+        console.log(
+          `[Cleanup] Periodic cleanup freed ${totalDeleted} rows. Running VACUUM...`
+        );
+        try {
+          const db = getDbInstance();
+          db.exec("VACUUM");
+          console.log("[Cleanup] VACUUM completed after periodic cleanup.");
+        } catch (vacErr) {
+          console.error("[Cleanup] VACUUM after cleanup failed:", vacErr);
+        }
+      }
+    } catch (err) {
+      console.error("[Cleanup] Periodic cleanup failed:", err);
+    }
+  }, CLEANUP_INTERVAL_MS);
+
+  // Don't keep the process alive solely for cleanup.
+  if (_cleanupSchedulerTimer && typeof _cleanupSchedulerTimer.unref === "function") {
+    _cleanupSchedulerTimer.unref();
+  }
+
+  console.log("[Cleanup] Background cleanup scheduler started (every 6 hours).");
+}
+
+/**
+ * Stop the background cleanup scheduler (for tests).
+ */
+export function stopCleanupScheduler(): void {
+  if (_cleanupSchedulerTimer) {
+    clearInterval(_cleanupSchedulerTimer);
+    _cleanupSchedulerTimer = null;
+  }
+}

src/server-init.ts

@@ -6,6 +6,7 @@ import { initAuditLog, cleanupExpiredLogs, logAuditEvent } from "./lib/complianc
 import { initConsoleInterceptor } from "./lib/consoleInterceptor";
 import { startBudgetResetJob } from "./lib/jobs/budgetResetJob";
 import { startReasoningCacheCleanupJob } from "./lib/jobs/reasoningCacheCleanupJob";
+import { startCleanupScheduler } from "./lib/db/cleanup";
 import { getSettings } from "./lib/db/settings";
 import { applyRuntimeSettings } from "./lib/config/runtimeSettings";
 import { setSystemPromptConfig } from "@omniroute/open-sse/services/systemPrompt.ts";
@@ -105,6 +106,7 @@ async function startServer() {
     await initializeCloudSync();
     startBudgetResetJob();
     startReasoningCacheCleanupJob();
+    startCleanupScheduler();
     startRuntimeConfigHotReload();
     startupLog.info("Server started with cloud sync initialized");
 

tests/unit/cleanup-column-fix.test.mjs

@@ -0,0 +1,114 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+
+// Source-level invariant tests for cleanup.ts fixes.
+// These verify the critical column name fixes that were causing silent cleanup failures.
+
+const CLEANUP_PATH = path.resolve(import.meta.dirname, "../../src/lib/db/cleanup.ts");
+const source = fs.readFileSync(CLEANUP_PATH, "utf-8");
+
+test("cleanup: compression_analytics uses 'timestamp' column (not 'created_at')", () => {
+  // The bug: cleanup used WHERE created_at < ? but the table has 'timestamp' column.
+  // This caused silent failures — 600K+ rows accumulated over 52 days.
+  assert.ok(
+    source.includes("DELETE FROM compression_analytics WHERE timestamp < ?"),
+    "compression_analytics cleanup must use 'timestamp' column, not 'created_at'"
+  );
+  assert.ok(
+    !source.includes("DELETE FROM compression_analytics WHERE created_at"),
+    "must NOT use created_at for compression_analytics (column doesn't exist)"
+  );
+});
+
+test("cleanup: call_logs uses 'timestamp' column (not 'created_at')", () => {
+  // Same bug as compression_analytics.
+  assert.ok(
+    source.includes("DELETE FROM call_logs WHERE timestamp < ?"),
+    "call_logs cleanup must use 'timestamp' column"
+  );
+  assert.ok(
+    !source.includes("DELETE FROM call_logs WHERE created_at"),
+    "must NOT use created_at for call_logs (column doesn't exist)"
+  );
+});
+
+test("cleanup: has proxy_logs cleanup function", () => {
+  assert.ok(
+    source.includes("cleanupProxyLogs"),
+    "must have cleanupProxyLogs function for the proxy_logs table"
+  );
+  assert.ok(
+    source.includes("DELETE FROM proxy_logs WHERE timestamp < ?"),
+    "proxy_logs cleanup must use timestamp column"
+  );
+});
+
+test("cleanup: proxy_logs is included in runAutoCleanup", () => {
+  assert.ok(
+    source.includes("proxyLogs: await cleanupProxyLogs()"),
+    "runAutoCleanup must include proxyLogs cleanup"
+  );
+});
+
+test("cleanup: has background scheduler (startCleanupScheduler)", () => {
+  assert.ok(
+    source.includes("startCleanupScheduler"),
+    "must export startCleanupScheduler for periodic background cleanup"
+  );
+  assert.ok(
+    source.includes("CLEANUP_INTERVAL_MS"),
+    "must have a cleanup interval constant"
+  );
+  assert.ok(
+    source.includes("VACUUM"),
+    "scheduler must run VACUUM after deletes to reclaim disk space"
+  );
+});
+
+test("cleanup: scheduler is wired into server-init.ts", () => {
+  const serverInitPath = path.resolve(import.meta.dirname, "../../src/server-init.ts");
+  const serverInit = fs.readFileSync(serverInitPath, "utf-8");
+  assert.ok(
+    serverInit.includes('import { startCleanupScheduler } from "./lib/db/cleanup"'),
+    "server-init.ts must import startCleanupScheduler"
+  );
+  assert.ok(
+    serverInit.includes("startCleanupScheduler()"),
+    "server-init.ts must call startCleanupScheduler() at startup"
+  );
+});
+
+test("cleanup: mcp_tool_audit uses correct table name (not 'mcp_audit_log')", () => {
+  assert.ok(
+    source.includes("DELETE FROM mcp_tool_audit WHERE"),
+    "must use correct table name mcp_tool_audit"
+  );
+  assert.ok(
+    !source.includes("DELETE FROM mcp_audit_log WHERE"),
+    "must NOT use non-existent table name mcp_audit_log"
+  );
+});
+
+test("cleanup: a2a_task_events uses correct table name (not 'a2a_events')", () => {
+  assert.ok(
+    source.includes("DELETE FROM a2a_task_events WHERE"),
+    "must use correct table name a2a_task_events"
+  );
+  assert.ok(
+    !source.includes("DELETE FROM a2a_events WHERE"),
+    "must NOT use non-existent table name a2a_events"
+  );
+});
+
+test("cleanup: memories uses correct table name (not 'memory_entries')", () => {
+  assert.ok(
+    source.includes("DELETE FROM memories WHERE"),
+    "must use correct table name memories"
+  );
+  assert.ok(
+    !source.includes("DELETE FROM memory_entries WHERE"),
+    "must NOT use non-existent table name memory_entries"
+  );
+});