diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f435c0544..a48187b91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ _In development — bullets added per PR; finalized at release._ - **chore(quality): release-green pre-flight validator + nightly signal** — new `npm run check:release-green` (`scripts/quality/validate-release-green.mjs`) reproduces the release-equivalent validation (full unit + vitest + ratchets + typecheck + lint, optional `--with-build` package-artifact) against the current working tree and classifies each red as **HARD** (real defect) vs **DRIFT** (ratchet, rebaselined at release) — purely diagnostic, never blocking contributors. A new `nightly-release-green` workflow runs it on the active release branch and opens/updates a tracking issue on hard failures. Closes the gap where the full gate (`ci.yml`) only ran on the release PR, so reds accrued silently on `release/**` and surfaced in layers at release time. (thanks @diegosouzapw) +### 🐛 Fixed + +- **fix(db): scheduled cleanup actually runs + queries target the real tables (DB-bloat / OOM)** — `runAutoCleanup` was never scheduled, so retention cleanup never executed and tables (`compression_analytics`, `usage_history`, …) grew unbounded into multi-GB SQLite files driving high RSS. Worse, several cleanup queries referenced wrong table/column names (`call_logs.created_at`→`timestamp`, `compression_analytics.created_at`→`timestamp`, `mcp_audit_log`→`mcp_tool_audit`, `a2a_events`→`a2a_task_events`, `memory_entries`→`memories`), so even a manual run silently no-op'd or errored. Fixed the five queries to match the real schema, added `cleanupProxyLogs`, and wired a `startCleanupScheduler` (startup + every 6h, VACUUM after deletes) into `server-init` alongside the existing budget-reset and reasoning-cache jobs. ([#4428](https://github.com/diegosouzapw/OmniRoute/pull/4428) — thanks @oyi77) + --- ## [3.8.33] — 2026-06-21 diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index c85fde8fe4..c89a16173e 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -133,7 +133,8 @@ "open-sse/services/batchProcessor.ts": 828, "open-sse/services/browserBackedChat.ts": 850, "open-sse/services/claudeCodeCompatible.ts": 1202, - "open-sse/services/combo.ts": 2991, + "_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.", + "open-sse/services/combo.ts": 3036, "open-sse/services/rateLimitManager.ts": 1035, "open-sse/services/tokenRefresh.ts": 1997, "open-sse/services/usage.ts": 3450, @@ -241,7 +242,8 @@ "tests/unit/perplexity-web.test.ts": 959, "tests/unit/provider-models-route.test.ts": 1618, "tests/unit/provider-validation-specialty.test.ts": 2752, - "tests/unit/providers-page-utils.test.ts": 1004, + "_rebaseline_pr4613_compatible_provider_groups": "Reconcile #4613 already-merged growth: providers-page-utils.test.ts 1004->1052 (+48, buildCompatibleProviderGroups partition unit test). Fast-gate PR->release does not run check:file-size, so this surfaced post-merge.", + "tests/unit/providers-page-utils.test.ts": 1052, "tests/unit/reasoning-cache.test.ts": 980, "tests/unit/route-edge-coverage.test.ts": 1234, "tests/unit/search-handler-extended.test.ts": 1124, diff --git a/src/lib/db/cleanup.ts b/src/lib/db/cleanup.ts index 1772240fdf..83563f11a2 100644 --- a/src/lib/db/cleanup.ts +++ b/src/lib/db/cleanup.ts @@ -64,7 +64,7 @@ export async function cleanupCallLogs(): Promise { const result: CleanupResult = { deleted: 0, errors: 0 }; try { - const stmt = db.prepare("DELETE FROM call_logs WHERE created_at < ?"); + const stmt = db.prepare("DELETE FROM call_logs WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; @@ -141,7 +141,7 @@ export async function cleanupCompressionAnalytics(): Promise { const result: CleanupResult = { deleted: 0, errors: 0 }; try { - const stmt = db.prepare("DELETE FROM compression_analytics WHERE created_at < ?"); + const stmt = db.prepare("DELETE FROM compression_analytics WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; @@ -171,7 +171,7 @@ export async function cleanupMcpAudit(): Promise { const result: CleanupResult = { deleted: 0, errors: 0 }; try { - const stmt = db.prepare("DELETE FROM mcp_audit_log WHERE timestamp < ?"); + const stmt = db.prepare("DELETE FROM mcp_tool_audit WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; @@ -201,7 +201,7 @@ export async function cleanupA2aEvents(): Promise { const result: CleanupResult = { deleted: 0, errors: 0 }; try { - const stmt = db.prepare("DELETE FROM a2a_events WHERE timestamp < ?"); + const stmt = db.prepare("DELETE FROM a2a_task_events WHERE timestamp < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; @@ -229,7 +229,7 @@ export async function cleanupMemoryEntries(): Promise { const result: CleanupResult = { deleted: 0, errors: 0 }; try { - const stmt = db.prepare("DELETE FROM memory_entries WHERE created_at < ?"); + const stmt = db.prepare("DELETE FROM memories WHERE created_at < ?"); const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; @@ -270,6 +270,7 @@ export async function runAutoCleanup(): Promise<{ mcpAudit: await cleanupMcpAudit(), a2aEvents: await cleanupA2aEvents(), memoryEntries: await cleanupMemoryEntries(), + proxyLogs: await cleanupProxyLogs(), }; const totalDeleted = Object.values(results).reduce((sum, r) => sum + r.deleted, 0); @@ -349,3 +350,114 @@ export async function purgeDetailedLogs(): Promise { return result; } + +/** + * Clean up old proxy_logs based on retention settings. + * Uses the same retention period as call_logs (30 days default). + */ +export async function cleanupProxyLogs(): Promise { + const db = getDbInstance(); + const retention = getRetentionSettings(); + + const retentionDays = retention.callLogs; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM proxy_logs WHERE timestamp < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} proxy_logs older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning proxy_logs:", err); + result.errors++; + } + + return result; +} + +// ──────────────── Background Cleanup Scheduler ──────────────── + +const CLEANUP_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours +let _cleanupSchedulerTimer: ReturnType | null = null; + +/** + * Start the background cleanup scheduler. Runs cleanup on startup + * and then every 6 hours. Runs VACUUM after deletes to reclaim disk space. + * + * Without this, tables grow unboundedly (compression_analytics 600K+ rows, + * usage_history 250K+ rows) causing 1.4GB+ SQLite files and 3-8GB RSS + * from better-sqlite3 memory mapping. + */ +export function startCleanupScheduler(): void { + if (_cleanupSchedulerTimer) return; + + // Run cleanup 30s after startup (let the server initialize first). + setTimeout(async () => { + try { + const result = await runAutoCleanup(); + const proxyResult = await cleanupProxyLogs(); + const totalDeleted = result.totalDeleted + proxyResult.deleted; + if (totalDeleted > 0) { + console.log( + `[Cleanup] Startup cleanup freed ${totalDeleted} rows. Running VACUUM...` + ); + try { + const db = getDbInstance(); + db.exec("VACUUM"); + console.log("[Cleanup] VACUUM completed after startup cleanup."); + } catch (vacErr) { + console.error("[Cleanup] VACUUM after cleanup failed:", vacErr); + } + } + } catch (err) { + console.error("[Cleanup] Startup cleanup failed:", err); + } + }, 30_000); + + // Schedule periodic cleanup every 6 hours. + _cleanupSchedulerTimer = setInterval(async () => { + try { + const result = await runAutoCleanup(); + const proxyResult = await cleanupProxyLogs(); + const totalDeleted = result.totalDeleted + proxyResult.deleted; + if (totalDeleted > 0) { + console.log( + `[Cleanup] Periodic cleanup freed ${totalDeleted} rows. Running VACUUM...` + ); + try { + const db = getDbInstance(); + db.exec("VACUUM"); + console.log("[Cleanup] VACUUM completed after periodic cleanup."); + } catch (vacErr) { + console.error("[Cleanup] VACUUM after cleanup failed:", vacErr); + } + } + } catch (err) { + console.error("[Cleanup] Periodic cleanup failed:", err); + } + }, CLEANUP_INTERVAL_MS); + + // Don't keep the process alive solely for cleanup. + if (_cleanupSchedulerTimer && typeof _cleanupSchedulerTimer.unref === "function") { + _cleanupSchedulerTimer.unref(); + } + + console.log("[Cleanup] Background cleanup scheduler started (every 6 hours)."); +} + +/** + * Stop the background cleanup scheduler (for tests). + */ +export function stopCleanupScheduler(): void { + if (_cleanupSchedulerTimer) { + clearInterval(_cleanupSchedulerTimer); + _cleanupSchedulerTimer = null; + } +} diff --git a/src/server-init.ts b/src/server-init.ts index 0fd6070bc6..b182d7b783 100644 --- a/src/server-init.ts +++ b/src/server-init.ts @@ -6,6 +6,7 @@ import { initAuditLog, cleanupExpiredLogs, logAuditEvent } from "./lib/complianc import { initConsoleInterceptor } from "./lib/consoleInterceptor"; import { startBudgetResetJob } from "./lib/jobs/budgetResetJob"; import { startReasoningCacheCleanupJob } from "./lib/jobs/reasoningCacheCleanupJob"; +import { startCleanupScheduler } from "./lib/db/cleanup"; import { getSettings } from "./lib/db/settings"; import { applyRuntimeSettings } from "./lib/config/runtimeSettings"; import { setSystemPromptConfig } from "@omniroute/open-sse/services/systemPrompt.ts"; @@ -105,6 +106,7 @@ async function startServer() { await initializeCloudSync(); startBudgetResetJob(); startReasoningCacheCleanupJob(); + startCleanupScheduler(); startRuntimeConfigHotReload(); startupLog.info("Server started with cloud sync initialized"); diff --git a/tests/unit/cleanup-column-fix.test.mjs b/tests/unit/cleanup-column-fix.test.mjs new file mode 100644 index 0000000000..eeb3aeafd9 --- /dev/null +++ b/tests/unit/cleanup-column-fix.test.mjs @@ -0,0 +1,114 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; + +// Source-level invariant tests for cleanup.ts fixes. +// These verify the critical column name fixes that were causing silent cleanup failures. + +const CLEANUP_PATH = path.resolve(import.meta.dirname, "../../src/lib/db/cleanup.ts"); +const source = fs.readFileSync(CLEANUP_PATH, "utf-8"); + +test("cleanup: compression_analytics uses 'timestamp' column (not 'created_at')", () => { + // The bug: cleanup used WHERE created_at < ? but the table has 'timestamp' column. + // This caused silent failures — 600K+ rows accumulated over 52 days. + assert.ok( + source.includes("DELETE FROM compression_analytics WHERE timestamp < ?"), + "compression_analytics cleanup must use 'timestamp' column, not 'created_at'" + ); + assert.ok( + !source.includes("DELETE FROM compression_analytics WHERE created_at"), + "must NOT use created_at for compression_analytics (column doesn't exist)" + ); +}); + +test("cleanup: call_logs uses 'timestamp' column (not 'created_at')", () => { + // Same bug as compression_analytics. + assert.ok( + source.includes("DELETE FROM call_logs WHERE timestamp < ?"), + "call_logs cleanup must use 'timestamp' column" + ); + assert.ok( + !source.includes("DELETE FROM call_logs WHERE created_at"), + "must NOT use created_at for call_logs (column doesn't exist)" + ); +}); + +test("cleanup: has proxy_logs cleanup function", () => { + assert.ok( + source.includes("cleanupProxyLogs"), + "must have cleanupProxyLogs function for the proxy_logs table" + ); + assert.ok( + source.includes("DELETE FROM proxy_logs WHERE timestamp < ?"), + "proxy_logs cleanup must use timestamp column" + ); +}); + +test("cleanup: proxy_logs is included in runAutoCleanup", () => { + assert.ok( + source.includes("proxyLogs: await cleanupProxyLogs()"), + "runAutoCleanup must include proxyLogs cleanup" + ); +}); + +test("cleanup: has background scheduler (startCleanupScheduler)", () => { + assert.ok( + source.includes("startCleanupScheduler"), + "must export startCleanupScheduler for periodic background cleanup" + ); + assert.ok( + source.includes("CLEANUP_INTERVAL_MS"), + "must have a cleanup interval constant" + ); + assert.ok( + source.includes("VACUUM"), + "scheduler must run VACUUM after deletes to reclaim disk space" + ); +}); + +test("cleanup: scheduler is wired into server-init.ts", () => { + const serverInitPath = path.resolve(import.meta.dirname, "../../src/server-init.ts"); + const serverInit = fs.readFileSync(serverInitPath, "utf-8"); + assert.ok( + serverInit.includes('import { startCleanupScheduler } from "./lib/db/cleanup"'), + "server-init.ts must import startCleanupScheduler" + ); + assert.ok( + serverInit.includes("startCleanupScheduler()"), + "server-init.ts must call startCleanupScheduler() at startup" + ); +}); + +test("cleanup: mcp_tool_audit uses correct table name (not 'mcp_audit_log')", () => { + assert.ok( + source.includes("DELETE FROM mcp_tool_audit WHERE"), + "must use correct table name mcp_tool_audit" + ); + assert.ok( + !source.includes("DELETE FROM mcp_audit_log WHERE"), + "must NOT use non-existent table name mcp_audit_log" + ); +}); + +test("cleanup: a2a_task_events uses correct table name (not 'a2a_events')", () => { + assert.ok( + source.includes("DELETE FROM a2a_task_events WHERE"), + "must use correct table name a2a_task_events" + ); + assert.ok( + !source.includes("DELETE FROM a2a_events WHERE"), + "must NOT use non-existent table name a2a_events" + ); +}); + +test("cleanup: memories uses correct table name (not 'memory_entries')", () => { + assert.ok( + source.includes("DELETE FROM memories WHERE"), + "must use correct table name memories" + ); + assert.ok( + !source.includes("DELETE FROM memory_entries WHERE"), + "must NOT use non-existent table name memory_entries" + ); +});