diff --git a/src/lib/db/cleanup.ts b/src/lib/db/cleanup.ts index 5855c140bb..c3cf716280 100644 --- a/src/lib/db/cleanup.ts +++ b/src/lib/db/cleanup.ts @@ -244,6 +244,66 @@ export async function cleanupMemoryEntries(): Promise { return result; } +/** + * Clean up old domain_cost_history based on retention settings. (#6848) + * Uses unix-epoch `timestamp` column (INTEGER). + */ +export async function cleanupDomainCostHistory(): Promise { + const db = getDbInstance(); + const retention = getRetentionSettings(); + + const retentionDays = retention.domainCostHistory; + const cutoffEpoch = Math.floor(Date.now() / 1000) - retentionDays * 86_400; + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM domain_cost_history WHERE timestamp < ?"); + const runResult = stmt.run(cutoffEpoch); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} domain_cost_history older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning domain_cost_history:", err); + result.errors++; + } + + return result; +} + +/** + * Clean up old compression_cache_stats based on retention settings. (#6848) + * Uses `created_at` column (DATETIME string). + */ +export async function cleanupCompressionCacheStats(): Promise { + const db = getDbInstance(); + const retention = getRetentionSettings(); + + const retentionDays = retention.compressionCacheStats; + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffISO = cutoffDate.toISOString(); + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM compression_cache_stats WHERE created_at < ?"); + const runResult = stmt.run(cutoffISO); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} compression_cache_stats older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning compression_cache_stats:", err); + result.errors++; + } + + return result; +} + /** * Clean up old xp_audit_log based on retention settings. */ @@ -274,6 +334,35 @@ export async function cleanupXpAuditLog(): Promise { return result; } +/** + * Clean up old compression_run_telemetry based on retention settings. (#6848) + * Uses unix-epoch `timestamp` column (INTEGER). + */ +export async function cleanupCompressionRunTelemetry(): Promise { + const db = getDbInstance(); + const retention = getRetentionSettings(); + + const retentionDays = retention.compressionRunTelemetry; + const cutoffEpoch = Math.floor(Date.now() / 1000) - retentionDays * 86_400; + + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare("DELETE FROM compression_run_telemetry WHERE timestamp < ?"); + const runResult = stmt.run(cutoffEpoch); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} compression_run_telemetry older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning compression_run_telemetry:", err); + result.errors++; + } + + return result; +} + /** * Run all cleanup functions if auto-cleanup is enabled. */ @@ -300,7 +389,10 @@ export async function runAutoCleanup(): Promise<{ mcpAudit: await cleanupMcpAudit(), a2aEvents: await cleanupA2aEvents(), memoryEntries: await cleanupMemoryEntries(), + domainCostHistory: await cleanupDomainCostHistory(), + compressionCacheStats: await cleanupCompressionCacheStats(), xpAuditLog: await cleanupXpAuditLog(), + compressionRunTelemetry: await cleanupCompressionRunTelemetry(), proxyLogs: await cleanupProxyLogs(), }; @@ -523,9 +615,7 @@ export async function cleanupProxyLogs(): Promise { const runResult = stmt.run(cutoffISO); result.deleted = runResult.changes; - console.log( - `[Cleanup] Deleted ${result.deleted} proxy_logs older than ${retentionDays} days` - ); + console.log(`[Cleanup] Deleted ${result.deleted} proxy_logs older than ${retentionDays} days`); } catch (err: unknown) { console.error("[Cleanup] Error cleaning proxy_logs:", err); result.errors++; @@ -557,9 +647,7 @@ export function startCleanupScheduler(): void { const proxyResult = await cleanupProxyLogs(); const totalDeleted = result.totalDeleted + proxyResult.deleted; if (totalDeleted > 0) { - console.log( - `[Cleanup] Startup cleanup freed ${totalDeleted} rows. Running VACUUM...` - ); + console.log(`[Cleanup] Startup cleanup freed ${totalDeleted} rows. Running VACUUM...`); try { const db = getDbInstance(); db.exec("VACUUM"); @@ -580,9 +668,7 @@ export function startCleanupScheduler(): void { const proxyResult = await cleanupProxyLogs(); const totalDeleted = result.totalDeleted + proxyResult.deleted; if (totalDeleted > 0) { - console.log( - `[Cleanup] Periodic cleanup freed ${totalDeleted} rows. Running VACUUM...` - ); + console.log(`[Cleanup] Periodic cleanup freed ${totalDeleted} rows. Running VACUUM...`); try { const db = getDbInstance(); db.exec("VACUUM"); diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index 8d037865e3..f29ac62856 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -49,7 +49,10 @@ const LEGACY_FLAT_KEYS: { callLogs: ["callLogs"], usageHistory: ["usageHistory"], memoryEntries: ["memoryEntries"], + domainCostHistory: ["domainCostHistory"], + compressionCacheStats: ["compressionCacheStats"], xpAuditLog: ["xpAuditLog"], + compressionRunTelemetry: ["compressionRunTelemetry"], autoCleanupEnabled: ["autoCleanupEnabled"], }, aggregation: { diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts index f69b214111..459f580334 100644 --- a/src/types/databaseSettings.ts +++ b/src/types/databaseSettings.ts @@ -46,7 +46,10 @@ export interface DatabaseSettings { callLogs: number; usageHistory: number; memoryEntries: number; + domainCostHistory: number; + compressionCacheStats: number; xpAuditLog: number; + compressionRunTelemetry: number; autoCleanupEnabled: boolean; }; @@ -107,7 +110,10 @@ export const DEFAULT_DATABASE_SETTINGS: Omit.test.ts`) + * would otherwise delete real rows from that database, not test rows. Do NOT + * remove the DATA_DIR override below. + */ + +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-6848-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +// Import the real functions — they use getDbInstance(), which resolves to the +// isolated temp DB created above (set before this import so the module's +// first getDbInstance() call already sees the isolated DATA_DIR). +const { + cleanupDomainCostHistory, + cleanupCompressionCacheStats, + cleanupXpAuditLog, + cleanupCompressionRunTelemetry, +} = await import("../../src/lib/db/cleanup.ts"); + +const { getDbInstance, resetDbInstance } = await import("../../src/lib/db/core.ts"); + +// Repo test rule: DB-touching tests must close the handle in test.after(), +// or the native test runner can hang indefinitely on a dangling connection. +test.after(() => { + resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +const DAY = 86_400; // seconds + +/** Ensure compression_run_telemetry table exists (created lazily in production). */ +function ensureTelemetryTable(): void { + const db = getDbInstance()!; + db.exec(` + CREATE TABLE IF NOT EXISTS compression_run_telemetry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp INTEGER NOT NULL, + request_id TEXT, + model TEXT, + provider TEXT, + source TEXT, + tokens_before INTEGER NOT NULL, + tokens_after INTEGER NOT NULL, + ratio REAL, + cost_delta REAL, + output_styles TEXT, + output_style_bypass TEXT, + output_tokens INTEGER + ) + `); +} + +// ─── Tests ─────────────────────────────────────────────────────────────── + +test("#6848 cleanupDomainCostHistory: deletes rows older than retention window", async () => { + const db = getDbInstance()!; + const now = Math.floor(Date.now() / 1000); + const insert = db.prepare( + "INSERT INTO domain_cost_history (api_key_id, cost, timestamp) VALUES (?, ?, ?)" + ); + + // 3 old (40 days ago), 2 recent (5 days ago) + insert.run("key1", 1.0, now - 40 * DAY); + insert.run("key1", 2.0, now - 40 * DAY); + insert.run("key1", 3.0, now - 40 * DAY); + insert.run("key1", 4.0, now - 5 * DAY); + insert.run("key1", 5.0, now - 5 * DAY); + + const result = await cleanupDomainCostHistory(); + + assert.strictEqual(result.deleted, 3); + assert.strictEqual(result.errors, 0); + + const remaining = db.prepare("SELECT COUNT(*) as cnt FROM domain_cost_history").get() as { + cnt: number; + }; + assert.strictEqual(remaining.cnt, 2); +}); + +test("#6848 cleanupCompressionCacheStats: deletes rows older than retention window", async () => { + const db = getDbInstance()!; + const oldDate = new Date(Date.now() - 40 * DAY * 1000).toISOString(); + const recentDate = new Date(Date.now() - 5 * DAY * 1000).toISOString(); + const insert = db.prepare( + "INSERT INTO compression_cache_stats (provider, compression_mode, created_at) VALUES (?, ?, ?)" + ); + + insert.run("openai", "auto", oldDate); + insert.run("openai", "auto", oldDate); + insert.run("anthropic", "auto", recentDate); + + const result = await cleanupCompressionCacheStats(); + + assert.strictEqual(result.deleted, 2); + assert.strictEqual(result.errors, 0); + + const remaining = db.prepare("SELECT COUNT(*) as cnt FROM compression_cache_stats").get() as { + cnt: number; + }; + assert.strictEqual(remaining.cnt, 1); +}); + +test("#6848 cleanupXpAuditLog: deletes rows older than retention window", async () => { + const db = getDbInstance()!; + const oldDate = new Date(Date.now() - 40 * DAY * 1000).toISOString(); + const recentDate = new Date(Date.now() - 5 * DAY * 1000).toISOString(); + const insert = db.prepare( + "INSERT INTO xp_audit_log (api_key_id, action, xp_earned, created_at) VALUES (?, ?, ?, ?)" + ); + + insert.run("key1", "login", 10, oldDate); + insert.run("key1", "login", 10, oldDate); + insert.run("key1", "login", 10, oldDate); + insert.run("key1", "login", 10, recentDate); + + const result = await cleanupXpAuditLog(); + + assert.strictEqual(result.deleted, 3); + assert.strictEqual(result.errors, 0); + + const remaining = db.prepare("SELECT COUNT(*) as cnt FROM xp_audit_log").get() as { cnt: number }; + assert.strictEqual(remaining.cnt, 1); +}); + +test("#6848 cleanupCompressionRunTelemetry: deletes rows older than retention window", async () => { + ensureTelemetryTable(); + const db = getDbInstance()!; + const now = Math.floor(Date.now() / 1000); + const insert = db.prepare( + "INSERT INTO compression_run_telemetry (timestamp, tokens_before, tokens_after) VALUES (?, ?, ?)" + ); + + insert.run(now - 40 * DAY, 1000, 500); + insert.run(now - 40 * DAY, 2000, 800); + insert.run(now - 5 * DAY, 1500, 600); + + const result = await cleanupCompressionRunTelemetry(); + + assert.strictEqual(result.deleted, 2); + assert.strictEqual(result.errors, 0); + + const remaining = db.prepare("SELECT COUNT(*) as cnt FROM compression_run_telemetry").get() as { + cnt: number; + }; + assert.strictEqual(remaining.cnt, 1); +}); + +test("#6848 no rows deleted when all data is within retention window (calls all 4 real functions)", async () => { + ensureTelemetryTable(); + const db = getDbInstance()!; + const now = Math.floor(Date.now() / 1000); + const recentISO = new Date().toISOString(); + + db.prepare("INSERT INTO domain_cost_history (api_key_id, cost, timestamp) VALUES (?, ?, ?)").run( + "k", + 1, + now - DAY + ); + db.prepare( + "INSERT INTO compression_cache_stats (provider, compression_mode, created_at) VALUES (?, ?, ?)" + ).run("p", "auto", recentISO); + db.prepare( + "INSERT INTO xp_audit_log (api_key_id, action, xp_earned, created_at) VALUES (?, ?, ?, ?)" + ).run("k", "a", 5, recentISO); + db.prepare( + "INSERT INTO compression_run_telemetry (timestamp, tokens_before, tokens_after) VALUES (?, ?, ?)" + ).run(now - DAY, 100, 50); + + const r1 = await cleanupDomainCostHistory(); + const r2 = await cleanupCompressionCacheStats(); + const r3 = await cleanupXpAuditLog(); + const r4 = await cleanupCompressionRunTelemetry(); + + assert.strictEqual(r1.deleted, 0); + assert.strictEqual(r2.deleted, 0); + assert.strictEqual(r3.deleted, 0); + assert.strictEqual(r4.deleted, 0); +}); + +test("#6848 DEFAULT_DATABASE_SETTINGS has new retention keys", async () => { + const mod = await import("../../src/types/databaseSettings.ts"); + const defaults = mod.DEFAULT_DATABASE_SETTINGS.retention; + + assert.ok(typeof defaults.domainCostHistory === "number"); + assert.ok(typeof defaults.compressionCacheStats === "number"); + assert.ok(typeof defaults.xpAuditLog === "number"); + assert.ok(typeof defaults.compressionRunTelemetry === "number"); + + assert.strictEqual(defaults.domainCostHistory, 30); + assert.strictEqual(defaults.compressionCacheStats, 30); + assert.strictEqual(defaults.xpAuditLog, 30); + assert.strictEqual(defaults.compressionRunTelemetry, 30); +});