From 84c9dfdd2c9b7ef933ac3ddf1ec2f8a712d6ee72 Mon Sep 17 00:00:00 2001 From: Dizzle <112548150+maxmad64bis@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:40:09 +0200 Subject: [PATCH] fix(config-audit): persist config audit log to SQLite with retention (#11103) Validated on the combined batch board over release/v3.8.50 tip d91238b7: static gates clean, typecheck:core clean, focused tests green. Config-audit survives restarts with bounded growth (migration 161 + OR IGNORE seed, retention wired into runAutoCleanup); route cabling deliberately out of scope. Thank you @maxmad64bis! --- .../fixes/11103-persist-config-audit-log.md | 1 + src/domain/configAudit.ts | 125 +++++++++++++----- src/lib/db/cleanup.ts | 26 ++++ src/lib/db/databaseSettings.ts | 1 + .../db/migrations/046_database_settings.sql | 1 + .../db/migrations/161_config_audit_log.sql | 15 +++ src/types/databaseSettings.ts | 2 + tests/unit/config-audit-persistence.test.ts | 124 +++++++++++++++++ 8 files changed, 265 insertions(+), 30 deletions(-) create mode 100644 changelog.d/fixes/11103-persist-config-audit-log.md create mode 100644 src/lib/db/migrations/161_config_audit_log.sql create mode 100644 tests/unit/config-audit-persistence.test.ts diff --git a/changelog.d/fixes/11103-persist-config-audit-log.md b/changelog.d/fixes/11103-persist-config-audit-log.md new file mode 100644 index 0000000000..aeb53b1781 --- /dev/null +++ b/changelog.d/fixes/11103-persist-config-audit-log.md @@ -0,0 +1 @@ +- **Config audit persistence:** persist the configuration audit trail to SQLite (`config_audit_log`) instead of an in-memory buffer capped at 1000 volatile entries, and bound its growth with `cleanupConfigAudit()` driven by the `retention.configAudit` setting (default 30 days), wired into `runAutoCleanup` ([#11103](https://github.com/diegosouzapw/OmniRoute/pull/11103)). diff --git a/src/domain/configAudit.ts b/src/domain/configAudit.ts index c2701b2a21..873d3b046a 100644 --- a/src/domain/configAudit.ts +++ b/src/domain/configAudit.ts @@ -13,6 +13,8 @@ * - Optional human notes */ +import { getDbInstance } from "../lib/db/core"; + /** Types of configuration entities that can be audited */ export type AuditTarget = "provider" | "combo" | "policy" | "connection" | "settings"; @@ -72,10 +74,8 @@ export interface ConfigSnapshot { data: Record; } -// ── In-memory store ────────────────────────────────────────────────────────── -// In production, persist to SQLite alongside other domain state. +// ── SQLite-backed store ─────────────────────────────────────────────────────── -let auditLog: ConfigAuditEntry[] = []; let idCounter = 0; function generateId(): string { @@ -85,6 +85,40 @@ function generateId(): string { return `audit-${ts}-${seq}`; } +function db() { + return getDbInstance(); +} + +interface ConfigAuditRow { + id: string; + timestamp: string; + action: string; + target: string; + target_id: string; + target_name: string; + before_json: string | null; + after_json: string | null; + diff_json: string; + source: string; + note: string | null; +} + +function rowToEntry(row: ConfigAuditRow): ConfigAuditEntry { + return { + id: row.id, + timestamp: row.timestamp, + action: row.action as AuditAction, + target: row.target as AuditTarget, + targetId: row.target_id, + targetName: row.target_name, + before: row.before_json === null ? null : (JSON.parse(row.before_json) as Record | null), + after: row.after_json === null ? null : (JSON.parse(row.after_json) as Record | null), + source: row.source as AuditSource, + diff: JSON.parse(row.diff_json) as ConfigDiff, + note: row.note, + }; +} + /** * Compute a structured diff between two configuration states. */ @@ -159,12 +193,24 @@ export function recordChange( note: note ?? null, }; - auditLog.push(entry); - - // Keep log bounded (max 1000 entries in memory) - if (auditLog.length > 1000) { - auditLog = auditLog.slice(-1000); - } + db().prepare( + `INSERT INTO config_audit_log + (id, timestamp, action, target, target_id, target_name, before_json, after_json, diff_json, source, note) + VALUES + (@id, @timestamp, @action, @target, @targetId, @targetName, @beforeJson, @afterJson, @diffJson, @source, @note)` + ).run({ + id: entry.id, + timestamp: entry.timestamp, + action: entry.action, + target: entry.target, + targetId: entry.targetId, + targetName: entry.targetName, + beforeJson: before === null ? null : JSON.stringify(before), + afterJson: after === null ? null : JSON.stringify(after), + diffJson: JSON.stringify(entry.diff), + source: entry.source, + note: entry.note, + }); return entry; } @@ -181,42 +227,57 @@ export function getAuditLog(options?: { limit?: number; offset?: number; }): { entries: ConfigAuditEntry[]; total: number } { - let filtered = auditLog; + const where: string[] = []; + const params: Record = {}; if (options?.target) { - filtered = filtered.filter((e) => e.target === options.target); + where.push("target = @target"); + params.target = options.target; } if (options?.targetId) { - filtered = filtered.filter((e) => e.targetId === options.targetId); + where.push("target_id = @targetId"); + params.targetId = options.targetId; } if (options?.action) { - filtered = filtered.filter((e) => e.action === options.action); + where.push("action = @action"); + params.action = options.action; } if (options?.source) { - filtered = filtered.filter((e) => e.source === options.source); + where.push("source = @source"); + params.source = options.source; } if (options?.since) { - filtered = filtered.filter((e) => e.timestamp >= options.since!); + where.push("timestamp >= @since"); + params.since = options.since; } - const total = filtered.length; + const whereSql = where.length > 0 ? `WHERE ${where.join(" AND ")}` : ""; - // Sort newest first - filtered = [...filtered].sort((a, b) => b.timestamp.localeCompare(a.timestamp)); + const totalRow = db() + .prepare(`SELECT COUNT(*) AS c FROM config_audit_log ${whereSql}`) + .get(params) as { c: number }; + const total = totalRow.c; - // Paginate const offset = options?.offset ?? 0; const limit = options?.limit ?? 50; - filtered = filtered.slice(offset, offset + limit); - return { entries: filtered, total }; + const rows = db() + .prepare( + `SELECT * FROM config_audit_log ${whereSql} ORDER BY datetime(timestamp) DESC, id DESC LIMIT @limit OFFSET @offset` + ) + .all({ ...params, limit, offset }) as ConfigAuditRow[]; + + return { entries: rows.map(rowToEntry), total }; } /** * Get a specific audit entry by ID. */ export function getAuditEntry(id: string): ConfigAuditEntry | null { - return auditLog.find((e) => e.id === id) ?? null; + const row = db() + .prepare("SELECT * FROM config_audit_log WHERE id = @id") + .get({ id }) as ConfigAuditRow | undefined; + return row ? rowToEntry(row) : null; } /** @@ -260,19 +321,23 @@ export function getAuditSummary(): { const byAction: Record = {}; const bySource: Record = {}; - for (const entry of auditLog) { - byTarget[entry.target] = (byTarget[entry.target] || 0) + 1; - byAction[entry.action] = (byAction[entry.action] || 0) + 1; - bySource[entry.source] = (bySource[entry.source] || 0) + 1; + const rows = db() + .prepare("SELECT * FROM config_audit_log ORDER BY datetime(timestamp) DESC, id DESC") + .all() as ConfigAuditRow[]; + + for (const row of rows) { + byTarget[row.target] = (byTarget[row.target] || 0) + 1; + byAction[row.action] = (byAction[row.action] || 0) + 1; + bySource[row.source] = (bySource[row.source] || 0) + 1; } return { - totalEntries: auditLog.length, + totalEntries: rows.length, byTarget, byAction, bySource, - oldestEntry: auditLog.length > 0 ? auditLog[0].timestamp : null, - newestEntry: auditLog.length > 0 ? auditLog[auditLog.length - 1].timestamp : null, + oldestEntry: rows.length > 0 ? rows[rows.length - 1].timestamp : null, + newestEntry: rows.length > 0 ? rows[0].timestamp : null, }; } @@ -280,6 +345,6 @@ export function getAuditSummary(): { * Reset the audit log. Useful for testing. */ export function resetAuditLog(): void { - auditLog = []; + db().prepare("DELETE FROM config_audit_log").run(); idCounter = 0; } diff --git a/src/lib/db/cleanup.ts b/src/lib/db/cleanup.ts index 617bf4c226..60e288bf4b 100644 --- a/src/lib/db/cleanup.ts +++ b/src/lib/db/cleanup.ts @@ -193,6 +193,31 @@ export async function cleanupMcpAudit(): Promise { return result; } +/** + * Clean up old config_audit_log based on retention settings. + */ +export async function cleanupConfigAudit(retentionDays = getRetentionSettings().configAudit): Promise { + const db = getDbInstance(); + const result: CleanupResult = { deleted: 0, errors: 0 }; + + try { + const stmt = db.prepare( + "DELETE FROM config_audit_log WHERE datetime(timestamp) < datetime('now', '-' || ? || ' days')" + ); + const runResult = stmt.run(String(retentionDays)); + result.deleted = runResult.changes; + + console.log( + `[Cleanup] Deleted ${result.deleted} config_audit_log older than ${retentionDays} days` + ); + } catch (err: unknown) { + console.error("[Cleanup] Error cleaning config_audit_log:", err); + result.errors++; + } + + return result; +} + /** * Clean up old a2a_task_events based on retention settings. */ @@ -420,6 +445,7 @@ export async function runAutoCleanup(): Promise<{ usageHistory: await cleanupUsageHistory(), compressionAnalytics: await cleanupCompressionAnalytics(), mcpAudit: await cleanupMcpAudit(), + configAudit: await cleanupConfigAudit(), a2aEvents: await cleanupA2aEvents(), memoryEntries: await cleanupMemoryEntries(), domainCostHistory: await cleanupDomainCostHistory(), diff --git a/src/lib/db/databaseSettings.ts b/src/lib/db/databaseSettings.ts index e18f2a66bd..0a12729242 100644 --- a/src/lib/db/databaseSettings.ts +++ b/src/lib/db/databaseSettings.ts @@ -46,6 +46,7 @@ const LEGACY_FLAT_KEYS: { quotaSnapshots: ["quotaSnapshots"], compressionAnalytics: ["compressionAnalytics"], mcpAudit: ["mcpAudit"], + configAudit: ["configAudit"], a2aEvents: ["a2aEvents"], callLogs: ["callLogs"], usageHistory: ["usageHistory"], diff --git a/src/lib/db/migrations/046_database_settings.sql b/src/lib/db/migrations/046_database_settings.sql index 57fd15c903..6fb9864390 100644 --- a/src/lib/db/migrations/046_database_settings.sql +++ b/src/lib/db/migrations/046_database_settings.sql @@ -25,6 +25,7 @@ INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSetting INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'quotaSnapshots', '90'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'compressionAnalytics', '30'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mcpAudit', '30'); +INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'configAudit', '30'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'a2aEvents', '30'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogs', '90'); INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'usageHistory', '365'); diff --git a/src/lib/db/migrations/161_config_audit_log.sql b/src/lib/db/migrations/161_config_audit_log.sql new file mode 100644 index 0000000000..0aad91ace2 --- /dev/null +++ b/src/lib/db/migrations/161_config_audit_log.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS config_audit_log ( + id TEXT PRIMARY KEY, + timestamp TEXT NOT NULL, + action TEXT NOT NULL, + target TEXT NOT NULL, + target_id TEXT NOT NULL, + target_name TEXT NOT NULL, + before_json TEXT, + after_json TEXT, + diff_json TEXT NOT NULL, + source TEXT NOT NULL, + note TEXT +); +CREATE INDEX IF NOT EXISTS idx_config_audit_log_target_created ON config_audit_log(target, timestamp); +CREATE INDEX IF NOT EXISTS idx_config_audit_log_created ON config_audit_log(timestamp); diff --git a/src/types/databaseSettings.ts b/src/types/databaseSettings.ts index 6bcc210e5f..2b4820e198 100644 --- a/src/types/databaseSettings.ts +++ b/src/types/databaseSettings.ts @@ -44,6 +44,7 @@ export interface DatabaseSettings { quotaSnapshots: number; compressionAnalytics: number; mcpAudit: number; + configAudit: number; a2aEvents: number; callLogs: number; usageHistory: number; @@ -114,6 +115,7 @@ export const DEFAULT_DATABASE_SETTINGS: Omit { + resetStorage(); +}); + +test.after(() => { + resetStorage(); +}); + +test("recordChange persists to SQLite, not memory", () => { + const db = core.getDbInstance(); + const tableRow = db + .prepare("SELECT count(*) as c FROM sqlite_master WHERE type='table' AND name='config_audit_log'") + .get() as CountRow; + assert.equal(tableRow.c, 1); + + const e = audit.recordChange("update", "provider", "p1", "My Provider", { a: 1 }, { a: 2 }, "api", null); + assert.equal(countRows(), 1); + + const { entries, total } = audit.getAuditLog({ target: "provider" }); + assert.equal(total, 1); + assert.equal(entries[0].id, e.id); + assert.deepEqual(entries[0].diff.changed, [{ key: "a", from: 1, to: 2 }]); +}); + +test("pagination + filters read from SQLite", () => { + audit.recordChange("create", "combo", "c1", "C1", null, { models: ["m1"] }, "dashboard"); + audit.recordChange("update", "combo", "c1", "C1", { models: ["m1"] }, { models: ["m1", "m2"] }, "api"); + + const { entries, total } = audit.getAuditLog({ target: "combo", limit: 1, offset: 0 }); + assert.equal(total, 2); + assert.equal(entries.length, 1); +}); + +test("getRollbackState returns the before snapshot", () => { + const e = audit.recordChange("update", "policy", "pol1", "Pol", { x: 1 }, { x: 2 }, "api"); + assert.deepEqual(audit.getRollbackState(e.id), { x: 1 }); +}); + +test("computeDiff stays pure", () => { + const d = audit.computeDiff({ a: 1 }, { a: 2, b: 3 }); + assert.deepEqual(d.added, ["b"]); + assert.deepEqual(d.changed, [{ key: "a", from: 1, to: 2 }]); +}); + +test("resetAuditLog clears persisted rows", () => { + audit.recordChange("update", "provider", "p1", "P1", { a: 1 }, { a: 2 }, "api"); + assert.equal(countRows(), 1); + audit.resetAuditLog(); + assert.equal(countRows(), 0); +}); + +test("cleanupConfigAudit prunes rows beyond retentionDays", async () => { + insertOldRow("audit-old", 40); + const r = await cleanup.cleanupConfigAudit(30); + assert.equal(r.deleted, 1); + assert.equal(countRows(), 0); +}); + +test("cleanupConfigAudit keeps recent rows within retention", async () => { + insertOldRow("audit-recent", 5); + const r = await cleanup.cleanupConfigAudit(30); + assert.equal(r.deleted, 0); + assert.equal(countRows(), 1); +}); + +test("runAutoCleanup includes a configAudit result", async () => { + insertOldRow("audit-old-2", 40); + const result = await cleanup.runAutoCleanup(); + assert.ok(result.results.configAudit); + assert.equal(typeof result.results.configAudit.deleted, "number"); + assert.equal(typeof result.results.configAudit.errors, "number"); + assert.equal(result.results.configAudit.deleted, 1); + assert.equal(countRows(), 0); +});