fix(config-audit): persist config audit log to SQLite with retention (#11103)

Validated on the combined batch board over release/v3.8.50 tip d91238b7: static gates clean, typecheck:core clean, focused tests green.

Config-audit survives restarts with bounded growth (migration 161 + OR IGNORE seed, retention wired into runAutoCleanup); route cabling deliberately out of scope. Thank you @maxmad64bis!

changelog.d/fixes/11103-persist-config-audit-log.md

@@ -0,0 +1 @@
+- **Config audit persistence:** persist the configuration audit trail to SQLite (`config_audit_log`) instead of an in-memory buffer capped at 1000 volatile entries, and bound its growth with `cleanupConfigAudit()` driven by the `retention.configAudit` setting (default 30 days), wired into `runAutoCleanup` ([#11103](https://github.com/diegosouzapw/OmniRoute/pull/11103)).

src/domain/configAudit.ts

@@ -13,6 +13,8 @@
  * - Optional human notes
  */
 
+import { getDbInstance } from "../lib/db/core";
+
 /** Types of configuration entities that can be audited */
 export type AuditTarget = "provider" | "combo" | "policy" | "connection" | "settings";
 
@@ -72,10 +74,8 @@ export interface ConfigSnapshot {
   data: Record<string, unknown>;
 }
 
-// ── In-memory store ──────────────────────────────────────────────────────────
-// In production, persist to SQLite alongside other domain state.
+// ── SQLite-backed store ───────────────────────────────────────────────────────
 
-let auditLog: ConfigAuditEntry[] = [];
 let idCounter = 0;
 
 function generateId(): string {
@@ -85,6 +85,40 @@ function generateId(): string {
   return `audit-${ts}-${seq}`;
 }
 
+function db() {
+  return getDbInstance();
+}
+
+interface ConfigAuditRow {
+  id: string;
+  timestamp: string;
+  action: string;
+  target: string;
+  target_id: string;
+  target_name: string;
+  before_json: string | null;
+  after_json: string | null;
+  diff_json: string;
+  source: string;
+  note: string | null;
+}
+
+function rowToEntry(row: ConfigAuditRow): ConfigAuditEntry {
+  return {
+    id: row.id,
+    timestamp: row.timestamp,
+    action: row.action as AuditAction,
+    target: row.target as AuditTarget,
+    targetId: row.target_id,
+    targetName: row.target_name,
+    before: row.before_json === null ? null : (JSON.parse(row.before_json) as Record<string, unknown> | null),
+    after: row.after_json === null ? null : (JSON.parse(row.after_json) as Record<string, unknown> | null),
+    source: row.source as AuditSource,
+    diff: JSON.parse(row.diff_json) as ConfigDiff,
+    note: row.note,
+  };
+}
+
 /**
  * Compute a structured diff between two configuration states.
  */
@@ -159,12 +193,24 @@ export function recordChange(
     note: note ?? null,
   };
 
-  auditLog.push(entry);
-
-  // Keep log bounded (max 1000 entries in memory)
-  if (auditLog.length > 1000) {
-    auditLog = auditLog.slice(-1000);
-  }
+  db().prepare(
+    `INSERT INTO config_audit_log
+       (id, timestamp, action, target, target_id, target_name, before_json, after_json, diff_json, source, note)
+     VALUES
+       (@id, @timestamp, @action, @target, @targetId, @targetName, @beforeJson, @afterJson, @diffJson, @source, @note)`
+  ).run({
+    id: entry.id,
+    timestamp: entry.timestamp,
+    action: entry.action,
+    target: entry.target,
+    targetId: entry.targetId,
+    targetName: entry.targetName,
+    beforeJson: before === null ? null : JSON.stringify(before),
+    afterJson: after === null ? null : JSON.stringify(after),
+    diffJson: JSON.stringify(entry.diff),
+    source: entry.source,
+    note: entry.note,
+  });
 
   return entry;
 }
@@ -181,42 +227,57 @@ export function getAuditLog(options?: {
   limit?: number;
   offset?: number;
 }): { entries: ConfigAuditEntry[]; total: number } {
-  let filtered = auditLog;
+  const where: string[] = [];
+  const params: Record<string, unknown> = {};
 
   if (options?.target) {
-    filtered = filtered.filter((e) => e.target === options.target);
+    where.push("target = @target");
+    params.target = options.target;
   }
   if (options?.targetId) {
-    filtered = filtered.filter((e) => e.targetId === options.targetId);
+    where.push("target_id = @targetId");
+    params.targetId = options.targetId;
   }
   if (options?.action) {
-    filtered = filtered.filter((e) => e.action === options.action);
+    where.push("action = @action");
+    params.action = options.action;
   }
   if (options?.source) {
-    filtered = filtered.filter((e) => e.source === options.source);
+    where.push("source = @source");
+    params.source = options.source;
   }
   if (options?.since) {
-    filtered = filtered.filter((e) => e.timestamp >= options.since!);
+    where.push("timestamp >= @since");
+    params.since = options.since;
   }
 
-  const total = filtered.length;
+  const whereSql = where.length > 0 ? `WHERE ${where.join(" AND ")}` : "";
 
-  // Sort newest first
-  filtered = [...filtered].sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+  const totalRow = db()
+    .prepare(`SELECT COUNT(*) AS c FROM config_audit_log ${whereSql}`)
+    .get(params) as { c: number };
+  const total = totalRow.c;
 
-  // Paginate
   const offset = options?.offset ?? 0;
   const limit = options?.limit ?? 50;
-  filtered = filtered.slice(offset, offset + limit);
 
-  return { entries: filtered, total };
+  const rows = db()
+    .prepare(
+      `SELECT * FROM config_audit_log ${whereSql} ORDER BY datetime(timestamp) DESC, id DESC LIMIT @limit OFFSET @offset`
+    )
+    .all({ ...params, limit, offset }) as ConfigAuditRow[];
+
+  return { entries: rows.map(rowToEntry), total };
 }
 
 /**
  * Get a specific audit entry by ID.
  */
 export function getAuditEntry(id: string): ConfigAuditEntry | null {
-  return auditLog.find((e) => e.id === id) ?? null;
+  const row = db()
+    .prepare("SELECT * FROM config_audit_log WHERE id = @id")
+    .get({ id }) as ConfigAuditRow | undefined;
+  return row ? rowToEntry(row) : null;
 }
 
 /**
@@ -260,19 +321,23 @@ export function getAuditSummary(): {
   const byAction: Record<string, number> = {};
   const bySource: Record<string, number> = {};
 
-  for (const entry of auditLog) {
-    byTarget[entry.target] = (byTarget[entry.target] || 0) + 1;
-    byAction[entry.action] = (byAction[entry.action] || 0) + 1;
-    bySource[entry.source] = (bySource[entry.source] || 0) + 1;
+  const rows = db()
+    .prepare("SELECT * FROM config_audit_log ORDER BY datetime(timestamp) DESC, id DESC")
+    .all() as ConfigAuditRow[];
+
+  for (const row of rows) {
+    byTarget[row.target] = (byTarget[row.target] || 0) + 1;
+    byAction[row.action] = (byAction[row.action] || 0) + 1;
+    bySource[row.source] = (bySource[row.source] || 0) + 1;
   }
 
   return {
-    totalEntries: auditLog.length,
+    totalEntries: rows.length,
     byTarget,
     byAction,
     bySource,
-    oldestEntry: auditLog.length > 0 ? auditLog[0].timestamp : null,
-    newestEntry: auditLog.length > 0 ? auditLog[auditLog.length - 1].timestamp : null,
+    oldestEntry: rows.length > 0 ? rows[rows.length - 1].timestamp : null,
+    newestEntry: rows.length > 0 ? rows[0].timestamp : null,
   };
 }
 
@@ -280,6 +345,6 @@ export function getAuditSummary(): {
  * Reset the audit log. Useful for testing.
  */
 export function resetAuditLog(): void {
-  auditLog = [];
+  db().prepare("DELETE FROM config_audit_log").run();
   idCounter = 0;
 }

src/lib/db/cleanup.ts

@@ -193,6 +193,31 @@ export async function cleanupMcpAudit(): Promise<CleanupResult> {
   return result;
 }
 
+/**
+ * Clean up old config_audit_log based on retention settings.
+ */
+export async function cleanupConfigAudit(retentionDays = getRetentionSettings().configAudit): Promise<CleanupResult> {
+  const db = getDbInstance();
+  const result: CleanupResult = { deleted: 0, errors: 0 };
+
+  try {
+    const stmt = db.prepare(
+      "DELETE FROM config_audit_log WHERE datetime(timestamp) < datetime('now', '-' || ? || ' days')"
+    );
+    const runResult = stmt.run(String(retentionDays));
+    result.deleted = runResult.changes;
+
+    console.log(
+      `[Cleanup] Deleted ${result.deleted} config_audit_log older than ${retentionDays} days`
+    );
+  } catch (err: unknown) {
+    console.error("[Cleanup] Error cleaning config_audit_log:", err);
+    result.errors++;
+  }
+
+  return result;
+}
+
 /**
  * Clean up old a2a_task_events based on retention settings.
  */
@@ -420,6 +445,7 @@ export async function runAutoCleanup(): Promise<{
     usageHistory: await cleanupUsageHistory(),
     compressionAnalytics: await cleanupCompressionAnalytics(),
     mcpAudit: await cleanupMcpAudit(),
+    configAudit: await cleanupConfigAudit(),
     a2aEvents: await cleanupA2aEvents(),
     memoryEntries: await cleanupMemoryEntries(),
     domainCostHistory: await cleanupDomainCostHistory(),

src/lib/db/databaseSettings.ts

@@ -46,6 +46,7 @@ const LEGACY_FLAT_KEYS: {
     quotaSnapshots: ["quotaSnapshots"],
     compressionAnalytics: ["compressionAnalytics"],
     mcpAudit: ["mcpAudit"],
+    configAudit: ["configAudit"],
     a2aEvents: ["a2aEvents"],
     callLogs: ["callLogs"],
     usageHistory: ["usageHistory"],

src/lib/db/migrations/046_database_settings.sql

@@ -25,6 +25,7 @@ INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSetting
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'quotaSnapshots', '90');
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'compressionAnalytics', '30');
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'mcpAudit', '30');
+INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'configAudit', '30');
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'a2aEvents', '30');
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'callLogs', '90');
 INSERT OR IGNORE INTO key_value (namespace, key, value) VALUES ('databaseSettings', 'usageHistory', '365');

src/lib/db/migrations/161_config_audit_log.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS config_audit_log (
+  id TEXT PRIMARY KEY,
+  timestamp TEXT NOT NULL,
+  action TEXT NOT NULL,
+  target TEXT NOT NULL,
+  target_id TEXT NOT NULL,
+  target_name TEXT NOT NULL,
+  before_json TEXT,
+  after_json TEXT,
+  diff_json TEXT NOT NULL,
+  source TEXT NOT NULL,
+  note TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_config_audit_log_target_created ON config_audit_log(target, timestamp);
+CREATE INDEX IF NOT EXISTS idx_config_audit_log_created ON config_audit_log(timestamp);

src/types/databaseSettings.ts

@@ -44,6 +44,7 @@ export interface DatabaseSettings {
     quotaSnapshots: number;
     compressionAnalytics: number;
     mcpAudit: number;
+    configAudit: number;
     a2aEvents: number;
     callLogs: number;
     usageHistory: number;
@@ -114,6 +115,7 @@ export const DEFAULT_DATABASE_SETTINGS: Omit<DatabaseSettings, "location" | "sta
     quotaSnapshots: 7,
     compressionAnalytics: 30,
     mcpAudit: 30,
+    configAudit: 30,
     a2aEvents: 30,
     callLogs: 30,
     usageHistory: 30,

tests/unit/config-audit-persistence.test.ts

@@ -0,0 +1,124 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-config-audit-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
+
+const core = await import("../../src/lib/db/core.ts");
+const cleanup = await import("../../src/lib/db/cleanup.ts");
+const audit = await import("../../src/domain/configAudit.ts");
+
+type CountRow = { c: number };
+
+function resetStorage() {
+  core.resetDbInstance();
+  fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+  fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
+}
+
+function countRows(): number {
+  const db = core.getDbInstance();
+  const row = db.prepare("SELECT COUNT(*) AS c FROM config_audit_log").get() as CountRow;
+  return row.c;
+}
+
+function insertOldRow(id: string, daysAgo: number) {
+  const db = core.getDbInstance();
+  const old = new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1000).toISOString();
+  db.prepare(
+    `INSERT INTO config_audit_log
+       (id, timestamp, action, target, target_id, target_name, before_json, after_json, diff_json, source, note)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+  ).run(
+    id,
+    old,
+    "update",
+    "provider",
+    "p1",
+    "P1",
+    null,
+    null,
+    JSON.stringify({ added: [], removed: [], changed: [], isEmpty: true }),
+    "api",
+    null
+  );
+}
+
+test.beforeEach(() => {
+  resetStorage();
+});
+
+test.after(() => {
+  resetStorage();
+});
+
+test("recordChange persists to SQLite, not memory", () => {
+  const db = core.getDbInstance();
+  const tableRow = db
+    .prepare("SELECT count(*) as c FROM sqlite_master WHERE type='table' AND name='config_audit_log'")
+    .get() as CountRow;
+  assert.equal(tableRow.c, 1);
+
+  const e = audit.recordChange("update", "provider", "p1", "My Provider", { a: 1 }, { a: 2 }, "api", null);
+  assert.equal(countRows(), 1);
+
+  const { entries, total } = audit.getAuditLog({ target: "provider" });
+  assert.equal(total, 1);
+  assert.equal(entries[0].id, e.id);
+  assert.deepEqual(entries[0].diff.changed, [{ key: "a", from: 1, to: 2 }]);
+});
+
+test("pagination + filters read from SQLite", () => {
+  audit.recordChange("create", "combo", "c1", "C1", null, { models: ["m1"] }, "dashboard");
+  audit.recordChange("update", "combo", "c1", "C1", { models: ["m1"] }, { models: ["m1", "m2"] }, "api");
+
+  const { entries, total } = audit.getAuditLog({ target: "combo", limit: 1, offset: 0 });
+  assert.equal(total, 2);
+  assert.equal(entries.length, 1);
+});
+
+test("getRollbackState returns the before snapshot", () => {
+  const e = audit.recordChange("update", "policy", "pol1", "Pol", { x: 1 }, { x: 2 }, "api");
+  assert.deepEqual(audit.getRollbackState(e.id), { x: 1 });
+});
+
+test("computeDiff stays pure", () => {
+  const d = audit.computeDiff({ a: 1 }, { a: 2, b: 3 });
+  assert.deepEqual(d.added, ["b"]);
+  assert.deepEqual(d.changed, [{ key: "a", from: 1, to: 2 }]);
+});
+
+test("resetAuditLog clears persisted rows", () => {
+  audit.recordChange("update", "provider", "p1", "P1", { a: 1 }, { a: 2 }, "api");
+  assert.equal(countRows(), 1);
+  audit.resetAuditLog();
+  assert.equal(countRows(), 0);
+});
+
+test("cleanupConfigAudit prunes rows beyond retentionDays", async () => {
+  insertOldRow("audit-old", 40);
+  const r = await cleanup.cleanupConfigAudit(30);
+  assert.equal(r.deleted, 1);
+  assert.equal(countRows(), 0);
+});
+
+test("cleanupConfigAudit keeps recent rows within retention", async () => {
+  insertOldRow("audit-recent", 5);
+  const r = await cleanup.cleanupConfigAudit(30);
+  assert.equal(r.deleted, 0);
+  assert.equal(countRows(), 1);
+});
+
+test("runAutoCleanup includes a configAudit result", async () => {
+  insertOldRow("audit-old-2", 40);
+  const result = await cleanup.runAutoCleanup();
+  assert.ok(result.results.configAudit);
+  assert.equal(typeof result.results.configAudit.deleted, "number");
+  assert.equal(typeof result.results.configAudit.errors, "number");
+  assert.equal(result.results.configAudit.deleted, 1);
+  assert.equal(countRows(), 0);
+});