Files
OmniRoute/src/lib/db/backupRetention.ts
Diego Rodrigues de Sa e Souza 4a53a53277 fix(db): prune pre-migration backups so db_backups stops growing unbounded (#10423)
* fix(db): prune pre-migration backups so db_backups stops growing unbounded

createPreMigrationBackup() wrote a VACUUM INTO snapshot on every migration run
and never pruned. On a long-lived instance db_backups/ reached 48.999 files /
204 GB against a 5,3 MB live database; a second devbox showed the same shape
(5.711 files / 24 GB).

The retention policy already existed in cleanupDbBackups() but nothing on the
migration path reached it — its only callers are backup.ts and the
/api/db-backups route, neither of which runs during a migration.

migrationRunner.ts cannot import backup.ts: core.ts imports migrationRunner.ts
and backup.ts imports core.ts, so that edge would close a cycle. The policy
therefore moves to a new core-free module, backupRetention.ts, which both call
sites share — cleanupDbBackups() now delegates to it rather than duplicating it.

At the migration call site the operator's maxFiles/retentionDays are read
through the adapter already open for the run; going through getDbInstance()
would re-enter database initialization. Pruning never throws, so housekeeping
cannot fail a migration.

Closes #10421

* chore(db): declare backupRetention as an intentionally-internal db module

check:db-rules requires every src/lib/db/ module to be either re-exported by
localDb.ts or listed in INTENTIONALLY_INTERNAL. backupRetention.ts is a shared
primitive consumed only by db/backup.ts and db/migrationRunner.ts — the same
category as the migrationRunner entry — so it belongs in the allowlist rather
than in the public re-export surface.

* test(db): include backupRetention in the audited INTENTIONALLY_INTERNAL list

check-db-rules-classification.test.ts freezes the exact membership of
INTENTIONALLY_INTERNAL, so adding the 40th entry has to be reflected there too
— the gate script and this test pin the same contract from opposite sides.

---------

Co-authored-by: Xiangzhe <bakryun0718@proton.me>
2026-08-14 21:27:16 -03:00

159 lines
4.8 KiB
TypeScript

/**
* Backup retention primitives — pure filesystem work, no `core.ts` dependency.
*
* This module exists so BOTH backup call sites can share one retention policy:
*
* - `backup.ts` (manual/API/auto backups) — resolves the operator's settings from the
* database and delegates here.
* - `migrationRunner.ts` (pre-migration snapshots) — cannot import `backup.ts`, because
* `core.ts` already imports `migrationRunner.ts` and `backup.ts` imports `core.ts`;
* that edge would close a cycle. Keeping the policy here, free of `core`, lets the
* migration path prune without one.
*
* Before #10421 the migration path had no retention at all and `db_backups/` grew
* without bound (observed: 48.999 files / 204 GB against a 5,3 MB live database).
*/
import fs from "fs";
import path from "path";
export const MAX_DB_BACKUPS = 20;
export const DEFAULT_DB_BACKUP_RETENTION_DAYS = 0;
export function parsePositiveInt(value: string | undefined, fallback: number) {
if (!value) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}
export function parseNonNegativeInt(value: string | undefined, fallback: number) {
if (value === undefined) return fallback;
const parsed = Number.parseInt(value, 10);
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
}
/**
* A backup "family" is the primary `.sqlite` file plus its SQLite sidecars
* (`-wal` / `-shm` / `-journal`). Retention operates on families so a sidecar is never
* orphaned from — or outlives — the snapshot it belongs to.
*/
export function getBackupFamilyBase(filename: string) {
if (filename.endsWith("-wal") || filename.endsWith("-shm")) return filename.slice(0, -4);
if (filename.endsWith("-journal")) return filename.slice(0, -8);
return filename;
}
export type BackupFamily = {
base: string;
hasPrimary: boolean;
primaryMtimeMs: number;
latestMtimeMs: number;
files: string[];
};
export function collectBackupFamilies(backupDir: string): BackupFamily[] {
if (!fs.existsSync(backupDir)) return [];
const families = new Map<string, BackupFamily>();
for (const name of fs.readdirSync(backupDir)) {
if (!name.startsWith("db_")) continue;
const base = getBackupFamilyBase(name);
const filePath = path.join(backupDir, name);
let stat;
try {
stat = fs.statSync(filePath);
} catch {
continue;
}
const family = families.get(base) || {
base,
hasPrimary: false,
primaryMtimeMs: 0,
latestMtimeMs: 0,
files: [],
};
family.files.push(name);
family.latestMtimeMs = Math.max(family.latestMtimeMs, stat.mtimeMs);
if (name === base && name.endsWith(".sqlite")) {
family.hasPrimary = true;
family.primaryMtimeMs = stat.mtimeMs;
}
families.set(base, family);
}
return [...families.values()];
}
export type PruneResult = {
deletedBackupFamilies: number;
deletedFiles: number;
keptBackupFamilies: number;
maxFiles: number;
retentionDays: number;
};
/**
* Delete backup families beyond `maxFiles` (newest kept), older than `retentionDays`
* (0 disables the age rule), or orphaned (sidecars whose primary is already gone).
*/
export function pruneBackupDirectory(options: {
backupDir: string;
maxFiles: number;
retentionDays: number;
}): PruneResult {
const { backupDir } = options;
const maxFiles = Math.max(1, options.maxFiles);
const retentionDays = Math.max(0, options.retentionDays);
if (!fs.existsSync(backupDir)) {
return {
deletedBackupFamilies: 0,
deletedFiles: 0,
keptBackupFamilies: 0,
maxFiles,
retentionDays,
};
}
const cutoffMs = retentionDays > 0 ? Date.now() - retentionDays * 24 * 60 * 60 * 1000 : 0;
const families = collectBackupFamilies(backupDir);
const primaryFamilies = families
.filter((family) => family.hasPrimary)
.sort((a, b) => b.primaryMtimeMs - a.primaryMtimeMs);
const keepPrimaryBases = new Set(primaryFamilies.slice(0, maxFiles).map((family) => family.base));
let deletedBackupFamilies = 0;
let deletedFiles = 0;
for (const family of families) {
const isOverflowPrimary = family.hasPrimary && !keepPrimaryBases.has(family.base);
const isExpired = retentionDays > 0 && family.latestMtimeMs < cutoffMs;
const isOrphan = !family.hasPrimary;
if (!isOverflowPrimary && !isExpired && !isOrphan) continue;
deletedBackupFamilies += 1;
for (const name of family.files) {
try {
fs.unlinkSync(path.join(backupDir, name));
deletedFiles += 1;
} catch {
/* ignore */
}
}
}
return {
deletedBackupFamilies,
deletedFiles,
keptBackupFamilies: collectBackupFamilies(backupDir).filter((family) => family.hasPrimary)
.length,
maxFiles,
retentionDays,
};
}