diff --git a/changelog.d/fixes/10736-corrupt-rotate-fence.md b/changelog.d/fixes/10736-corrupt-rotate-fence.md new file mode 100644 index 0000000000..dd2abc4fc4 --- /dev/null +++ b/changelog.d/fixes/10736-corrupt-rotate-fence.md @@ -0,0 +1 @@ +- **fix(db):** pause call-log rotation and record SQLITE_CORRUPT on `/api/db/health` instead of retrying writes against a malformed pager ([#10736](https://github.com/diegosouzapw/OmniRoute/issues/10736)) diff --git a/src/lib/db/healthCheck.ts b/src/lib/db/healthCheck.ts index 3916580036..cb46d28e08 100644 --- a/src/lib/db/healthCheck.ts +++ b/src/lib/db/healthCheck.ts @@ -48,6 +48,53 @@ export function describeDbDriver(db: Pick): Db }; } +export interface PagerCorruptionNote { + source: string; + message: string; + at: string; +} + +let pagerCorruption: PagerCorruptionNote | null = null; + +function pagerErrorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + if (error && typeof error === "object" && "message" in error) { + return String((error as { message?: unknown }).message ?? ""); + } + return String(error ?? "unknown"); +} + +export function isSqlitePagerCorruptError(error: unknown): boolean { + const code = + error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code || "") + : ""; + const message = pagerErrorMessage(error); + return ( + code === "SQLITE_CORRUPT" || + code === "SQLITE_NOTADB" || + code === "SQLITE_IOERR" || + /malformed|SQLITE_CORRUPT|SQLITE_NOTADB|SQLITE_IOERR/i.test(message) + ); +} + +export function notePagerCorruption(source: string, error: unknown): void { + const message = pagerErrorMessage(error) || "unknown"; + pagerCorruption = { + source, + message, + at: new Date().toISOString(), + }; +} + +export function getPagerCorruption(): PagerCorruptionNote | null { + return pagerCorruption; +} + +export function resetPagerCorruption(): void { + pagerCorruption = null; +} + interface RunDbHealthCheckOptions { autoRepair?: boolean; createBackupBeforeRepair?: () => boolean; @@ -425,6 +472,14 @@ export function runDbHealthCheck( const expectedSchemaVersion = options.expectedSchemaVersion || "1"; const checkedAt = new Date().toISOString(); const issues: DbHealthIssue[] = []; + if (pagerCorruption) { + issues.push({ + type: "integrity_check_failed", + table: "sqlite", + description: `Pager reported SQLITE_CORRUPT during ${pagerCorruption.source}: ${pagerCorruption.message}`, + count: 1, + }); + } let repairedCount = 0; let backupCreated = false; let backupAttempted = false; diff --git a/src/lib/usage/callLogRotation.ts b/src/lib/usage/callLogRotation.ts index a5d63a5d7e..58ac3066ad 100644 --- a/src/lib/usage/callLogRotation.ts +++ b/src/lib/usage/callLogRotation.ts @@ -20,6 +20,7 @@ import { type CallLogDetailState, } from "./callLogArtifacts"; import { getCallLogMaxEntries, getCallLogRetentionDays, getCallLogsTableMaxRows } from "../logEnv"; +import { isSqlitePagerCorruptError, notePagerCorruption } from "../db/healthCheck"; const CALL_LOG_ROTATE_THROTTLE_MS = 60_000; const CALL_LOG_ROTATE_BATCH_SIZE = 100; @@ -308,7 +309,29 @@ export function trimCallLogsToMaxRows( return { deletedRows, deletedArtifacts }; } +let callLogRotatePaused = false; + +export function isCallLogRotatePaused(): boolean { + return callLogRotatePaused; +} + +export function resetCallLogRotateFence(): void { + callLogRotatePaused = false; +} + +export function handleCallLogRotateError(error: unknown): void { + const message = error instanceof Error ? error.message : String(error); + console.error("[callLogs] Failed to rotate request artifacts:", message); + if (!isSqlitePagerCorruptError(error)) return; + callLogRotatePaused = true; + notePagerCorruption("call-log-rotate", error); + console.error( + "[callLogs] SQLITE_CORRUPT during rotation; pausing further rotate writes. Check /api/db/health." + ); +} + export function rotateCallLogs() { + if (callLogRotatePaused) return; try { if (!CALL_LOGS_DIR || !fs.existsSync(CALL_LOGS_DIR)) return; @@ -324,7 +347,7 @@ export function rotateCallLogs() { minAgeMs: CALL_LOG_ORPHAN_MIN_AGE_MS, }); } catch (error) { - console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message); + handleCallLogRotateError(error); } } @@ -335,7 +358,7 @@ function runScheduledCallLogRotation() { try { rotateCallLogs(); } catch (error) { - console.error("[callLogs] Failed to rotate request artifacts:", (error as Error).message); + handleCallLogRotateError(error); } finally { callLogRotateInFlight = false; } diff --git a/tests/unit/call-log-rotate-corrupt.test.ts b/tests/unit/call-log-rotate-corrupt.test.ts new file mode 100644 index 0000000000..a2c6aa833a --- /dev/null +++ b/tests/unit/call-log-rotate-corrupt.test.ts @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + getPagerCorruption, + isSqlitePagerCorruptError, + notePagerCorruption, + resetPagerCorruption, +} from "../../src/lib/db/healthCheck.ts"; +import { + handleCallLogRotateError, + isCallLogRotatePaused, + resetCallLogRotateFence, +} from "../../src/lib/usage/callLogRotation.ts"; + +test("isSqlitePagerCorruptError matches SQLITE_CORRUPT and malformed pager errors", () => { + assert.equal(isSqlitePagerCorruptError({ code: "SQLITE_CORRUPT", message: "x" }), true); + assert.equal( + isSqlitePagerCorruptError(new Error("database disk image is malformed")), + true + ); + assert.equal(isSqlitePagerCorruptError(new Error("SQLITE_IOERR")), true); + assert.equal(isSqlitePagerCorruptError(new Error("simulated opendir failure")), false); +}); + +test("notePagerCorruption surfaces on the next health check snapshot", () => { + resetPagerCorruption(); + notePagerCorruption("call-log-rotate", { code: "SQLITE_CORRUPT", message: "malformed" }); + const noted = getPagerCorruption(); + assert.ok(noted); + assert.equal(noted.source, "call-log-rotate"); + assert.match(noted.message, /malformed|SQLITE_CORRUPT/); + resetPagerCorruption(); + assert.equal(getPagerCorruption(), null); +}); + +test("handleCallLogRotateError fences further rotation on SQLITE_CORRUPT", () => { + resetCallLogRotateFence(); + resetPagerCorruption(); + const corrupt = Object.assign(new Error("database disk image is malformed"), { + code: "SQLITE_CORRUPT", + }); + handleCallLogRotateError(corrupt); + assert.equal(isCallLogRotatePaused(), true); + assert.ok(getPagerCorruption()); + resetCallLogRotateFence(); + resetPagerCorruption(); +}); + +test("handleCallLogRotateError does not fence ordinary filesystem errors", () => { + resetCallLogRotateFence(); + resetPagerCorruption(); + handleCallLogRotateError(new Error("simulated opendir failure")); + assert.equal(isCallLogRotatePaused(), false); + assert.equal(getPagerCorruption(), null); +});